diff --git a/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch b/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch new file mode 100644 index 0000000000..ef5148b6fc --- /dev/null +++ b/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch @@ -0,0 +1,13 @@ +diff --git a/lib/pool/pool.js b/lib/pool/pool.js +index 659d80681ce11387e3d4eee645c5cbf3df7bcb30..bad49c310096fd02ecaddf907642aec78c44ced8 100644 +--- a/lib/pool/pool.js ++++ b/lib/pool/pool.js +@@ -134,6 +134,8 @@ var Pool = /** @class */ (function () { + request.reject((0, neo4j_driver_core_1.newError)("Connection acquisition timed out in ".concat(_this._acquisitionTimeout, " ms. Pool status: Active conn count = ").concat(activeCount, ", Idle conn count = ").concat(idleCount, "."))); + } + }, _this._acquisitionTimeout); ++ // https://github.com/neo4j/neo4j-javascript-driver/pull/1196 ++ timeoutId.unref(); + request = new PendingRequest(key, acquisitionContext, config, resolve, reject, timeoutId, _this._log); + allRequests[key].push(request); + _this._processPendingAcquireRequests(address); diff --git a/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch b/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch new file mode 100644 index 0000000000..4609926391 --- /dev/null +++ b/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch @@ -0,0 +1,17 @@ +diff --git a/lib/transaction.js b/lib/transaction.js +index 8c97e77e79bc40856fc51c0f0d315fd6e4e5e763..1d16fcac7009d9c506bcdbabdef7890b2c8296f2 100644 +--- a/lib/transaction.js ++++ b/lib/transaction.js +@@ -284,6 +284,12 @@ var Transaction = /** @class */ (function () { + // error will be "acknowledged" by sending a RESET message + // database will then forget about this transaction and cleanup all corresponding resources + // it is thus safe to move this transaction to a FAILED state and disallow any further interactions with it ++ ++ if (this._state === _states.FAILED) { ++ // already failed, nothing to do ++ // if we call onError for each result again, we might run into an infinite loop, that causes an OOM eventually ++ return Promise.resolve(null) ++ } + this._state = _states.FAILED; + this._onClose(); + this._results.forEach(function (result) { diff --git a/package.json b/package.json index 231ed89b2d..d0a4b77e57 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "luxon": "^3.4.3", "mime": "beta", "nanoid": "^4.0.2", - "neo4j-driver": "^5.14.0", + "neo4j-driver": "^5.20.0", "p-retry": "^5.1.2", "pako": "^2.1.0", "pkg-up": "^4.0.0", @@ -150,6 +150,8 @@ }, "resolutions": { "cypher-query-builder/neo4j-driver": "^5.9.0", + "neo4j-driver-bolt-connection@npm:5.20.0": "patch:neo4j-driver-bolt-connection@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch", + "neo4j-driver-core@npm:5.20.0": "patch:neo4j-driver-core@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch", "@apollo/server-plugin-landing-page-graphql-playground": "npm:empty-npm-package@*", "@nestjs/cli/fork-ts-checker-webpack-plugin": "npm:empty-npm-package@*", "@nestjs/cli/webpack": "npm:empty-npm-package@*", diff --git a/src/components/authentication/authentication.edgedb.repository.ts b/src/components/authentication/authentication.edgedb.repository.ts index e3417a0947..267656df69 100644 --- a/src/components/authentication/authentication.edgedb.repository.ts +++ b/src/components/authentication/authentication.edgedb.repository.ts @@ -19,6 +19,7 @@ export class AuthenticationEdgeDBRepository await this.db.waitForConnection({ forever: true, maxTimeout: { seconds: 10 }, + unref: true, }); return await this.getRootUserId(); } diff --git a/src/components/authentication/authentication.repository.ts b/src/components/authentication/authentication.repository.ts index bc587a7426..6a35d88191 100644 --- a/src/components/authentication/authentication.repository.ts +++ b/src/components/authentication/authentication.repository.ts @@ -28,6 +28,7 @@ export class AuthenticationRepository { { forever: true, maxTimeout: { seconds: 10 }, + unref: true, }, async () => { // Ensure the root user exists, if not keep waiting diff --git a/src/core/core.module.ts b/src/core/core.module.ts index 71029caa49..5dc6b94a46 100644 --- a/src/core/core.module.ts +++ b/src/core/core.module.ts @@ -17,6 +17,7 @@ import { ExceptionNormalizer } from './exception/exception.normalizer'; import { GraphqlModule } from './graphql'; import { ResourceModule } from './resources/resource.module'; import { ScalarProviders } from './scalars.resolver'; +import { ShutdownHookProvider } from './shutdown.hook'; import { TimeoutInterceptor } from './timeout.interceptor'; import { TracingModule } from './tracing'; import { ValidationModule } from './validation/validation.module'; @@ -46,6 +47,7 @@ import { WaitResolver } from './wait.resolver'; { provide: APP_INTERCEPTOR, useClass: TimeoutInterceptor }, WaitResolver, ...ScalarProviders, + ShutdownHookProvider, ], controllers: [CoreController], exports: [ @@ -59,6 +61,7 @@ import { WaitResolver } from './wait.resolver'; EmailModule, EventsModule, ResourceModule, + ShutdownHookProvider, TracingModule, ValidationModule, ], diff --git a/src/core/database/database.service.ts b/src/core/database/database.service.ts index 15e983cc06..77ca616797 100644 --- a/src/core/database/database.service.ts +++ b/src/core/database/database.service.ts @@ -1,7 +1,10 @@ import { Injectable } from '@nestjs/common'; import { entries, mapKeys } from '@seedcompany/common'; import { Connection, node, Query, relation } from 'cypher-query-builder'; +import { LazyGetter } from 'lazy-get-decorator'; import { pickBy, startCase } from 'lodash'; +import { Duration } from 'luxon'; +import { defer, firstValueFrom, shareReplay, takeUntil } from 'rxjs'; import { DuplicateException, ID, @@ -15,6 +18,7 @@ import { import { AbortError, retry, RetryOptions } from '~/common/retry'; import { ConfigService } from '../config/config.service'; import { ILogger, Logger } from '../logger'; +import { ShutdownHook } from '../shutdown.hook'; import { DbChanges } from './changes'; import { createBetterError, @@ -56,6 +60,7 @@ export class DatabaseService { constructor( private readonly db: Connection, private readonly config: ConfigService, + private readonly shutdown$: ShutdownHook, @Logger('database:service') private readonly logger: ILogger, ) {} @@ -120,7 +125,20 @@ export class DatabaseService { return q; } - async getServerInfo(): Promise { + async getServerInfo() { + return await firstValueFrom(this.serverInfo$); + } + @LazyGetter() private get serverInfo$() { + return defer(() => this.queryServerInfo()).pipe( + takeUntil(this.shutdown$), + shareReplay({ + refCount: false, + bufferSize: 1, + windowTime: Duration.from('3 mins').toMillis(), + }), + ); + } + private async queryServerInfo(): Promise { // @ts-expect-error Yes this is private, but we have a special use case. // We need to run this query with a session that's not configured to use the // database that may not exist. @@ -171,7 +189,7 @@ export class DatabaseService { if (!dbName || info.databases.some((db) => db.name === dbName)) { return; // already exists or assuming default exists } - await this.runAdminCommand('CREATE', dbName, info); + await this.runAdminCommand('CREATE', dbName); } async dropDb() { @@ -179,14 +197,10 @@ export class DatabaseService { if (!dbName) { return; // don't drop the default db } - await this.runAdminCommand('DROP', dbName, await this.getServerInfo()); + await this.runAdminCommand('DROP', dbName); } - private async runAdminCommand( - action: 'CREATE' | 'DROP', - dbName: string, - _info: ServerInfo, - ) { + private async runAdminCommand(action: 'CREATE' | 'DROP', dbName: string) { // @ts-expect-error Yes this is private, but we have a special use case. // We need to run this query with a session that's not configured to use the // database we are trying to create. diff --git a/src/core/database/migration/migration.module.ts b/src/core/database/migration/migration.module.ts index 2d5dc86aa8..48067fb173 100644 --- a/src/core/database/migration/migration.module.ts +++ b/src/core/database/migration/migration.module.ts @@ -18,9 +18,11 @@ export class MigrationModule implements OnModuleInit { ) {} async onModuleInit() { + const entryCmd = process.argv.join(''); if ( !this.config.dbAutoMigrate || - process.argv.join('').includes('console') + entryCmd.includes('console') || + entryCmd.includes('repl') ) { return; } diff --git a/src/core/shutdown.hook.ts b/src/core/shutdown.hook.ts new file mode 100644 index 0000000000..3137c7eff2 --- /dev/null +++ b/src/core/shutdown.hook.ts @@ -0,0 +1,19 @@ +import { OnApplicationShutdown, Provider } from '@nestjs/common'; +import { Observable, Subject } from 'rxjs'; + +export class ShutdownHook extends Observable {} + +class ShutdownHookImpl + extends Subject + implements OnApplicationShutdown, ShutdownHook +{ + onApplicationShutdown() { + this.next(); + this.complete(); + } +} + +export const ShutdownHookProvider: Provider = { + provide: ShutdownHook, + useClass: ShutdownHookImpl, +}; diff --git a/yarn.lock b/yarn.lock index 37a525cdca..5f2a2a0cac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5430,7 +5430,7 @@ __metadata: luxon: "npm:^3.4.3" mime: "npm:beta" nanoid: "npm:^4.0.2" - neo4j-driver: "npm:^5.14.0" + neo4j-driver: "npm:^5.20.0" p-retry: "npm:^5.1.2" pako: "npm:^2.1.0" pkg-up: "npm:^4.0.0" @@ -10311,32 +10311,50 @@ __metadata: languageName: node linkType: hard -"neo4j-driver-bolt-connection@npm:5.14.0": - version: 5.14.0 - resolution: "neo4j-driver-bolt-connection@npm:5.14.0" +"neo4j-driver-bolt-connection@npm:5.20.0": + version: 5.20.0 + resolution: "neo4j-driver-bolt-connection@npm:5.20.0" dependencies: buffer: "npm:^6.0.3" - neo4j-driver-core: "npm:5.14.0" + neo4j-driver-core: "npm:5.20.0" string_decoder: "npm:^1.3.0" - checksum: 10c0/5e070a4b307473ffc51d90fc3423e173af3e8ce11b398feca5e86b22655e3ed8b113af8e13e1fae2e08a1bea24a00db84c0c6fbee37b6897589a15ebf712348b + checksum: 10c0/a60b51192a37a5c86b7fcdbac4ef56a647c3ebe8596ab813b7354188ee3659426f14aaad7fa1025ebc5e07eecabe89c8f2e33d95663c9cd8e49263c12c287c22 languageName: node linkType: hard -"neo4j-driver-core@npm:5.14.0": - version: 5.14.0 - resolution: "neo4j-driver-core@npm:5.14.0" - checksum: 10c0/e432a8e7054ba7ecd4d4767bcb0d5a8ea79c35235ae383c5b5fb6195920714e8c124659cc116493e60b86565f4a3ab7e32211f14ea175cb659451327e6309df6 +"neo4j-driver-bolt-connection@patch:neo4j-driver-bolt-connection@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch": + version: 5.20.0 + resolution: "neo4j-driver-bolt-connection@patch:neo4j-driver-bolt-connection@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-bolt-connection-npm-5.20.0-1f7809f435.patch::version=5.20.0&hash=c7fd0b" + dependencies: + buffer: "npm:^6.0.3" + neo4j-driver-core: "npm:5.20.0" + string_decoder: "npm:^1.3.0" + checksum: 10c0/81c33ad9203a1d948deffca4d98b6ed162e46f9bdf1e2867fa4e9361a6e63c7b61208432d84ccd7ef9c0fcac7ff8b9e8ccc80226a28d102f195fe53d5d517347 + languageName: node + linkType: hard + +"neo4j-driver-core@npm:5.20.0": + version: 5.20.0 + resolution: "neo4j-driver-core@npm:5.20.0" + checksum: 10c0/162ef4953bf04643c7d21b777b5cc0a9fb01aad7e9098bae5eb272de9d88d877808c43406f04db4c245306f501af31f7f5d9d3115f0f635758af394bae5fba17 + languageName: node + linkType: hard + +"neo4j-driver-core@patch:neo4j-driver-core@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch": + version: 5.20.0 + resolution: "neo4j-driver-core@patch:neo4j-driver-core@npm%3A5.20.0#~/.yarn/patches/neo4j-driver-core-npm-5.20.0-99216f6938.patch::version=5.20.0&hash=60e3d3" + checksum: 10c0/957b21375a430ae1973dac04adcdd2806595e5811c70ed46d2273f6df36c977ce94a7dedbf0dd2d7b5bf724ae06c1f86a29550ddabd6c532304257ecbe74a0c8 languageName: node linkType: hard -"neo4j-driver@npm:^5.14.0, neo4j-driver@npm:^5.9.0": - version: 5.14.0 - resolution: "neo4j-driver@npm:5.14.0" +"neo4j-driver@npm:^5.20.0, neo4j-driver@npm:^5.9.0": + version: 5.20.0 + resolution: "neo4j-driver@npm:5.20.0" dependencies: - neo4j-driver-bolt-connection: "npm:5.14.0" - neo4j-driver-core: "npm:5.14.0" + neo4j-driver-bolt-connection: "npm:5.20.0" + neo4j-driver-core: "npm:5.20.0" rxjs: "npm:^7.8.1" - checksum: 10c0/e6b5def70af63a756c23e0e10153c95bf454b07540024bbe166b9b5bd954cfca0e79c2be101ec7bf1832d1a58650ac74acb330f903e40b3213a7727a83cb64d0 + checksum: 10c0/1ed4cc3bcf4d0a5f6742aa5a2224ee1f1006ccedd5e62e3c15d612e4dd46d1ec67f1039a69aa008123da1ec74ff5369f760cb9e068b49b1624efbcdf622671a1 languageName: node linkType: hard