diff --git a/lib/db/controller.test.ts b/lib/db/controller.test.ts index 1ec55b7c..c7870b94 100644 --- a/lib/db/controller.test.ts +++ b/lib/db/controller.test.ts @@ -1,5 +1,6 @@ import * as mongodb from 'mongodb'; import { DatabaseController } from './controller'; +import { DatabaseConnectionError } from '../workerErrors'; import '../../env-test'; /** @@ -24,4 +25,47 @@ describe('Database Controller Test', () => { expect(result).toBe(true); }); }); + + describe('initial handshake retry', () => { + const mongoModule = jest.requireActual('mongodb'); + let connectSpy: jest.SpyInstance; + + beforeEach(() => { + process.env.MONGO_RECONNECT_TRIES = '5'; + process.env.MONGO_RECONNECT_INTERVAL = '1'; + jest.spyOn(console, 'warn').mockImplementation(() => undefined); + connectSpy = jest.spyOn(mongoModule, 'connect'); + }); + + afterEach(() => { + delete process.env.MONGO_RECONNECT_TRIES; + delete process.env.MONGO_RECONNECT_INTERVAL; + jest.restoreAllMocks(); + }); + + it('retries the initial connection until it succeeds', async () => { + const fakeDb = {} as mongodb.Db; + const fakeClient = { db: jest.fn().mockReturnValue(fakeDb) } as unknown as mongodb.MongoClient; + + connectSpy + .mockRejectedValueOnce(new Error('unreachable')) + .mockRejectedValueOnce(new Error('unreachable')) + .mockResolvedValueOnce(fakeClient); + + const controller = new DatabaseController('mongodb://localhost:27017/test'); + const result = await controller.connect(); + + expect(connectSpy).toHaveBeenCalledTimes(3); + expect(result).toBe(fakeDb); + }); + + it('throws DatabaseConnectionError after exhausting all retries', async () => { + connectSpy.mockRejectedValue(new Error('unreachable')); + + const controller = new DatabaseController('mongodb://localhost:27017/test'); + + await expect(controller.connect()).rejects.toBeInstanceOf(DatabaseConnectionError); + expect(connectSpy).toHaveBeenCalledTimes(5); + }); + }); }); diff --git a/lib/db/controller.ts b/lib/db/controller.ts index 220db627..07de49ac 100644 --- a/lib/db/controller.ts +++ b/lib/db/controller.ts @@ -1,5 +1,22 @@ import { GridFSBucket, MongoClient, Db, connect } from 'mongodb'; import { DatabaseConnectionError } from '../workerErrors'; +import { positiveIntEnv } from '../utils/positiveIntEnv'; + +/** + * How many times to retry the initial Mongo handshake before giving up + */ +const DEFAULT_RECONNECT_TRIES = 60; + +/** + * Delay between initial-handshake retries, in ms + */ +const DEFAULT_RECONNECT_INTERVAL_MS = 3000; + +/** + * Bounds how long a single attempt waits for an available server, so a retry + * fails fast during an outage instead of hanging on the 30s driver default + */ +const SERVER_SELECTION_TIMEOUT_MS = 10000; /** * Database connection singleton @@ -46,28 +63,49 @@ export class DatabaseController { } /** - * Connect to database - * Requires `MONGO_DSN` environment variable to be set + * Connect to the database, retrying with a fixed backoff while the server is + * unreachable so a worker booting during a Mongo outage waits instead of + * crash-looping. The driver auto-recovers already-open connections on its + * own, so this retry covers the initial handshake only. * - * @throws {Error} if `MONGO_DSN` is not set + * Tunable via MONGO_RECONNECT_TRIES (default 60) and + * MONGO_RECONNECT_INTERVAL in ms (default 3000). + * + * @throws {DatabaseConnectionError} if every attempt fails */ public async connect(): Promise { if (this.db) { - return; + return this.db; } - try { - this.connection = await connect(this.connectionUri, { - useNewUrlParser: true, - useUnifiedTopology: true, - ...(this.appName ? { appName: this.appName } : {}), - }); - this.db = await this.connection.db(); + const tries = positiveIntEnv(process.env.MONGO_RECONNECT_TRIES, DEFAULT_RECONNECT_TRIES); + const intervalMs = positiveIntEnv(process.env.MONGO_RECONNECT_INTERVAL, DEFAULT_RECONNECT_INTERVAL_MS); - return this.db; - } catch (err) { - throw new DatabaseConnectionError(err); + for (let attempt = 1; attempt <= tries; attempt++) { + try { + this.connection = await connect(this.connectionUri, { + useNewUrlParser: true, + useUnifiedTopology: true, + serverSelectionTimeoutMS: SERVER_SELECTION_TIMEOUT_MS, + ...(this.appName ? { appName: this.appName } : {}), + }); + this.db = this.connection.db(); + + return this.db; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + + console.warn(`[Mongo] connect attempt ${attempt}/${tries} failed: ${message}`); + + if (attempt >= tries) { + throw new DatabaseConnectionError(err); + } + + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } } + + throw new DatabaseConnectionError('Failed to connect to MongoDB'); } /** diff --git a/lib/utils/positiveIntEnv.ts b/lib/utils/positiveIntEnv.ts new file mode 100644 index 00000000..06aff2e9 --- /dev/null +++ b/lib/utils/positiveIntEnv.ts @@ -0,0 +1,16 @@ +/** + * Parses a positive-integer env var, using `fallback` for missing, non-numeric, + * zero or negative values + * + * @param value - raw env var value + * @param fallback - default for an invalid value + */ +export function positiveIntEnv(value: string | undefined, fallback: number): number { + const parsed = Number(value); + + if (!Number.isFinite(parsed) || parsed < 1) { + return fallback; + } + + return Math.floor(parsed); +}