fix(db): retry initial Mongo connection to avoid worker crash-loop

lib/db/controller.test.ts

@@ -1,5 +1,6 @@
 import * as mongodb from 'mongodb';
 import { DatabaseController } from './controller';
+import { DatabaseConnectionError } from '../workerErrors';
 import '../../env-test';
 
 /**
@@ -24,4 +25,47 @@ describe('Database Controller Test', () => {
       expect(result).toBe(true);
     });
   });
+
+  describe('initial handshake retry', () => {
+    const mongoModule = jest.requireActual('mongodb');
+    let connectSpy: jest.SpyInstance;
+
+    beforeEach(() => {
+      process.env.MONGO_RECONNECT_TRIES = '5';
+      process.env.MONGO_RECONNECT_INTERVAL = '1';
+      jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+      connectSpy = jest.spyOn(mongoModule, 'connect');
+    });
+
+    afterEach(() => {
+      delete process.env.MONGO_RECONNECT_TRIES;
+      delete process.env.MONGO_RECONNECT_INTERVAL;
+      jest.restoreAllMocks();
+    });
+
+    it('retries the initial connection until it succeeds', async () => {
+      const fakeDb = {} as mongodb.Db;
+      const fakeClient = { db: jest.fn().mockReturnValue(fakeDb) } as unknown as mongodb.MongoClient;
+
+      connectSpy
+        .mockRejectedValueOnce(new Error('unreachable'))
+        .mockRejectedValueOnce(new Error('unreachable'))
+        .mockResolvedValueOnce(fakeClient);
+
+      const controller = new DatabaseController('mongodb://localhost:27017/test');
+      const result = await controller.connect();
+
+      expect(connectSpy).toHaveBeenCalledTimes(3);
+      expect(result).toBe(fakeDb);
+    });
+
+    it('throws DatabaseConnectionError after exhausting all retries', async () => {
+      connectSpy.mockRejectedValue(new Error('unreachable'));
+
+      const controller = new DatabaseController('mongodb://localhost:27017/test');
+
+      await expect(controller.connect()).rejects.toBeInstanceOf(DatabaseConnectionError);
+      expect(connectSpy).toHaveBeenCalledTimes(5);
+    });
+  });
 });

lib/db/controller.ts

@@ -1,5 +1,22 @@
 import { GridFSBucket, MongoClient, Db, connect } from 'mongodb';
 import { DatabaseConnectionError } from '../workerErrors';
+import { positiveIntEnv } from '../utils/positiveIntEnv';
+
+/**
+ * How many times to retry the initial Mongo handshake before giving up
+ */
+const DEFAULT_RECONNECT_TRIES = 60;
+
+/**
+ * Delay between initial-handshake retries, in ms
+ */
+const DEFAULT_RECONNECT_INTERVAL_MS = 3000;
+
+/**
+ * Bounds how long a single attempt waits for an available server, so a retry
+ * fails fast during an outage instead of hanging on the 30s driver default
+ */
+const SERVER_SELECTION_TIMEOUT_MS = 10000;
 
 /**
  * Database connection singleton
@@ -46,28 +63,49 @@ export class DatabaseController {
   }
 
   /**
-   * Connect to database
-   * Requires `MONGO_DSN` environment variable to be set
+   * Connect to the database, retrying with a fixed backoff while the server is
+   * unreachable so a worker booting during a Mongo outage waits instead of
+   * crash-looping. The driver auto-recovers already-open connections on its
+   * own, so this retry covers the initial handshake only.
    *
-   * @throws {Error} if `MONGO_DSN` is not set
+   * Tunable via MONGO_RECONNECT_TRIES (default 60) and
+   * MONGO_RECONNECT_INTERVAL in ms (default 3000).
+   *
+   * @throws {DatabaseConnectionError} if every attempt fails
    */
   public async connect(): Promise<Db> {
     if (this.db) {
-      return;
+      return this.db;
     }
 
-    try {
-      this.connection = await connect(this.connectionUri, {
-        useNewUrlParser: true,
-        useUnifiedTopology: true,
-        ...(this.appName ? { appName: this.appName } : {}),
-      });
-      this.db = await this.connection.db();
+    const tries = positiveIntEnv(process.env.MONGO_RECONNECT_TRIES, DEFAULT_RECONNECT_TRIES);
+    const intervalMs = positiveIntEnv(process.env.MONGO_RECONNECT_INTERVAL, DEFAULT_RECONNECT_INTERVAL_MS);
 
-      return this.db;
-    } catch (err) {
-      throw new DatabaseConnectionError(err);
+    for (let attempt = 1; attempt <= tries; attempt++) {
+      try {
+        this.connection = await connect(this.connectionUri, {
+          useNewUrlParser: true,
+          useUnifiedTopology: true,
+          serverSelectionTimeoutMS: SERVER_SELECTION_TIMEOUT_MS,
+          ...(this.appName ? { appName: this.appName } : {}),
+        });
+        this.db = this.connection.db();
+
+        return this.db;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+
+        console.warn(`[Mongo] connect attempt ${attempt}/${tries} failed: ${message}`);
+
+        if (attempt >= tries) {
+          throw new DatabaseConnectionError(err);
+        }
+
+        await new Promise((resolve) => setTimeout(resolve, intervalMs));
+      }
     }
+
+    throw new DatabaseConnectionError('Failed to connect to MongoDB');
   }
 
   /**

lib/utils/positiveIntEnv.ts

@@ -0,0 +1,16 @@
+/**
+ * Parses a positive-integer env var, using `fallback` for missing, non-numeric,
+ * zero or negative values
+ *
+ * @param value - raw env var value
+ * @param fallback - default for an invalid value
+ */
+export function positiveIntEnv(value: string | undefined, fallback: number): number {
+  const parsed = Number(value);
+
+  if (!Number.isFinite(parsed) || parsed < 1) {
+    return fallback;
+  }
+
+  return Math.floor(parsed);
+}