Convert MongoDB from standalone to replica set

.gitignore

@@ -34,7 +34,7 @@ Session.vim
 scripts/*.js
 !scripts/frontendScripts.js
 !scripts/jestTest.js
-!scripts/setupMongo.js
+!scripts/startDatabase.js
 database/*.js
 *.log
 *-debug.log*

.vscode/tasks.json

@@ -28,9 +28,9 @@
     },
     {
       "label": "run-mongo",
-      "command": "mongod",
+      "command": "npm",
       "type": "process",
-      "args": ["--dbpath", "${workspaceFolder}/mongo_database"],
+      "args": ["run", "database"],
       "problemMatcher": "$tsc"
     }
   ]

Backend/appsettings.json

@@ -1,7 +1,7 @@
 {
   "MongoDB": {
-    "ConnectionString": "mongodb://localhost:27017",
-    "ContainerConnectionString": "mongodb://database:27017",
+    "ConnectionString": "mongodb://localhost:27017/?replicaSet=rs0",
+    "ContainerConnectionString": "mongodb://database:27017/?replicaSet=rs0",
     "CombineDatabase": "CombineDatabase"
   },
   "Logging": {

README.md

@@ -473,7 +473,7 @@ npm run license-report-frontend
 
 To browse the database locally during development, open [MongoDB Compass](https://www.mongodb.com/try/download/compass).
 
-1. Under New Connection, enter `mongodb://localhost:27017`
+1. Under New Connection, enter `mongodb://localhost:27017/?replicaSet=rs0`
 2. Under Databases, select CombineDatabase
 
 ### Add or Update Dictionary Files

database/Dockerfile

@@ -5,21 +5,21 @@
 #   - Intel/AMD 64-bit
 #   - ARM 64-bit
 ############################################################
-FROM mongo:7.0.28-jammy@sha256:8ddd3db4d2638eb914cce56284e2f0d6daf140bba31679b2af86f7d790a4c77e
+FROM mongo:7.0.30-jammy@sha256:aee9bae9f1a5507a51e19f24b015162cbcd7004695d99175dbccc427e20760e2
 
 WORKDIR /
 
-RUN mkdir /data/semantic-domains
+RUN mkdir -p /data/semantic-domains /opt/thecombine
 
 # Copy semantic domain import files
 COPY semantic_domains/* /data/semantic-domains/
 
 # from https://hub.docker.com/_/mongo
-# Initializing a fresh instance
-# When a container is started for the first time it will execute files
-# with extensions .sh and .js that are found in /docker-entrypoint-initdb.d.
-# Files will be executed in alphabetical order. .js files will be executed
-# by mongosh (mongo on versions below 6) using the database specified by
-# the MONGO_INITDB_DATABASE variable, if it is present, or test otherwise.
-# You may also switch databases within the .js script.
-COPY init/* /docker-entrypoint-initdb.d/
+# Scripts in /docker-entrypoint-initdb.d run only on first startup of an empty
+# data directory. We intentionally keep setup scripts out of initdb.d and run
+# them from Kubernetes postStart to avoid first-boot race conditions.
+COPY init/update-semantic-domains.sh /opt/thecombine/update-semantic-domains.sh
+
+# Replica set readiness/alignment runs from Kubernetes postStart hook on every
+# container start, so this script is intentionally kept out of initdb.d.
+COPY init/00-replica-set.js /opt/thecombine/00-replica-set.js

database/init/00-replica-set.js

@@ -0,0 +1,81 @@
+// Ensure a single-node replica set is initialized and advertising the
+// expected host for this environment.
+//
+// MONGO_INITDB_REPLICA_HOST can be set to the resolvable hostname:port
+// used to advertise this member (e.g. "database:27017" in Kubernetes).
+const host = process.env.MONGO_INITDB_REPLICA_HOST || "localhost:27017";
+const maxWaitMs = 60 * 1000;
+const intervalMs = 1000;
+const start = Date.now();
+
+/** Ensure the primary host is correctly configured */
+function ensurePrimaryHost(forceReconfig) {
+  let conf;
+  try {
+    conf = rs.conf();
+  } catch (error) {
+    conf = db.getSiblingDB("local").system.replset.findOne();
+    if (!forceReconfig || !conf) {
+      throw error;
+    }
+  }
+
+  if (!conf.members?.length) {
+    throw new Error("Replica set config has no members");
+  }
+
+  if (conf.members[0].host !== host) {
+    print(`Updating replica set member host to ${host}`);
+    conf.members[0].host = host;
+    conf.version = (conf.version || 1) + 1;
+    rs.reconfig(conf, { force: forceReconfig });
+    return false;
+  }
+
+  return true;
+}
+
+// Wait for replica set to be initialized.
+let replicaSetInitiated = false;
+while (Date.now() - start < maxWaitMs) {
+  try {
+    rs.initiate({ _id: "rs0", members: [{ _id: 0, host: host }] });
+    print(`Initialized replica set with host ${host}`);
+    replicaSetInitiated = true;
+    break;
+  } catch (err) {
+    if (String(err).includes("already initialized")) {
+      print("Replica set already initialized");
+      replicaSetInitiated = true;
+      break;
+    }
+
+    print(`Replica set init deferred: ${err}`);
+  }
+
+  sleep(intervalMs);
+}
+if (!replicaSetInitiated) {
+  throw new Error(`Replica set not initialized after ${maxWaitMs}ms`);
+}
+
+// Wait for this member to be PRIMARY with the correct host.
+while (Date.now() - start < maxWaitMs) {
+  try {
+    if (db.hello().isWritablePrimary) {
+      if (ensurePrimaryHost(false)) {
+        print(`Replica set is PRIMARY with correct host: ${host}`);
+        quit(0);
+      }
+    } else {
+      ensurePrimaryHost(true);
+    }
+  } catch (err) {
+    print(`Host alignment deferred: ${err}`);
+  }
+
+  sleep(intervalMs);
+}
+throw new Error(
+  `Replica set did not reach PRIMARY state with host ${host} after ${maxWaitMs}ms`
+);

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -46,6 +46,40 @@ spec:
         - image: {{ include "database.containerImage" . }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy }}
           name: database
+          args:
+            - "--replSet"
+            - "rs0"
+          lifecycle:
+            postStart:
+              exec:
+                command:
+                  - /bin/sh
+                  - -c
+                  - |
+                    set -e
+                    echo "[postStart] Waiting for mongod to accept connections"
+                    attempts=0
+                    until mongosh --quiet --host 127.0.0.1 --eval "db.adminCommand({ ping: 1 }).ok" >/dev/null 2>&1; do
+                      attempts=$((attempts + 1))
+                      if [ "${attempts}" -ge 120 ]; then
+                        echo "[postStart] Failed to connect to mongod after ${attempts} attempts, exiting"
+                        exit 1
+                      fi
+                      sleep 1
+                    done
+                    echo "[postStart] Ensuring replica set host"
+                    mongosh --quiet --host 127.0.0.1 /opt/thecombine/00-replica-set.js || exit $?
+                    needs_semantic_import="$(mongosh --quiet --host 127.0.0.1 --eval "const combineDb = db.getSiblingDB('CombineDatabase'); const treeCount = combineDb.SemanticDomainTree.countDocuments({}); const domainCount = combineDb.SemanticDomains.countDocuments({}); print(treeCount === 0 || domainCount === 0 ? 'yes' : 'no');")"
+                    if [ "${needs_semantic_import}" = "yes" ]; then
+                      /bin/bash /opt/thecombine/update-semantic-domains.sh
+                    fi
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: MONGO_INITDB_REPLICA_HOST
+              value: "$(POD_IP):27017"
           ports:
             - containerPort: 27017
           resources:

docs/deploy/README.md

@@ -415,7 +415,7 @@ Notes:
   rerun manually:
 
   ```console
-  kubectl -n thecombine exec deployment/database -- /docker-entrypoint-initdb.d/update-semantic-domains.sh
+  kubectl -n thecombine exec deployment/database -- /opt/thecombine/update-semantic-domains.sh
   ```
 
 ## Maintenance

package.json

@@ -8,8 +8,7 @@
     "backend": "dotnet watch --project Backend/BackendFramework.csproj",
     "build": "parcel build",
     "build:analyze": "npm run build -- --reporter @parcel/reporter-bundle-analyzer",
-    "predatabase": "node scripts/setupMongo.js",
-    "database": "mongod --dbpath=./mongo_database",
+    "database": "node scripts/startDatabase.js",
     "drop-database": "tsc scripts/dropDB.ts && node scripts/dropDB.js",
     "find-circular-deps": "npx --ignore-scripts -y madge -c src/index.tsx --ts-config tsconfig.json",
     "fmt-backend": "      dotnet format                     && dotnet format                     Backend.Tests",

scripts/startDatabase.js

@@ -0,0 +1,173 @@
+"use strict";
+
+const { spawn, spawnSync } = require("child_process");
+const { emitKeypressEvents } = require("readline");
+
+const { ensureDir } = require("fs-extra");
+
+const dbPath = "./mongo_database";
+const replSetName = "rs0";
+const maxAttempts = 30;
+const retryIntervalSeconds = 1;
+const mongoshTimeoutSeconds = 10;
+let mongodProcess;
+let exiting = false;
+
+/** Check if TTY and raw mode are available. */
+function canUseRawMode() {
+  return process.stdin.isTTY && typeof process.stdin.setRawMode === "function";
+}
+
+/** Disable raw mode. */
+function stopRawMode() {
+  if (canUseRawMode()) {
+    process.stdin.setRawMode(false);
+    process.stdin.pause();
+  }
+}
+
+/** Enable raw mode to capture Ctrl+C when it doesn't otherwise work. */
+function startRawMode() {
+  if (canUseRawMode()) {
+    emitKeypressEvents(process.stdin);
+    process.stdin.setRawMode(true);
+    process.stdin.resume();
+    process.stdin.on("keypress", (_text, key) => {
+      if (key?.ctrl && key.name === "c") {
+        forceExit(130);
+      }
+    });
+  }
+}
+
+/** Unless already exiting, forcibly kill mongod and exit process. */
+function forceExit(code = 0) {
+  if (exiting) {
+    return;
+  }
+
+  exiting = true;
+
+  stopRawMode();
+
+  if (mongodProcess && !mongodProcess.killed) {
+    mongodProcess.kill("SIGKILL");
+  }
+  process.exit(code);
+}
+
+/** Set up handlers for various interrupts. */
+function setUpInterruptHandling() {
+  process.on("SIGINT", () => forceExit(130));
+  process.on("SIGBREAK", () => forceExit(131));
+  process.on("SIGTERM", () => forceExit(143));
+  startRawMode();
+}
+
+/** Convert error to string. */
+function getErrorMessage(error) {
+  return error instanceof Error
+    ? `${error.code}: ${error.message}`
+    : String(error);
+}
+
+/** Run a mongosh command with timeout and interrupt handling. */
+function runMongosh(args, options = {}) {
+  const result = spawnSync("mongosh", [...args, "--quiet"], {
+    timeout: mongoshTimeoutSeconds * 1000,
+    killSignal: "SIGTERM",
+    ...options,
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+  if (result.signal) {
+    throw new Error(`mongosh exited due to signal ${result.signal}`);
+  }
+
+  return result;
+}
+
+/** Ping with mongosh until available, up to a preset number of attempts. */
+async function waitForMongo() {
+  for (let i = 0; i < maxAttempts; i++) {
+    let result;
+    try {
+      result = runMongosh(["--eval", "db.adminCommand('ping')"]);
+    } catch (err) {
+      console.warn(`ping attempt ${i + 1} failed: ${getErrorMessage(err)}`);
+    }
+    if (result?.status === 0) {
+      return true;
+    }
+    if (i < maxAttempts - 1) {
+      await new Promise((res) => setTimeout(res, retryIntervalSeconds * 1000));
+    }
+  }
+
+  console.error(`MongoDB pings failed after ${maxAttempts} attempts.`);
+  return false;
+}
+
+/** Start a replica set if not already initialized. */
+async function initReplicaSet() {
+  try {
+    const result = runMongosh(
+      [
+        "--eval",
+        `try { rs.status() } catch { rs.initiate({ _id: '${replSetName}', members: [{ _id: 0, host: 'localhost:27017' }] }) }`,
+      ],
+      { stdio: "inherit" }
+    );
+    return result.status === 0;
+  } catch (error) {
+    console.error(`Replica set init failed: ${getErrorMessage(error)}`);
+    return false;
+  }
+}
+
+async function main() {
+  setUpInterruptHandling();
+
+  await ensureDir(dbPath);
+
+  // Start the mongod process
+  mongodProcess = spawn(
+    "mongod",
+    ["--dbpath", dbPath, "--replSet", replSetName, "--quiet"],
+    { stdio: "inherit" }
+  );
+
+  // Exit when the mongod process errors
+  mongodProcess.on("error", (err) => {
+    console.error(`mongod error: ${err.message}`);
+    forceExit(1);
+  });
+
+  // Exit when the mongod process exits
+  mongodProcess.on("exit", (code, signal) => {
+    if (exiting) {
+      return;
+    }
+
+    if (code || signal) {
+      console.error(`mongod exited with code ${code}, signal ${signal}`);
+    }
+    forceExit(signal ? 1 : (code ?? 1));
+  });
+
+  if (!(await waitForMongo())) {
+    console.error("MongoDB did not start in time");
+    forceExit(1);
+  }
+  if (!(await initReplicaSet())) {
+    console.error("Replica set initialization failed");
+    forceExit(1);
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  forceExit(1);
+});