diff --git a/Dockerfile.mongodb b/Dockerfile.mongodb index 2dbe282..c2fd6c3 100644 --- a/Dockerfile.mongodb +++ b/Dockerfile.mongodb @@ -1,9 +1,26 @@ FROM mongo:latest -# Copy database dump and scripts -COPY database/ /data/database/ COPY scripts/create-indexes.js /scripts/create-indexes.js +# The sample dump is NOT baked into the image. It used to be — `COPY +# database/ /data/database/` — but `database/` is gitignored (the 4DN +# backup was removed from the repo in 2fc98f8), so on a clean checkout +# the directory does not exist at all and the COPY failed the build +# outright with `"/database": not found`. The build only appeared to +# work on machines where a stray file such as `.DS_Store` happened to +# keep the directory alive. +# +# Mounting the dump at run time instead (see the `mongodb` target in the +# Makefile) fixes that and is the better shape regardless: the image no +# longer varies with whatever a developer happens to have on disk, and +# obtaining a dump later is a container restart rather than a rebuild. +# The directory is created here so `mongorestore` always has a valid +# target — it exits 1 on a path that does not exist, which under +# `set -e` below would take the container down — and so running the +# image with no mount at all degrades to "empty database" rather than +# to a crash. +RUN mkdir -p /data/database + # Create startup script (development mode only) COPY <<'EOF' /startup.sh #!/bin/bash @@ -22,9 +39,19 @@ until mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; do done echo "MongoDB started." -# Restore database -echo "Restoring database..." -mongorestore --gzip /data/database +# Restore the dump, if one was mounted. `set -e` is deliberately kept: +# a failed restore of a dump that IS present should stop the container +# rather than leave a half-loaded database looking healthy. The empty +# case is not a failure though, so it is reported as such instead of +# being left to mongorestore's "0 document(s) restored" trace, which +# reads like something went wrong. +if [ -n "$(ls -A /data/database 2>/dev/null)" ]; then + echo "Restoring database from /data/database..." + mongorestore --gzip /data/database +else + echo "No dump at /data/database — starting with an empty database." + echo "Populate it with 'curl -X POST http://localhost:8000/sync'." +fi # Create indexes echo "Creating indexes..." diff --git a/Makefile b/Makefile index 79b0abb..a163976 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,13 @@ mongodb: @docker rm mongodb 2>/dev/null || true @echo "Building MongoDB image..." docker build -t cfdb-mongodb -f Dockerfile.mongodb . + @# database/ is gitignored and empty on a clean checkout. It is a + @# drop-in point for an optional mongodump, mounted read-only rather + @# than baked into the image (see Dockerfile.mongodb). Created here so + @# the mount source always exists and so the drop-in point is visible. + @mkdir -p database @echo "Starting MongoDB container..." - docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 cfdb-mongodb + docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 -v "$(CURDIR)/database:/data/database:ro" cfdb-mongodb @echo "MongoDB container starting on port 27017. Check logs with: docker logs -f mongodb" build-materialize: diff --git a/README.md b/README.md index f29c260..819d433 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,13 @@ Requires Python 3.11 or later. ### Docker Startup ```bash -# 1. Start MongoDB (restores sample data and creates indexes) +# 1. Start MongoDB (creates indexes, and restores a dump if one is present) make mongodb # 2. Start the API server make api -# 3. (Optional) Sync latest DCC metadata +# 3. Populate the database curl -X POST http://localhost:8000/sync ``` @@ -49,6 +49,10 @@ This starts: - MongoDB on port 27017 (with indexes) - GraphQL/REST API on port 8000 +**The database starts empty.** A 4DN sample dump used to be committed under `database/`, but it was removed and the directory gitignored, so a clean checkout has no data to restore and step 3 is how you get some — it is not optional. `make mongodb` still creates every index, so an empty database is a working one rather than a broken one. + +`database/` remains the drop-in point if you do have a dump: put a `mongodump --gzip` tree there (`database/cfdb/*.bson.gz`) and it is restored on the next `make mongodb`. It is mounted read-only at run time rather than baked into the image, so obtaining a dump later costs a container restart and not a rebuild. + ### Production (TLS/X.509) ```bash @@ -80,7 +84,7 @@ Run `./certs/generate-certs.sh --help` for full usage information. | Target | Description | |--------|-------------| -| `make mongodb` | Build and start MongoDB with sample data and indexes | +| `make mongodb` | Build and start MongoDB, creating indexes. The database starts empty unless a `mongodump --gzip` tree is present in `database/`, which is restored if one is. | | `make api` | Build and start the API container | | `make materialize-files` | Manually materialize all file metadata (usually done via sync) | | `make materialize-dcc DCC=hubmap` | Materialize a single DCC | diff --git a/scripts/create-indexes.js b/scripts/create-indexes.js index c7ebff3..7ee6614 100644 --- a/scripts/create-indexes.js +++ b/scripts/create-indexes.js @@ -17,7 +17,37 @@ function ensureIndex(coll, keys, opts) { throw new Error("ensureIndex requires opts.name"); } var name = opts.name; - var existing = coll.getIndexes(); + // ``getIndexes`` raises NamespaceNotFound (code 26, "ns does not + // exist") when the collection has never been written to. On a + // database with no dump restored — the normal state of a clean + // checkout, since the sample dump is gitignored and no longer + // shipped — that is every collection this script touches. The bare + // ``createIndex`` calls elsewhere in this file are immune because + // ``createIndex`` creates the namespace implicitly; this helper is + // the only code path that reads the existing indexes first, which + // is why ``jobs`` was the single collection that aborted the run + // (and, under ``set -e`` in Dockerfile.mongodb's startup script, + // took the whole container down with it on exit 1). + // + // A namespace that does not exist trivially holds no index that + // could conflict, so treat it as the empty list and fall through to + // ``createIndex``, which creates the collection along with the + // index. This preserves the drop-and-recreate contract exactly: a + // collection with no indexes cannot produce a name match, so the + // IndexOptionsConflict path below is unreachable in this case and + // stays untouched for the case it was written for. Only + // NamespaceNotFound is swallowed — any other server error still + // propagates, so a genuine failure is not masked into a silently + // half-indexed database. + var existing; + try { + existing = coll.getIndexes(); + } catch (e) { + if (e.codeName !== "NamespaceNotFound" && e.code !== 26) { + throw e; + } + existing = []; + } var match = null; for (var i = 0; i < existing.length; i++) { if (existing[i].name === name) {