feat: make dev.up wait until services are actually running

.github/workflows/provisioning-tests.yml

@@ -73,7 +73,10 @@ jobs:
         run:  make dev.provision.${{matrix.services}}
 
       - name: "Bring up services"
-        run:  make dev.up.${{matrix.services}}
+        # Use the without-wait variant: this job only provisions a subset of
+        # services, so their un-provisioned transitive dependencies can never
+        # become healthy.
+        run:  make dev.up.without-wait.${{matrix.services}}
 
       - name: "Wait for services to become ready"
         run: |

Makefile

@@ -55,7 +55,8 @@
         dev.shell.cms_watcher dev.shell.xqueue dev.shell.xqueue_consumer \
         dev.static dev.static.lms dev.static.cms dev.stats dev.status \
         dev.stop dev.up dev.up.attach dev.up.shell \
-        dev.up.without-deps dev.up.without-deps.shell dev.up.with-programs \
+        dev.up.without-deps dev.up.without-deps.shell dev.up.without-wait \
+        dev.up.with-programs \
         dev.up.with-watchers dev.validate docs \
         help requirements impl-dev.clone.https impl-dev.clone.ssh impl-dev.provision \
         impl-dev.pull impl-dev.pull.without-deps impl-dev.up impl-dev.up.attach \
@@ -254,22 +255,27 @@ dev.up.with-watchers.%: ## Bring up services and their dependencies + asset watc
 dev.up.without-deps: _expects-service-list.dev.up.without-deps
 
 dev.up.without-deps.%: dev.check-memory ## Bring up services by themselves.
-	docker compose up -d --no-deps $$(echo $* | tr + " ")
+	docker compose up -d --wait --no-deps $$(echo $* | tr + " ")
 
 dev.up.without-deps.shell: _expects-service.dev.up.without-deps.shell
 
 dev.up.without-deps.shell.%: ## Bring up a service by itself + shell into it.
 	make dev.up.without-deps.$*
 	make dev.shell.$*
 
+dev.up.without-wait: _expects-service-list.dev.up.without-wait
+
+dev.up.without-wait.%: dev.check-memory ## Bring up services and their dependencies without waiting for them to become healthy.
+	docker compose up -d $$(echo $* | tr + " ")
+
 dev.up:
 	@scripts/make_warn_default_large.sh "$@"
 
 dev.up.large-and-slow: dev.up.$(DEFAULT_SERVICES) ## Bring up default services.
 	@echo # at least one statement so that dev.up.% doesn't run too
 
 dev.up.%: dev.check-memory ## Bring up services and their dependencies.
-	docker compose up -d $$(echo $* | tr + " ")
+	docker compose up -d --wait $$(echo $* | tr + " ")
 ifeq ($(ALWAYS_CACHE_PROGRAMS),true)
 	make dev.cache-programs
 endif

common.yml

@@ -0,0 +1,89 @@
+# Abstract base services, inherited by concrete services via `extends`.
+services:
+
+  # Traffic-serving backend app (Django/IDA) services.
+  backend-app:
+    stdin_open: true  # Allows `make dev.attach.<service>` to work correctly.
+    tty: true
+    environment:
+      # This default works well for the vast majority of django services.
+      # Limiting the checks to just the "urls" tag is a proxy for "check the
+      # bare minimum needed to validate that there are no missing imports.
+      DEVSERVER_CHECK: "python manage.py check --tag urls"
+    # Concrete services may set these extra environment variables:
+    #
+    # - DEVSERVER_PRECHECK allows some services to run extra commands at the very beginning.
+    #   This is rarely needed.
+    # - DEVSERVER_CHECK runs once and exits quickly on failure.  This allows docker-compose to give
+    #   near-instant feedback to users when a service cannot start (e.g. due to missing imports).
+    # - DEVSERVER_CMD runs inside an infinite loop to allow `make dev.restart-devserver.<service>`
+    #   to work correctly.
+    #
+    # Examples:
+    #
+    #   environment:
+    #     DEVSERVER_PRECHECK: "source /edx/app/foo/foo_env"   # optional
+    #     DEVSERVER_CHECK: "python manage.py check"           # optional
+    #     DEVSERVER_CMD: "python manage.py runserver 0.0.0.0:18000"
+    command:
+      - bash
+      - -c
+      - |
+          if [ -n "$$DEVSERVER_PRECHECK" ]; then eval "$$DEVSERVER_PRECHECK"; fi
+          if [ -n "$$DEVSERVER_CHECK" ]; then
+              eval "$$DEVSERVER_CHECK" || { echo "pre-flight check failed; giving up." >&2; exit 1; }
+          fi
+          while true; do
+              eval "$$DEVSERVER_CMD"
+              sleep 2
+          done
+    # Configure a healthcheck to allow `make dev.up.<servce>` to give accurate feedback about
+    # whether the service actually started successfully and is serving traffic.
+    # Set HEALTHCHECK_TARGET to the full URL curl should hit, e.g. "http://localhost:18000/heartbeat"
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS $$HEALTHCHECK_TARGET || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 20
+      start_period: 240s
+      start_interval: 5s  # poll fast during grace period so a healthy boot flips status quickly
+
+  # All microfrontends.
+  microfrontend:
+    # Use `npm ci` rather than `npm install` for a few reasons:
+    #
+    # - Repeatability: Respect the currently checked out package
+    #   versions rather than upgrading when package.json and
+    #   package-lock.json don't match. (Two people using this at
+    #   different times on the same commit should get the same
+    #   results.)
+    # - Immutability: Don't change the repo's working directory
+    #   unexpectedly when there's a lock mismatch.
+    #
+    # Fail fast if package install fails to avoid mysterious
+    # errors later.
+    command:
+      - bash
+      - -c
+      - |
+          npm ci || exit 1
+          if [ -n "$${PARAGON_BRAND_PACKAGE}" ]; then
+              npx paragon install-theme "$${PARAGON_BRAND_PACKAGE}" || exit 1
+          fi
+          while true; do
+              npm start
+              sleep 2
+          done
+    stdin_open: true
+    tty: true
+    image: node:18
+    environment:
+      - NODE_ENV=development
+    # More generous than backend-app because a cold `npm ci` can take
+    # several minutes.
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS $$HEALTHCHECK_TARGET || exit 1"]
+      interval: 10s
+      timeout: 10s
+      retries: 30
+      start_period: 300s