refactor justfiles / test commands

.github/workflows/ci.yml

@@ -32,11 +32,8 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
-      - name: run sqlx migration up & down
-        run: |
-          just sqlx-migrate-run \
-            sqlx-check \
-            sqlx-migrate-revert
+      - name: Test database migrations
+        run: just test-database-migrations
 
       - name: shut down test environment
         if: ${{ always() }}
@@ -90,12 +87,27 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
+      - name: Expose GitHub cache credentials
+        uses: crazy-max/ghaction-github-runtime@v4
+
       - uses: taiki-e/install-action@v2
         with:
-          tool: just
+          tool: just,sqlx-cli
 
-      - name: Run GUI tests
-        run: just run-gui-tests
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 24
+
+      - name: Run end-to-end GUI tests
+        env:
+          # run all necessary admin commands from the docker images, as
+          # we would in prod.
+          DOCSRS_CLI_MODE: docker
+          # run the preparation / builds for the e2e tests not
+          # on the gh action host, but inside our builder docker image.
+          # Like this we know this works.
+          DOCSRS_BUILDER_CLI_MODE: docker
+        run: just prepare-gui-tests run-gui-tests-e2e
 
       - name: shut down test environment
         if: ${{ always() }}

.github/workflows/commit-sqlx-changes.yml

@@ -35,43 +35,30 @@ jobs:
           git fetch origin
           git rebase origin/main
 
-      - name: install `just`
-        run: sudo snap install --edge --classic just
+      - name: Install development tools
+        uses: taiki-e/install-action@v2
+        with:
+          tool: just,sqlx-cli
 
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
-      - name: Launch postgres
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
-
-      - name: install SQLX CLI
-        run: cargo install sqlx-cli --no-default-features --features postgres
-
-      - name: run database migrations
-        run: cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL
-
       - name: Commit sqlx changes on main branch if any
         id: sqlx_check
         run: |
-          just sqlx-prepare
+          just sqlx-update
           # If there are differences in the SQLX files, then we push them on the main branch
-          if git status --porcelain .sqlx ; then
+          if [ -n "$(git status --porcelain .sqlx)" ]; then
             echo "need_pr=1" >> "$GITHUB_OUTPUT"
           else
             echo "need_pr=0" >> "$GITHUB_OUTPUT"
           fi
 
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
       - name: Open pull request
         if: ${{ steps.sqlx_check.outputs.need_pr == '1' }}

.github/workflows/docker.yml

@@ -15,6 +15,10 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: just
+
       - name: setup docker buildx
         uses: docker/setup-buildx-action@v4
 
@@ -31,17 +35,4 @@ jobs:
             *.cache-to=type=gha,mode=max,scope=docs-rs-images
 
       - name: smoke test packaged binaries
-        run: |
-          set -euo pipefail
-
-          docker buildx bake --file docker-bake.hcl --print |
-            jq -r '
-              . as $bake
-              | $bake.group.default.targets[] as $target
-              | $bake.target[$target].tags[0]
-                // error("Bake target \($target) has no image tag")
-            ' |
-            while IFS= read -r image; do
-              echo "testing $image..."
-              docker run --rm "$image" --help
-            done
+        run: just docker-smoke-test

.gitignore

@@ -13,5 +13,6 @@ target
 .rustwide-docker
 archive_cache
 .workspace
+node_modules/
 docs/book/
 docs/src/generated/

Justfile

@@ -18,9 +18,13 @@ _default:
 
 import 'justfiles/book.just'
 import 'justfiles/cli.just'
+import 'justfiles/docker.just'
+import 'justfiles/linting.just'
 import 'justfiles/services.just'
+import 'justfiles/sqlx.just'
 import 'justfiles/testing.just'
 import 'justfiles/utils.just'
 
-psql:
+# Open a PostgreSQL shell for the local docs.rs database.
+psql: compose-up-resources
     psql $DOCSRS_DATABASE_URL

README.md

@@ -47,49 +47,63 @@ $ mkdir -p ignored/cratesfyi-prefix/crates.io-index
 $ SQLX_OFFLINE=1 cargo build
 ```
 
-Start PostgreSQL and the local S3 service, then initialize them:
+Start PostgreSQL and the local S3 service, then initialize the database:
 
 ```console
-$ docker compose up --wait db s3
-$ . ./.env
-$ cargo run --bin docs_rs_admin -- database migrate
+$ just compose-up-resources
+$ just sqlx-migrate-run
 ```
 
-Commands run outside Docker Compose need the environment variables from `.env`.
-Either source it as above or use a dotenv integration for your shell.
+Most recipes start these resources automatically; use
+`just compose-up-resources` when running application commands directly. The
+`cli`, `builder`, `watcher`, and Compose application recipes also apply pending
+migrations before starting.
+
+The `just` recipes load `.env` and start PostgreSQL and S3 when needed. Commands
+run directly with Cargo need the same environment variables; source `.env` or
+use a dotenv integration for your shell before running them.
 
 Large local files should go in `ignored/`, which is excluded from both Git and
 Docker build contexts.
 
 ### Run the web server
 
 ```console
-$ . ./.env
-$ cargo run --bin docs_rs_web
+$ just web
 ```
 
 The site is available at <http://localhost:3000>. To restart it automatically
-when Rust source or templates change, install `cargo-watch` and run:
+when web, template, asset, shared-library, or workspace configuration files
+change, run:
 
 ```console
-$ . ./.env
-$ cargo watch -x "run --bin docs_rs_web"
+$ just web-watch
 ```
 
+The watch command runs from the repository root and ignores changes confined to
+other application binaries, such as the builder and registry watcher.
+
 ### Build documentation for a crate
 
 Set up or update the docs.rs nightly toolchain, then build a release:
 
 ```console
-$ . ./.env
-$ cargo run --bin docs_rs_builder -- build update-toolchain
-$ cargo run --bin docs_rs_builder -- build crate regex 1.3.1
+$ just builder build update-toolchain
+$ just builder build crate regex 1.3.1
 ```
 
+The `builder` recipe uses `DOCSRS_BUILDER_CLI_MODE`: it defaults to `local` on
+amd64 Linux and `docker` on other platforms. Set the variable explicitly to
+override that choice.
+
+The `cli` and `watcher` recipes similarly use `DOCSRS_CLI_MODE`, but default to
+`local` on every platform. Set either mode variable to `docker` to keep using
+the same high-level recipe through its corresponding Compose service.
+
 To test a local package instead:
 
 ```console
-$ cargo run --bin docs_rs_builder -- build crate --local /path/to/package
+$ just builder build crate --local /path/to/package
 ```
 
 Some workspace packages must first be packaged with Cargo. See
@@ -99,14 +113,21 @@ If you only need an existing release in your local environment, import it
 instead of running the builder:
 
 ```console
-$ . ./.env
-$ cargo run -p docs_rs_import_release -- regex latest
+$ just import-release regex latest
 ```
 
 ### Run with Docker Compose only
 
 If running the Rust binaries on the host is impractical, the `just` recipes can
-also run them in Docker Compose:
+keep the same interface while running them through Docker Compose. Add these
+settings to `.env`:
+
+```dotenv
+DOCSRS_CLI_MODE=docker
+DOCSRS_BUILDER_CLI_MODE=docker
+```
+
+Then use the normal recipes:
 
 ```console
 $ just cli-db-migrate
@@ -123,13 +144,15 @@ $ just compose-up-watcher
 Common one-off commands include:
 
 ```console
-$ just cli-build-update-toolchain
-$ just cli-build-crate regex 1.3.1
-$ just cli-queue-add regex 1.3.1
+$ just builder build update-toolchain
+$ just builder build crate regex 1.3.1
+$ just cli queue add regex 1.3.1
 ```
 
-Use `just --list` to see all available recipes. Tests are not currently
-supported in the Docker-Compose-only development environment.
+Use `just --list` to see all available recipes. The Rust test suite still runs
+on the host; the GUI suite has a container integration mode described below. The
+lower-level `docker-run` recipe is available when a specific Compose service
+must be selected explicitly, but is not needed for normal development.
 
 To stop the services while retaining their data, or to remove their local data:
 
@@ -159,15 +182,28 @@ Run the ignored builder tests separately with:
 $ just run-builder-tests
 ```
 
+Test database migrations through both SQLx CLI and `docs_rs_admin` with:
+
+```console
+$ just test-database-migrations
+```
+
+After changing queries or migrations, apply migrations and regenerate the
+committed SQLx offline metadata with:
+
+```console
+$ just sqlx-update
+```
+
 Run the complete lint suite with:
 
 ```console
 $ just lint
 ```
 
 Linting GitHub Actions workflows requires
-[`actionlint`](https://github.com/rhysd/actionlint/blob/main/docs/install.md). If
-it is not installed, that check is skipped with a warning.
+[`actionlint`](https://github.com/rhysd/actionlint/blob/main/docs/install.md).
+If it is not installed, that check is skipped with a warning.
 
 Run all formatters with:
 
@@ -178,21 +214,46 @@ $ just format
 If files are not formatted correctly, this command rewrites them and exits with
 an error so that you can review the changes.
 
-Run browser-based GUI tests with:
+Prepare the GUI fixture crates once with:
+
+```console
+$ just prepare-gui-tests
+```
+
+The builder follows `DOCSRS_BUILDER_CLI_MODE`, so it normally runs on the host
+on amd64 Linux and through the packaged builder image elsewhere. The generated
+fixture data remains in PostgreSQL and S3.
+
+Run the GUI tests against a temporary host web server with:
 
 ```console
 $ just run-gui-tests
 ```
 
+This reuses the existing fixtures, so changes limited to templates, CSS,
+JavaScript, or web behavior do not require another preparation step. Fixture
+data remains available after `just compose-down`, while
+`just compose-down-and-wipe` removes it.
+
+To reproduce the container integration setup used in CI, run:
+
+```console
+$ DOCSRS_CLI_MODE=docker DOCSRS_BUILDER_CLI_MODE=docker \
+    just prepare-gui-tests run-gui-tests-e2e
+```
+
+This applies migrations through the packaged admin image, builds fixtures
+through the packaged builder image, and serves them with the packaged web image.
+The Node/Puppeteer browser runner remains on the host.
+
 These tests use
 [browser-ui-test](https://github.com/GuillaumeGomez/browser-UI-test/); its
 [script documentation](https://github.com/GuillaumeGomez/browser-UI-test/blob/main/goml-script.md)
-describes the test format. To run the browser test runner manually against an
-already-running web server, install the package and invoke the script directly:
+describes the test format. To run only the browser assertions against a web
+server already listening on port 3000, use:
 
 ```console
-$ npm install browser-ui-test
-$ node gui-tests/tester.js
+$ just run-gui-browser-tests
 ```
 
 The test suite needs at least 4096 open file descriptors. If tests fail or time

docker-compose.yml

@@ -275,22 +275,6 @@ services:
             - metrics
             - full
 
-    gui_tests:
-        platform: "linux/amd64"
-        build:
-            context: .
-            dockerfile: ./dockerfiles/Dockerfile-gui-tests
-            <<: *docker-cache
-        network_mode: "host"
-        extra_hosts:
-            - "host.docker.internal:host-gateway"
-        volumes:
-            - "${PWD}:/build/out"
-        profiles:
-            # gui_tests should not be run as background daemon.
-            # Run via `just run-gui-tests`.
-            - manual
-
 volumes:
     postgres-data: {}
     minio-data: {}