From dc0aee212a3baf0b249af12cf21a8ce4dcd87906 Mon Sep 17 00:00:00 2001 From: Jack Edwards Date: Fri, 31 Jul 2026 20:06:59 -0500 Subject: [PATCH 1/4] Deploy over native ssh and scp instead of third-party actions appleboy/scp-action and appleboy/ssh-action are composite actions that download a drone-scp or drone-ssh binary from GitHub releases at run time, with no checksum verification. Pinning those actions to a commit SHA does not cover the code that actually runs, and this is the workflow that holds the production SSH key. The OpenSSH client on the hosted runners covers everything this workflow needs: three single-file copies and five short remote commands. A host alias in ~/.ssh/config keeps the per-step configuration out of the way. Neither action was verifying the app server host key, since no fingerprint was ever configured. Host key checking is now strict and reads from a known_hosts file, so deploying requires a new APPSERVER_SSH_KNOWN_HOSTS secret. The workflow fails early with instructions when it is missing. Co-Authored-By: Claude Opus 5 --- .github/workflows/deploy-to-environment.yml | 114 +++++++++----------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/.github/workflows/deploy-to-environment.yml b/.github/workflows/deploy-to-environment.yml index fd47e022..43b78fd2 100644 --- a/.github/workflows/deploy-to-environment.yml +++ b/.github/workflows/deploy-to-environment.yml @@ -24,83 +24,73 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Configure SSH + env: + SSH_HOST: ${{ secrets.APPSERVER_SSH_HOST }} + SSH_PORT: ${{ secrets.APPSERVER_SSH_PORT }} + SSH_USER: ${{ secrets.APPSERVER_SSH_USER }} + SSH_PRIVATE_KEY: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.APPSERVER_SSH_KNOWN_HOSTS }} + run: | + if [ -z "$SSH_KNOWN_HOSTS" ]; then + echo "The APPSERVER_SSH_KNOWN_HOSTS secret is empty." >&2 + echo "Populate it with the output of: ssh-keyscan -p " >&2 + exit 1 + fi + + mkdir -p ~/.ssh + chmod 700 ~/.ssh + + printf '%s\n' "$SSH_PRIVATE_KEY" > ~/.ssh/id_deploy + chmod 600 ~/.ssh/id_deploy + + printf '%s\n' "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/known_hosts + + { + echo "Host appserver" + echo " HostName $SSH_HOST" + echo " Port $SSH_PORT" + echo " User $SSH_USER" + echo " IdentityFile ~/.ssh/id_deploy" + echo " IdentitiesOnly yes" + echo " StrictHostKeyChecking yes" + } > ~/.ssh/config + chmod 600 ~/.ssh/config + - name: Stop service - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - script: | + run: | + ssh appserver ' if [ "$(systemctl --user is-active crypter.service)" = "active" ]; then echo "Stopping service" - systemctl --user stop crypter.service; + systemctl --user stop crypter.service fi + ' - name: Push latest systemctl service file - uses: appleboy/scp-action@ff85246acaad7bdce478db94a363cd2bf7c90345 # v1.0.0 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - source: Environments/${{ github.event.inputs.environment }}/crypter.service - target: .config/systemd/user/ - strip_components: 2 - + env: + environment_name: ${{ github.event.inputs.environment }} + run: | + ssh appserver 'mkdir -p .config/systemd/user' + scp "Environments/$environment_name/crypter.service" appserver:.config/systemd/user/ + - name: Reload systemctl daemon - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - script: systemctl --user daemon-reload + run: ssh appserver 'systemctl --user daemon-reload' - name: Push latest Docker Compose file - uses: appleboy/scp-action@ff85246acaad7bdce478db94a363cd2bf7c90345 # v1.0.0 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - source: docker-compose.yml - target: crypter-web-container/ + run: | + ssh appserver 'mkdir -p crypter-web-container' + scp docker-compose.yml appserver:crypter-web-container/ - name: Push latest Docker Compose override file if deploying to Staging server if: github.event.inputs.environment == 'staging' - uses: appleboy/scp-action@ff85246acaad7bdce478db94a363cd2bf7c90345 # v1.0.0 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - source: docker-compose.override.yml - target: crypter-web-container/ + run: scp docker-compose.override.yml appserver:crypter-web-container/ - name: Pull latest images - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - script: docker compose --project-directory crypter-web-container --profile ${{ env.docker_compose_profile }} pull + run: ssh appserver 'docker compose --project-directory crypter-web-container --profile ${{ env.docker_compose_profile }} pull' - name: Migrate database - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - script: docker compose --project-directory crypter-web-container --profile ${{ env.docker_compose_profile }} run api /app/efbundle + run: ssh appserver 'docker compose --project-directory crypter-web-container --profile ${{ env.docker_compose_profile }} run api /app/efbundle' - name: Start service - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.APPSERVER_SSH_HOST }} - port: ${{ secrets.APPSERVER_SSH_PORT }} - username: ${{ secrets.APPSERVER_SSH_USER }} - key: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} - script: systemctl --user start crypter.service + run: ssh appserver 'systemctl --user start crypter.service' From 06ce0b1555462603b31437885d2e9cd084a10311 Mon Sep 17 00:00:00 2001 From: Jack Edwards Date: Fri, 31 Jul 2026 21:32:54 -0500 Subject: [PATCH 2/4] Document recording the app server host key Deploying now requires an APPSERVER_SSH_KNOWN_HOSTS secret per environment. Capturing the host key belongs with the rest of the one-time server setup, next to the SSH user it authenticates against. Co-Authored-By: Claude Opus 5 --- .../Server Setup/Web Server Setup.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Documentation/Production/Server Setup/Web Server Setup.md b/Documentation/Production/Server Setup/Web Server Setup.md index eed62091..d783924c 100644 --- a/Documentation/Production/Server Setup/Web Server Setup.md +++ b/Documentation/Production/Server Setup/Web Server Setup.md @@ -14,6 +14,27 @@ Create an SSH user and add corresponding details to the environment secrets with The user will need permissions to Docker, so add the user to the `docker` group. +## Record the host key + +The deploy workflow verifies the host key of the server it connects to, so record that key while the server is being set up. + +On the server, print the fingerprint of each host key: + +`ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub` + +From a workstation, capture the same keys in `known_hosts` format and print their fingerprints: + +```bash +ssh-keyscan -p > known_hosts +ssh-keygen -lf known_hosts +``` + +The fingerprints must match. Comparing them is what makes the captured key trustworthy, because `ssh-keyscan` on its own only reports whatever answers on the network. + +Add the contents of `known_hosts` to the environment secrets as `APPSERVER_SSH_KNOWN_HOSTS`. Every environment has its own server and its own host key, so record one for each. + +Rebuilding a server generates a new host key. Deploys fail with `REMOTE HOST IDENTIFICATION HAS CHANGED` until the secret is updated to match. + ## Copy the .env file Locate the `.env` file at the root of this repository, [here](../../../.env). From 092b0eaaed1394c87460a852df32aec1fa6e6050 Mon Sep 17 00:00:00 2001 From: Jack Edwards Date: Fri, 31 Jul 2026 22:42:15 -0500 Subject: [PATCH 3/4] Record the host key from the workstation, not the server The previous instructions checked a single Ed25519 fingerprint against the server, but ssh-keyscan captures every key type the server offers unless it is told otherwise, so the remaining keys went into the secret unverified. Not every server has an Ed25519 key to begin with. Reading fingerprints off the server was also circular. Reaching it means connecting over ssh, at which point the workstation already holds the key that step was meant to confirm. known_hosts is indexed by host and port, and ssh writes a bare hostname only for port 22. An entry copied from a workstation that connects on another port is never consulted, and the deploy fails under strict host key checking as though no key had been recorded. Co-Authored-By: Claude Opus 5 --- .../Production/Server Setup/Web Server Setup.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Documentation/Production/Server Setup/Web Server Setup.md b/Documentation/Production/Server Setup/Web Server Setup.md index d783924c..5b1c1cdf 100644 --- a/Documentation/Production/Server Setup/Web Server Setup.md +++ b/Documentation/Production/Server Setup/Web Server Setup.md @@ -18,18 +18,25 @@ The user will need permissions to Docker, so add the user to the `docker` group. The deploy workflow verifies the host key of the server it connects to, so record that key while the server is being set up. -On the server, print the fingerprint of each host key: +The secret holds `known_hosts` lines exactly as ssh writes them. The host field has to match `APPSERVER_SSH_HOST` and `APPSERVER_SSH_PORT`: a bare hostname on port 22, and `[host]:port` on any other port. A line recorded under a different name or port is never consulted, so the deploy fails as though no key had been recorded at all. -`ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub` +Take the key from a workstation that already connects to the server, which by this point is whichever one was used to set it up. Print the entry it trusts: -From a workstation, capture the same keys in `known_hosts` format and print their fingerprints: +```bash +ssh-keygen -F '[]:' -f ~/.ssh/known_hosts +``` + +Drop the brackets and the port if that workstation connects over port 22. An entry that already carries the port the deploy uses can go straight into the secret, ignoring the leading comment line. + +An entry recorded under any other port has to be recaptured under the right one, then checked against the entry already trusted: ```bash -ssh-keyscan -p > known_hosts +ssh-keyscan -t -p > known_hosts ssh-keygen -lf known_hosts +ssh-keygen -F '' -f ~/.ssh/known_hosts | ssh-keygen -lf - ``` -The fingerprints must match. Comparing them is what makes the captured key trustworthy, because `ssh-keyscan` on its own only reports whatever answers on the network. +The fingerprints must match. Comparing them is what makes the scan trustworthy, because `ssh-keyscan` on its own only reports whatever answers on the network. Pass `-t` for the key type that was checked, so nothing unverified lands in the secret. Add the contents of `known_hosts` to the environment secrets as `APPSERVER_SSH_KNOWN_HOSTS`. Every environment has its own server and its own host key, so record one for each. From a79c7c869350a19a8e6133d12d427c1a140e7396 Mon Sep 17 00:00:00 2001 From: Jack Edwards Date: Fri, 31 Jul 2026 22:50:46 -0500 Subject: [PATCH 4/4] Bound the deploy job and fail fast on bad SSH input appleboy/ssh-action applied a ten minute command_timeout that native ssh has no equivalent for, so a wedged image pull or a migration blocked on a lock would otherwise run until the six hour job default expires. BatchMode stops ssh from falling back to password and keyboard-interactive authentication when the key is rejected. Those attempts read an empty passphrase from a closed stdin and fail three times over before the step gives up, reporting a permission problem rather than the key problem. Co-Authored-By: Claude Opus 5 --- .github/workflows/deploy-to-environment.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/deploy-to-environment.yml b/.github/workflows/deploy-to-environment.yml index 43b78fd2..f1e7700e 100644 --- a/.github/workflows/deploy-to-environment.yml +++ b/.github/workflows/deploy-to-environment.yml @@ -17,6 +17,7 @@ env: jobs: deploy-web-container: runs-on: ubuntu-latest + timeout-minutes: 30 environment: name: ${{ github.event.inputs.environment }} @@ -32,6 +33,11 @@ jobs: SSH_PRIVATE_KEY: ${{ secrets.APPSERVER_SSH_PRIVATE_KEY }} SSH_KNOWN_HOSTS: ${{ secrets.APPSERVER_SSH_KNOWN_HOSTS }} run: | + if [ -z "$SSH_PRIVATE_KEY" ]; then + echo "The APPSERVER_SSH_PRIVATE_KEY secret is empty." >&2 + exit 1 + fi + if [ -z "$SSH_KNOWN_HOSTS" ]; then echo "The APPSERVER_SSH_KNOWN_HOSTS secret is empty." >&2 echo "Populate it with the output of: ssh-keyscan -p " >&2 @@ -55,6 +61,7 @@ jobs: echo " IdentityFile ~/.ssh/id_deploy" echo " IdentitiesOnly yes" echo " StrictHostKeyChecking yes" + echo " BatchMode yes" } > ~/.ssh/config chmod 600 ~/.ssh/config