From 6144303d95028ffdbfe083c22a5dd045216355cc Mon Sep 17 00:00:00 2001 From: Michael Yankelev Date: Wed, 19 Aug 2026 18:23:57 +0200 Subject: [PATCH 1/5] chore(ci): bound and cache the web-e2e Playwright install The install step normally takes 24 seconds, but it intermittently stops making progress for 10-15 minutes and ends only when someone cancels the job. Three consecutive attempts on one PR never reached the suite at all. `--with-deps` bundles a browser download and an apt install, so neither the logs nor a timeout could attribute the stall. Split them, cache the browser on the resolved Playwright version, and cap each attempt so a stall retries rather than running unbounded. --- .github/workflows/web-e2e.yml | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index 010f524d2..de94f99b8 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -169,8 +169,44 @@ jobs: env: VITE_E2E_HOOK: 'true' + # `--with-deps` bundles two independent failure surfaces: a browser + # download and an apt install. Either can stall a mirror connection that + # has no read timeout, and bundled they are unattributable. Split, cached + # and bounded — each attempt is capped so a stall retries instead of + # running until someone cancels the job. + - name: Resolve the Playwright version + id: playwright + working-directory: tests/web-e2e + run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + id: playwright-cache + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.playwright.outputs.version }} + - name: Install Playwright Chromium - run: pnpm --filter @cipherbox/web-e2e exec playwright install --with-deps chromium + if: steps.playwright-cache.outputs.cache-hit != 'true' + timeout-minutes: 8 + run: | + for attempt in 1 2 3; do + if timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install chromium; then + exit 0 + fi + echo "::warning::playwright install chromium did not finish within 150s (attempt $attempt)" + done + exit 1 + + - name: Install the Playwright system dependencies + timeout-minutes: 8 + run: | + for attempt in 1 2 3; do + if timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium; then + exit 0 + fi + echo "::warning::playwright install-deps did not finish within 150s (attempt $attempt)" + done + exit 1 # Named `test:e2e`, not `test`: the workspace-wide `Test` gate runs no # suite that needs a live stack. From 1668272de0ee3608edbc7845ed102fcd45c62f82 Mon Sep 17 00:00:00 2001 From: Michael Yankelev Date: Wed, 19 Aug 2026 18:41:04 +0200 Subject: [PATCH 2/5] chore(ci): retry only a stalled Playwright install, not a failed one --- .github/workflows/web-e2e.yml | 36 ++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index de94f99b8..fa87438c7 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -169,11 +169,9 @@ jobs: env: VITE_E2E_HOOK: 'true' - # `--with-deps` bundles two independent failure surfaces: a browser - # download and an apt install. Either can stall a mirror connection that - # has no read timeout, and bundled they are unattributable. Split, cached - # and bounded — each attempt is capped so a stall retries instead of - # running until someone cancels the job. + # `--with-deps` bundles a browser download with an apt install, so a stall + # in either is unattributable. A step-level timeout alone would not help: + # it kills the step, not the stalled attempt, so nothing retries. - name: Resolve the Playwright version id: playwright working-directory: tests/web-e2e @@ -190,22 +188,42 @@ jobs: timeout-minutes: 8 run: | for attempt in 1 2 3; do - if timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install chromium; then + install_status=0 + timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install chromium || install_status=$? + if [ "$install_status" -eq 0 ]; then exit 0 fi - echo "::warning::playwright install chromium did not finish within 150s (attempt $attempt)" + # Only a stall is worth retrying. 124 is `timeout`'s own code; any + # other status is the install failing, and repeating it three times + # would bury the cause this step exists to surface. + if [ "$install_status" -ne 124 ]; then + echo "::error::playwright install chromium failed with status $install_status" + exit "$install_status" + fi + echo "::warning::playwright install chromium stalled past 150s (attempt $attempt of 3)" done + echo "::error::playwright install chromium stalled on every attempt" exit 1 - name: Install the Playwright system dependencies timeout-minutes: 8 run: | for attempt in 1 2 3; do - if timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium; then + install_status=0 + timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium || install_status=$? + if [ "$install_status" -eq 0 ]; then exit 0 fi - echo "::warning::playwright install-deps did not finish within 150s (attempt $attempt)" + # Only a stall is worth retrying. 124 is `timeout`'s own code; any + # other status is the install failing, and repeating it three times + # would bury the cause this step exists to surface. + if [ "$install_status" -ne 124 ]; then + echo "::error::playwright install-deps failed with status $install_status" + exit "$install_status" + fi + echo "::warning::playwright install-deps stalled past 150s (attempt $attempt of 3)" done + echo "::error::playwright install-deps stalled on every attempt" exit 1 # Named `test:e2e`, not `test`: the workspace-wide `Test` gate runs no From 694201a515218f692d2f71f76946575e4a0b8c8c Mon Sep 17 00:00:00 2001 From: Michael Yankelev Date: Wed, 19 Aug 2026 18:52:08 +0200 Subject: [PATCH 3/5] chore(ci): kill a Playwright install that ignores SIGTERM, and record its true status --- .github/workflows/web-e2e.yml | 50 +++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index fa87438c7..091cd1bad 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -185,18 +185,23 @@ jobs: - name: Install Playwright Chromium if: steps.playwright-cache.outputs.cache-hit != 'true' - timeout-minutes: 8 + timeout-minutes: 10 run: | + # The marker records the install's own status, so a stall is told + # apart from the command exiting 124 on its own. `--kill-after` + # matters most: without it `timeout` waits forever on a child that + # ignores SIGTERM, which is the stall this step exists to bound. + marker="$RUNNER_TEMP/playwright-browser-status" for attempt in 1 2 3; do - install_status=0 - timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install chromium || install_status=$? - if [ "$install_status" -eq 0 ]; then - exit 0 - fi - # Only a stall is worth retrying. 124 is `timeout`'s own code; any - # other status is the install failing, and repeating it three times - # would bury the cause this step exists to surface. - if [ "$install_status" -ne 124 ]; then + rm -f "$marker" + # shellcheck disable=SC2016 # the inner shell expands $? and $1, not this one + timeout --kill-after=30 150 bash -c \ + 'pnpm --filter @cipherbox/web-e2e exec playwright install chromium; echo "$?" > "$1"' _ "$marker" || true + if [ -f "$marker" ]; then + install_status=$(cat "$marker") + if [ "$install_status" -eq 0 ]; then + exit 0 + fi echo "::error::playwright install chromium failed with status $install_status" exit "$install_status" fi @@ -206,18 +211,23 @@ jobs: exit 1 - name: Install the Playwright system dependencies - timeout-minutes: 8 + timeout-minutes: 10 run: | + # The marker records the install's own status, so a stall is told + # apart from the command exiting 124 on its own. `--kill-after` + # matters most: without it `timeout` waits forever on a child that + # ignores SIGTERM, which is the stall this step exists to bound. + marker="$RUNNER_TEMP/playwright-deps-status" for attempt in 1 2 3; do - install_status=0 - timeout 150 pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium || install_status=$? - if [ "$install_status" -eq 0 ]; then - exit 0 - fi - # Only a stall is worth retrying. 124 is `timeout`'s own code; any - # other status is the install failing, and repeating it three times - # would bury the cause this step exists to surface. - if [ "$install_status" -ne 124 ]; then + rm -f "$marker" + # shellcheck disable=SC2016 # the inner shell expands $? and $1, not this one + timeout --kill-after=30 150 bash -c \ + 'pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium; echo "$?" > "$1"' _ "$marker" || true + if [ -f "$marker" ]; then + install_status=$(cat "$marker") + if [ "$install_status" -eq 0 ]; then + exit 0 + fi echo "::error::playwright install-deps failed with status $install_status" exit "$install_status" fi From 4bf0eadbb387bcf78e09badf1d75d94c243c7979 Mon Sep 17 00:00:00 2001 From: Michael Yankelev Date: Wed, 19 Aug 2026 19:03:26 +0200 Subject: [PATCH 4/5] chore(ci): stop retrying the Playwright apt install, which poisons its own retry A killed attempt leaves a root-owned apt-get holding the dpkg lock, because Playwright runs install-deps under sudo and this shell cannot reap it. The second attempt then died on that lock rather than the original stall, so the retry turned a slow mirror into a hard failure. One bounded attempt instead. --- .github/workflows/web-e2e.yml | 39 +++++++++++++++-------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index 091cd1bad..f2230d1a1 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -210,30 +210,25 @@ jobs: echo "::error::playwright install chromium stalled on every attempt" exit 1 + # Deliberately no retry: Playwright runs this under sudo, so a killed + # attempt leaves a root-owned `apt-get` holding the dpkg lock that this + # shell cannot reap, and the next attempt dies on that lock instead of the + # original stall. One attempt, bounded generously — a slow mirror is slow, + # not stuck. - name: Install the Playwright system dependencies - timeout-minutes: 10 + timeout-minutes: 12 run: | - # The marker records the install's own status, so a stall is told - # apart from the command exiting 124 on its own. `--kill-after` - # matters most: without it `timeout` waits forever on a child that - # ignores SIGTERM, which is the stall this step exists to bound. - marker="$RUNNER_TEMP/playwright-deps-status" - for attempt in 1 2 3; do - rm -f "$marker" - # shellcheck disable=SC2016 # the inner shell expands $? and $1, not this one - timeout --kill-after=30 150 bash -c \ - 'pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium; echo "$?" > "$1"' _ "$marker" || true - if [ -f "$marker" ]; then - install_status=$(cat "$marker") - if [ "$install_status" -eq 0 ]; then - exit 0 - fi - echo "::error::playwright install-deps failed with status $install_status" - exit "$install_status" - fi - echo "::warning::playwright install-deps stalled past 150s (attempt $attempt of 3)" - done - echo "::error::playwright install-deps stalled on every attempt" + install_status=0 + timeout --kill-after=30 600 \ + pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium || install_status=$? + if [ "$install_status" -eq 0 ]; then + exit 0 + fi + if [ "$install_status" -eq 124 ] || [ "$install_status" -eq 137 ]; then + echo "::error::playwright install-deps did not finish within 600s" + else + echo "::error::playwright install-deps failed with status $install_status" + fi exit 1 # Named `test:e2e`, not `test`: the workspace-wide `Test` gate runs no From 6a3472f599f086b1f126367ccc3665ffca45f848 Mon Sep 17 00:00:00 2001 From: Michael Yankelev Date: Wed, 19 Aug 2026 19:12:36 +0200 Subject: [PATCH 5/5] chore(ci): let a stalled Playwright apt install warn instead of gating the suite --- .github/workflows/web-e2e.yml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index f2230d1a1..4f72cf729 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -210,26 +210,21 @@ jobs: echo "::error::playwright install chromium stalled on every attempt" exit 1 - # Deliberately no retry: Playwright runs this under sudo, so a killed - # attempt leaves a root-owned `apt-get` holding the dpkg lock that this - # shell cannot reap, and the next attempt dies on that lock instead of the - # original stall. One attempt, bounded generously — a slow mirror is slow, - # not stuck. + # The runner image already carries Chromium's shared libraries, so this + # is a no-op on a healthy runner — a passing run installs nothing and + # takes seconds. It stays only to cover an image that drops one. + # + # So a stall here must not gate the suite: apt is bounded, never retried + # (Playwright runs it under sudo, and a killed attempt leaves a root-owned + # apt-get holding the dpkg lock that the next attempt dies on), and a + # failure warns rather than fails. If a library really is missing, the + # smoke slice says so in the browser launch error. - name: Install the Playwright system dependencies - timeout-minutes: 12 + timeout-minutes: 6 run: | - install_status=0 - timeout --kill-after=30 600 \ - pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium || install_status=$? - if [ "$install_status" -eq 0 ]; then - exit 0 - fi - if [ "$install_status" -eq 124 ] || [ "$install_status" -eq 137 ]; then - echo "::error::playwright install-deps did not finish within 600s" - else - echo "::error::playwright install-deps failed with status $install_status" - fi - exit 1 + timeout --kill-after=30 240 \ + pnpm --filter @cipherbox/web-e2e exec playwright install-deps chromium \ + || echo "::warning::playwright install-deps did not complete; the smoke slice shows whether a library is actually missing" # Named `test:e2e`, not `test`: the workspace-wide `Test` gate runs no # suite that needs a live stack.