fix(seedbox): give the watchdog a bus to systemd, and stop passing off a daemon we watched die

src/lib/seedbox/provision.test.ts

@@ -1,12 +1,16 @@
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi, beforeEach } from 'vitest';
 
 import {
   buildProvisionScript,
   generateSeedboxToken,
   parseSteps,
+  provisionTorlink,
   DEFAULT_FILES_PORT,
   DEFAULT_SERVE_PORT,
 } from './provision';
+import { execRemote } from './ssh-transport';
+
+vi.mock('./ssh-transport', () => ({ execRemote: vi.fn() }));
 
 describe('seedbox provisioner', () => {
   describe('generateSeedboxToken', () => {
@@ -233,3 +237,107 @@ describe('seedbox provisioner — install must leave a daemon that stays up', ()
     expect(script).not.toContain('WATCH="$HOME/Downloads/watch"\nmkdir');
   });
 });
+
+describe('watchdog — must be able to reach systemd from cron', () => {
+  const script = buildProvisionScript('TOK123_-', DEFAULT_SERVE_PORT, DEFAULT_FILES_PORT);
+  const wd = script.match(/cat > "\$WD" <<WDEOF[\s\S]*?\nWDEOF/)?.[0] ?? '';
+
+  it('extracts the watchdog body', () => {
+    expect(wd).toBeTruthy();
+  });
+
+  // Comments in this block mention systemctl by name, so order must be asserted
+  // against real commands only — matching raw text would prove nothing.
+  const wdCommands = wd
+    .split('\n')
+    .filter((line) => !line.trim().startsWith('#'))
+    .join('\n');
+
+  it('sets XDG_RUNTIME_DIR/DBUS before calling systemctl --user', () => {
+    // cron has no login session: without these, `systemctl --user` fails with
+    // "Failed to connect to user scope bus" on every run, so the watchdog never
+    // actually restarted the units it exists to restart.
+    expect(wdCommands).toContain('XDG_RUNTIME_DIR');
+    expect(wdCommands).toContain('DBUS_SESSION_BUS_ADDRESS');
+    expect(wdCommands.indexOf('XDG_RUNTIME_DIR')).toBeLessThan(wdCommands.indexOf('systemctl --user'));
+    expect(wdCommands.indexOf('DBUS_SESSION_BUS_ADDRESS')).toBeLessThan(
+      wdCommands.indexOf('systemctl --user')
+    );
+  });
+
+  it('resolves the uid when the watchdog runs, not when it is written', () => {
+    // A bare $(id -u) inside the unquoted heredoc would bake the PROVISIONING
+    // shell's uid into the file.
+    expect(wd).toContain('/run/user/\\$(id -u)');
+    expect(wd).not.toMatch(/XDG_RUNTIME_DIR="\/run\/user\/\d+"/);
+  });
+
+  it('re-probes health before falling back to an unsupervised daemon', () => {
+    // Spawning a bare daemon while the unit is mid-restart races it for the
+    // port; the loser then crash-loops forever against a port it cannot bind.
+    const restart = wd.indexOf('systemctl --user restart');
+    const fallback = wd.indexOf('--daemon');
+    expect(restart).toBeGreaterThan(-1);
+    expect(fallback).toBeGreaterThan(restart);
+    const between = wd.slice(restart, fallback);
+    expect(between).toMatch(/curl[^\n]*\/health/);
+    expect(between).toMatch(/sleep \d+/);
+    // The old `restart ... && exit 0` short-circuit treated a queued restart as
+    // proof the port was bound.
+    expect(wd).not.toContain('restart torlink-serve.service torlink-files.service >/dev/null 2>&1 && exit 0');
+  });
+
+  it('retries the stability probe so one dropped packet cannot fail a healthy box', () => {
+    const stable = script.slice(script.indexOf('--- does it STAY up? ---'));
+    expect(stable).toContain('STABLE=0');
+    expect(stable).toMatch(/for _ in 1 2 3/);
+  });
+});
+
+describe('provisionTorlink — a daemon we watched die is not a successful install', () => {
+  const ssh = {
+    host: 'seedbox.example.com',
+    port: 22,
+    user: 'ubuntu',
+    privateKey: 'KEY',
+    privateKeyPath: null,
+    watchDir: null,
+    addCommand: 'true',
+  };
+
+  const runWith = async (lines: string[]) => {
+    vi.mocked(execRemote).mockResolvedValue({ stdout: lines.join('\n'), stderr: '' } as never);
+    return provisionTorlink(ssh, { token: 'TOK' });
+  };
+
+  beforeEach(() => vi.mocked(execRemote).mockReset());
+
+  it('fails the install when the daemon died inside the stability window', async () => {
+    const result = await runWith([
+      'STEP|install|ok|npm i -g @profullstack/torlink@latest',
+      'STEP|serve|ok|add-API on 9161',
+      'STEP|health|ok|{"ok":true}',
+      'STEP|stable|fail|torlink started but died within ~15s — Main process exited',
+      'RESULT|ok',
+    ]);
+    expect(result.ok).toBe(false);
+    // No token ⇒ the route 502s instead of wiring a dead box into the config.
+    expect(result.token).toBeNull();
+    expect(result.steps.find((s) => s.name === 'stable')?.status).toBe('fail');
+  });
+
+  it('succeeds when the daemon is still serving after the window', async () => {
+    const result = await runWith([
+      'STEP|serve|ok|add-API on 9161',
+      'STEP|stable|ok|still serving 15s after start',
+      'RESULT|ok',
+    ]);
+    expect(result.ok).toBe(true);
+    expect(result.token).toBe('TOK');
+  });
+
+  it('does not require a stable step that an older script never emitted', async () => {
+    const result = await runWith(['STEP|serve|ok|add-API on 9161', 'RESULT|ok']);
+    expect(result.ok).toBe(true);
+  });
+});

src/lib/seedbox/provision.ts

@@ -366,7 +366,15 @@ fi
 # provisioner that reports success for a daemon which is already gone is worse
 # than useless. Re-probe after a delay and report what killed it.
 sleep 12
-if curl -fsS -m 5 "http://127.0.0.1:$SERVE_PORT/health" >/dev/null 2>&1; then
+# Retry rather than judging on a single curl: this step now GATES the whole
+# install (see provisionTorlink), so one dropped probe must not fail a box whose
+# daemon is actually healthy.
+STABLE=0
+for _ in 1 2 3; do
+  if curl -fsS -m 5 "http://127.0.0.1:$SERVE_PORT/health" >/dev/null 2>&1; then STABLE=1; break; fi
+  sleep 2
+done
+if [ "$STABLE" = "1" ]; then
   emit stable ok "still serving 15s after start"
 else
   emit stable fail "torlink started but died within ~15s — $(why_dead)"
@@ -470,13 +478,30 @@ cat > "$WD" <<WDEOF
 #!/usr/bin/env bash
 # Installed by media-streamer. Restarts torlink when its add-API stops answering.
 export PATH="$UNIT_PATH"
+# cron runs with no login session, so \`systemctl --user\` has no bus to talk to
+# and dies with "Failed to connect to user scope bus". That failure is silent:
+# the restart below was skipped on EVERY run and the watchdog fell straight
+# through to the unsupervised fallback, which then fought the still-enabled unit
+# for $SERVE_PORT. Point systemctl at the lingering user manager explicitly.
+# \\$(id -u) is evaluated when the watchdog RUNS, not when this heredoc is written.
+export XDG_RUNTIME_DIR="/run/user/\\$(id -u)"
+export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/\\$(id -u)/bus"
 curl -fsS -m 5 "http://127.0.0.1:$SERVE_PORT/health" >/dev/null 2>&1 && exit 0
 echo "\\$(date -Is) /health did not answer on $SERVE_PORT — restarting torlink" >> "$HOME/.torlnk-watchdog.log"
 # Clear any 'failed' state first: systemctl restart refuses a unit that tripped
 # the start limit ("start request repeated too quickly"), so without this the
 # watchdog would silently no-op on exactly the boxes that need it most.
 systemctl --user reset-failed torlink-serve.service torlink-files.service >/dev/null 2>&1 || true
-systemctl --user restart torlink-serve.service torlink-files.service >/dev/null 2>&1 && exit 0
+systemctl --user restart torlink-serve.service torlink-files.service >/dev/null 2>&1
+# A systemd restart is asynchronous, so a zero exit status does not mean the port
+# is bound yet. Re-probe before falling back: spawning a bare daemon while the
+# unit is mid-restart races it for $SERVE_PORT, and whichever loses crash-loops
+# forever against a port it can never bind.
+for _ in 1 2 3 4 5 6; do
+  curl -fsS -m 3 "http://127.0.0.1:$SERVE_PORT/health" >/dev/null 2>&1 && exit 0
+  sleep 2
+done
+# Still dead ⇒ systemd is not managing this box (or cannot). Unsupervised it is.
 export TORLINK_API_TOKEN="$TOK"
 export TORLINK_FILES_TOKEN="$TOK"
 export TORLINK_MAX_DOWNLOADS=2
@@ -549,6 +574,13 @@ export async function provisionTorlink(
 
   const { steps, result } = parseSteps(raw);
   const serveOk = steps.some((s) => s.name === 'serve' && s.status === 'ok');
-  const ok = result === 'ok' && serveOk;
+  // A daemon that died inside the stability window is NOT a successful install.
+  // The script already detects this and emits `stable fail`, but success used to
+  // hinge on the `serve` step alone — so the caller wired the fresh token into
+  // the account config and showed a green "installed", and the very next status
+  // poll reported ECONNREFUSED. Reporting success for a daemon we watched die is
+  // exactly the failure this step exists to catch, so let it veto.
+  const stableFailed = steps.some((s) => s.name === 'stable' && s.status === 'fail');
+  const ok = result === 'ok' && serveOk && !stableFailed;
   return { ok, steps, token: ok ? token : null, servePort, filesPort, raw };
 }