From 09d5ea128e9b86e7bcbc716fafb20ab0f9994c55 Mon Sep 17 00:00:00 2001 From: figulusproject <269854178+figulusproject@users.noreply.github.com> Date: Fri, 14 Aug 2026 21:10:13 -0400 Subject: [PATCH] add a status subcommand for --detach jobs, bump zod-commands to 1.1.3 for its per-command override schema support --- CHANGELOG.md | 3 +- apps/cli/package.json | 2 +- apps/cli/src/cliArgs.test.ts | 69 +++++++--- apps/cli/src/cliArgs.ts | 241 ++++++++++++++++++++--------------- apps/cli/src/runCli.test.ts | 204 ++++++++++++++++++++++++++++- apps/cli/src/runCli.ts | 129 ++++++++++++++++++- docs/cli.md | 19 +++ docs/flags.md | 2 +- package-lock.json | 10 +- package.json | 2 +- 10 files changed, 548 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 487b9a3..d2d49bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - `@zod-crawler/components`: a new shared shadcn/Tailwind UI package holding the UI and crawl components moved out of `apps/web`, so `apps/web` and `apps/web-demo` render from the same components. - `tools/cors-proxy`: a self-hosted CORS proxy at `zodcrawler.figulus.dev/proxy` that the demo falls back to when a target API doesn't allow direct cross-origin fetches, rate-limited per IP and capped daily. - `tools/site-router`: routes `zodcrawler.figulus.dev`'s docs, the new demo, and everything else to the right place. +- `@zod-crawler/cli`: a `status` subcommand (`zod-crawler status --output `) reports on a crawl started with `--detach` - running, finished (with its recorded exit code), or crashed - by reading `/zod-crawler.pid` and `/zod-crawler.status.json`, both written by the detached run itself. ### Changed - CI workflow (`ci.yml`) and `apps/cli`/`apps/web`'s Docker images now run on Node 22/24/26 instead of 20/22/24. -- `@zod-crawler/cli`: upgraded from `zod-cli-flags@1.0.2` to its renamed successor, `zod-commands@1.1.2`. `--ids`/`--ids-file` now use `exclusiveGroups` to declare "exactly one of" instead of a hand-written check in the parse transform, so `cli.usage` renders it as `(--ids | --ids-file )`. +- `@zod-crawler/cli`: upgraded from `zod-cli-flags@1.0.2` to its renamed successor, `zod-commands@1.1.3`. `--ids`/`--ids-file` now use `exclusiveGroups` to declare "exactly one of" instead of a hand-written check in the parse transform, so `cli.usage` renders it as `(--ids | --ids-file )`. ## [1.1.0] - 2026-08-12 diff --git a/apps/cli/package.json b/apps/cli/package.json index 60b3b53..2a35135 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -25,7 +25,7 @@ "@zod-crawler/pipeline-node": "^1.1.0", "prettier": "^3.8.3", "zod": "^4.4.3", - "zod-commands": "^1.1.2", + "zod-commands": "^1.1.3", "zod-transformers": "^1.0.0" }, "peerDependencies": { diff --git a/apps/cli/src/cliArgs.test.ts b/apps/cli/src/cliArgs.test.ts index 15e5c47..d36af7d 100644 --- a/apps/cli/src/cliArgs.test.ts +++ b/apps/cli/src/cliArgs.test.ts @@ -9,7 +9,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids", "a, b ,a,c", "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.ids).toEqual(["a", "b", "c"]); expect(result.settings.delayMs).toBe(7500); expect(result.settings.schemaName).toBe("InferredSchema"); @@ -31,7 +31,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.emitCrawlCandidates).toBe(true); expect(result.settings.crawlCandidatesFormat).toBe("txt"); } @@ -50,7 +50,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.crawlCandidatesFormat).toBe("yaml"); } }); @@ -96,7 +96,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.useZodTransformers).toBe(true); } }); @@ -111,7 +111,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.stopOnError).toBe(true); } }); @@ -120,7 +120,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.detach).toBe(false); } }); @@ -130,7 +130,8 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out", flag]); expect(result.ok, `${flag} should parse`).toBe(true); - if (result.ok) expect(result.settings.detach).toBe(true); + if (result.ok && result.command === "crawl") + expect(result.settings.detach).toBe(true); } }); @@ -149,7 +150,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.urlTemplate).toBe("https://example.com/{id}.json"); expect(result.settings.delayMs).toBe(100); expect(result.settings.schemaName).toBe("MySchema"); @@ -264,7 +265,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.redisUrl).toBeUndefined(); expect(result.settings.concurrency).toBeUndefined(); } @@ -282,7 +283,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.redisUrl).toBe("redis://localhost:6379"); expect(result.settings.concurrency).toBe(1); } @@ -293,7 +294,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.redisUrl).toBe("redis://env-host:6379"); } }); @@ -310,7 +311,7 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.redisUrl).toBe("redis://flag-host:6379"); } }); @@ -328,7 +329,8 @@ describe("parseCliArgs", () => { ]); expect(result.ok).toBe(true); - if (result.ok) expect(result.settings.concurrency).toBe(4); + if (result.ok && result.command === "crawl") + expect(result.settings.concurrency).toBe(4); }); it("rejects --concurrency without a Redis URL (flag or env)", () => { @@ -383,7 +385,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.ids).toEqual(["a", "b", "c"]); } }); @@ -410,7 +412,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.ids).toEqual(["a", "b", "c"]); } }); @@ -425,7 +427,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.ids).toEqual(["a", "b", "c"]); } }); @@ -437,7 +439,7 @@ describe("parseCliArgs", () => { const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]); expect(result.ok).toBe(true); - if (result.ok) { + if (result.ok && result.command === "crawl") { expect(result.settings.ids).toEqual(["a", "c", "b"]); } }); @@ -465,4 +467,37 @@ describe("parseCliArgs", () => { if (!result.ok) expect(result.message).toMatch(/--ids-file/); }); }); + + it('accepts an explicit leading "crawl" token, same as omitting it', () => { + const result = parseCliArgs([ + "crawl", + "--ids", + "a", + "--output", + "/tmp/out", + ]); + + expect(result.ok).toBe(true); + if (result.ok && result.command === "crawl") { + expect(result.settings.ids).toEqual(["a"]); + } + }); + + describe("status", () => { + it("parses --output into outputDir", () => { + const result = parseCliArgs(["status", "--output", "/tmp/out"]); + + expect(result.ok).toBe(true); + if (result.ok && result.command === "status") { + expect(result.settings.outputDir).toBe("/tmp/out"); + } + }); + + it("rejects a missing --output", () => { + const result = parseCliArgs(["status"]); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.message).toMatch(/--output/); + }); + }); }); diff --git a/apps/cli/src/cliArgs.ts b/apps/cli/src/cliArgs.ts index b168f96..5eac524 100644 --- a/apps/cli/src/cliArgs.ts +++ b/apps/cli/src/cliArgs.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; import { z } from "zod"; -import { defineCli } from "zod-commands"; +import { defineCli, defineCommands } from "zod-commands"; import { numberString } from "zod-transformers"; import { SCHEMA_NAME_PATTERN, @@ -29,10 +29,16 @@ export interface CliSettings { detach: boolean; } +export interface StatusSettings { + outputDir: string; +} + export type ParseCliArgsResult = - { ok: true; settings: CliSettings } | { ok: false; message: string }; + | { ok: true; command: "crawl"; settings: CliSettings } + | { ok: true; command: "status"; settings: StatusSettings } + | { ok: false; message: string; usage: string }; -const cli = defineCli({ +const crawlCli = defineCli({ flags: { ids: { schema: z.string().optional(), placeholder: "id1,id2,..." }, idsFile: { @@ -85,129 +91,164 @@ const cli = defineCli({ exclusiveGroups: [{ flags: ["ids", "idsFile"], required: true }], }); -export const usage = cli.usage; +const settingsSchema = crawlCli.flagsSchema.transform( + (raw, ctx): CliSettings => { + let rawIds: string[]; + if (raw.ids !== undefined) { + rawIds = raw.ids + .split(",") + .map((id) => id.trim()) + .filter((id) => id.length > 0); + } else { + const idsFileFormat = crawlCandidatesFormatFromExtension(raw.idsFile!); + if (idsFileFormat === undefined) { + ctx.addIssue({ + code: "custom", + message: `--ids-file "${raw.idsFile}" has an unrecognized extension - expected one of: ${CRAWL_CANDIDATES_FORMATS.join(", ")} (or .yml).`, + }); + return z.NEVER; + } + + let fileContents: string; + try { + fileContents = readFileSync(raw.idsFile!, "utf8"); + } catch (error) { + ctx.addIssue({ + code: "custom", + message: `Could not read --ids-file "${raw.idsFile}": ${ + error instanceof Error ? error.message : String(error) + }`, + }); + return z.NEVER; + } + + try { + rawIds = idsParsers[idsFileFormat](fileContents); + } catch (error) { + ctx.addIssue({ + code: "custom", + message: `Could not parse --ids-file "${raw.idsFile}" as ${idsFileFormat}: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + return z.NEVER; + } + } -const settingsSchema = cli.flagsSchema.transform((raw, ctx): CliSettings => { - let rawIds: string[]; - if (raw.ids !== undefined) { - rawIds = raw.ids - .split(",") - .map((id) => id.trim()) - .filter((id) => id.length > 0); - } else { - const idsFileFormat = crawlCandidatesFormatFromExtension(raw.idsFile!); - if (idsFileFormat === undefined) { + const ids = [...new Set(rawIds)]; + if (ids.length === 0) { ctx.addIssue({ code: "custom", - message: `--ids-file "${raw.idsFile}" has an unrecognized extension - expected one of: ${CRAWL_CANDIDATES_FORMATS.join(", ")} (or .yml).`, + message: "At least one id is required (from --ids or --ids-file).", }); return z.NEVER; } - let fileContents: string; - try { - fileContents = readFileSync(raw.idsFile!, "utf8"); - } catch (error) { + if (raw.urlTemplate !== undefined && !raw.urlTemplate.includes("{id}")) { ctx.addIssue({ code: "custom", - message: `Could not read --ids-file "${raw.idsFile}": ${ - error instanceof Error ? error.message : String(error) - }`, + message: `--url-template must contain the literal "{id}" placeholder, got "${raw.urlTemplate}".`, }); return z.NEVER; } - try { - rawIds = idsParsers[idsFileFormat](fileContents); - } catch (error) { + if (raw.output === undefined || raw.output.length === 0) { + ctx.addIssue({ code: "custom", message: "--output is required." }); + return z.NEVER; + } + + const delayMs = raw.delay ?? DEFAULT_DELAY_MS; + + const schemaName = raw.schemaName ?? DEFAULT_SCHEMA_NAME; + if (!SCHEMA_NAME_PATTERN.test(schemaName)) { ctx.addIssue({ code: "custom", - message: `Could not parse --ids-file "${raw.idsFile}" as ${idsFileFormat}: ${ - error instanceof Error ? error.message : String(error) - }`, + message: `--schema-name must be a valid identifier, got "${schemaName}".`, }); return z.NEVER; } - } - const ids = [...new Set(rawIds)]; - if (ids.length === 0) { - ctx.addIssue({ - code: "custom", - message: "At least one id is required (from --ids or --ids-file).", - }); - return z.NEVER; - } - - if (raw.urlTemplate !== undefined && !raw.urlTemplate.includes("{id}")) { - ctx.addIssue({ - code: "custom", - message: `--url-template must contain the literal "{id}" placeholder, got "${raw.urlTemplate}".`, - }); - return z.NEVER; - } + if (raw.crawlCandidatesFormat !== undefined && !raw.emitCrawlCandidates) { + ctx.addIssue({ + code: "custom", + message: + "--crawl-candidates-format requires --emit-crawl-candidates - it only applies to the crawl candidates file.", + }); + return z.NEVER; + } + const crawlCandidatesFormat: CrawlCandidatesFormat = + raw.crawlCandidatesFormat ?? "txt"; - if (raw.output === undefined || raw.output.length === 0) { - ctx.addIssue({ code: "custom", message: "--output is required." }); - return z.NEVER; - } + const redisUrl = raw.redisUrl ?? process.env.REDIS_URL; + if (raw.concurrency !== undefined && redisUrl === undefined) { + ctx.addIssue({ + code: "custom", + message: + "--concurrency requires --redis-url (or a REDIS_URL environment variable) - it only applies to the BullMQ-backed queue.", + }); + return z.NEVER; + } + const concurrency = + redisUrl !== undefined + ? (raw.concurrency ?? DEFAULT_CONCURRENCY) + : undefined; + + return { + ids, + urlTemplate: raw.urlTemplate, + outputDir: raw.output, + delayMs, + schemaName, + emitCrawlCandidates: raw.emitCrawlCandidates, + crawlCandidatesFormat, + useZodTransformers: raw.zodTransformers, + stopOnError: raw.stopOnError, + redisUrl, + concurrency, + detach: raw.detach, + }; + }, +); - const delayMs = raw.delay ?? DEFAULT_DELAY_MS; +const statusCli = defineCli({ + flags: { + output: { schema: z.string().optional(), placeholder: "dir" }, + }, +}); - const schemaName = raw.schemaName ?? DEFAULT_SCHEMA_NAME; - if (!SCHEMA_NAME_PATTERN.test(schemaName)) { - ctx.addIssue({ - code: "custom", - message: `--schema-name must be a valid identifier, got "${schemaName}".`, - }); - return z.NEVER; - } +const statusSettingsSchema = statusCli.flagsSchema.transform( + (raw, ctx): StatusSettings => { + if (raw.output === undefined || raw.output.length === 0) { + ctx.addIssue({ code: "custom", message: "--output is required." }); + return z.NEVER; + } + return { outputDir: raw.output }; + }, +); - if (raw.crawlCandidatesFormat !== undefined && !raw.emitCrawlCandidates) { - ctx.addIssue({ - code: "custom", - message: - "--crawl-candidates-format requires --emit-crawl-candidates - it only applies to the crawl candidates file.", - }); - return z.NEVER; - } - const crawlCandidatesFormat: CrawlCandidatesFormat = - raw.crawlCandidatesFormat ?? "txt"; - - const redisUrl = raw.redisUrl ?? process.env.REDIS_URL; - if (raw.concurrency !== undefined && redisUrl === undefined) { - ctx.addIssue({ - code: "custom", - message: - "--concurrency requires --redis-url (or a REDIS_URL environment variable) - it only applies to the BullMQ-backed queue.", - }); - return z.NEVER; - } - const concurrency = - redisUrl !== undefined - ? (raw.concurrency ?? DEFAULT_CONCURRENCY) - : undefined; - - return { - ids, - urlTemplate: raw.urlTemplate, - outputDir: raw.output, - delayMs, - schemaName, - emitCrawlCandidates: raw.emitCrawlCandidates, - crawlCandidatesFormat, - useZodTransformers: raw.zodTransformers, - stopOnError: raw.stopOnError, - redisUrl, - concurrency, - detach: raw.detach, - }; +const cli = defineCommands({ + commands: { + crawl: { cli: crawlCli, schema: settingsSchema }, + status: { cli: statusCli, schema: statusSettingsSchema }, + }, + defaultCommand: "crawl", }); +export const usage = cli.usage; + export function parseCliArgs(argv: string[]): ParseCliArgsResult { - const result = cli.parse(argv, settingsSchema); + const result = cli.parse(argv); if (!result.success) { - return { ok: false, message: result.error.message }; + const failedUsage = + result.command?.[0] === "status" ? statusCli.usage : crawlCli.usage; + return { ok: false, message: result.error.message, usage: failedUsage }; + } + if (result.command[0] === "status") { + return { + ok: true, + command: "status", + settings: result.data as StatusSettings, + }; } - return { ok: true, settings: result.data }; + return { ok: true, command: "crawl", settings: result.data as CliSettings }; } diff --git a/apps/cli/src/runCli.test.ts b/apps/cli/src/runCli.test.ts index 6dca117..86139c5 100644 --- a/apps/cli/src/runCli.test.ts +++ b/apps/cli/src/runCli.test.ts @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { EventEmitter } from "node:events"; -import { readdir, readFile, mkdtemp, rm } from "node:fs/promises"; +import { readdir, readFile, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -82,6 +82,10 @@ describe("runCli", () => { const cacheFiles = await readdir(path.join(outputDir, "cache")); // Two id cache files plus the manifest. expect(cacheFiles.filter((f) => f !== "manifest.json")).toHaveLength(2); + + const outputFiles = await readdir(outputDir); + expect(outputFiles).not.toContain("zod-crawler.pid"); + expect(outputFiles).not.toContain("zod-crawler.status.json"); }); it("writes new, deduped crawl candidates when --emit-crawl-candidates is set", async () => { @@ -346,8 +350,11 @@ describe("runCli", () => { expect(code).toBe(0); expect(spawn).toHaveBeenCalledTimes(1); - const [command, args] = vi.mocked(spawn).mock.calls[0]; + const [command, args, options] = vi.mocked(spawn).mock.calls[0]; expect(command).toBe(process.execPath); + expect( + (options as { env?: Record }).env, + ).toMatchObject({ ZOD_CRAWLER_DETACHED: "1" }); expect(args).toEqual( expect.arrayContaining([ "--output", @@ -385,6 +392,12 @@ describe("runCli", () => { ), ).toBe(true); + const pidFile = JSON.parse( + await readFile(path.join(outputDir, "zod-crawler.pid"), "utf8"), + ); + expect(pidFile).toMatchObject({ pid: child.pid, ids: 2 }); + expect(typeof pidFile.startedAt).toBe("string"); + consoleLog.mockRestore(); }); @@ -460,4 +473,191 @@ describe("runCli", () => { consoleError.mockRestore(); }); }); + + describe("ZOD_CRAWLER_DETACHED (child-side status tracking)", () => { + const originalEnv = process.env.ZOD_CRAWLER_DETACHED; + + beforeEach(() => { + process.env.ZOD_CRAWLER_DETACHED = "1"; + }); + + afterEach(() => { + if (originalEnv === undefined) delete process.env.ZOD_CRAWLER_DETACHED; + else process.env.ZOD_CRAWLER_DETACHED = originalEnv; + }); + + it("writes zod-crawler.status.json with exit code 0 on success", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => jsonResponse({ title: "Book" })), + ); + + const code = await runCli([ + "--ids", + "a", + "--output", + outputDir, + "--delay", + "100", + "--url-template", + "https://example.com/{id}.json", + ]); + + expect(code).toBe(0); + const statusFile = JSON.parse( + await readFile(path.join(outputDir, "zod-crawler.status.json"), "utf8"), + ); + expect(statusFile.exitCode).toBe(0); + expect(typeof statusFile.finishedAt).toBe("string"); + }); + + it("writes zod-crawler.status.json with a non-zero exit code on failure", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + ({ ok: false, status: 404, statusText: "Not Found" }) as Response, + ), + ); + + const code = await runCli([ + "--ids", + "a", + "--output", + outputDir, + "--delay", + "100", + "--url-template", + "https://example.com/{id}.json", + "--stop-on-error", + ]); + + expect(code).toBe(1); + const statusFile = JSON.parse( + await readFile(path.join(outputDir, "zod-crawler.status.json"), "utf8"), + ); + expect(statusFile.exitCode).toBe(1); + }); + }); + + describe("status", () => { + it("reports no crawl found when there's no pidfile", async () => { + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + const code = await runCli(["status", "--output", outputDir]); + + expect(code).toBe(1); + expect( + consoleLog.mock.calls.some((call) => + String(call[0]).includes("No detached crawl found"), + ), + ).toBe(true); + consoleLog.mockRestore(); + }); + + it("reports running when the pid is alive and no status file exists", async () => { + await writeFile( + path.join(outputDir, "zod-crawler.pid"), + JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + ids: 2, + }), + ); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + const code = await runCli(["status", "--output", outputDir]); + + expect(code).toBe(0); + expect( + consoleLog.mock.calls.some((call) => + String(call[0]).includes("status: running"), + ), + ).toBe(true); + consoleLog.mockRestore(); + }); + + it("reports the recorded exit code when a status file is present", async () => { + await writeFile( + path.join(outputDir, "zod-crawler.pid"), + JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + ids: 1, + }), + ); + await writeFile( + path.join(outputDir, "zod-crawler.status.json"), + JSON.stringify({ exitCode: 0, finishedAt: new Date().toISOString() }), + ); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + const code = await runCli(["status", "--output", outputDir]); + + expect(code).toBe(0); + expect( + consoleLog.mock.calls.some((call) => + String(call[0]).includes("succeeded"), + ), + ).toBe(true); + consoleLog.mockRestore(); + }); + + it("reports failure when the recorded exit code is non-zero", async () => { + await writeFile( + path.join(outputDir, "zod-crawler.pid"), + JSON.stringify({ + pid: process.pid, + startedAt: new Date().toISOString(), + ids: 1, + }), + ); + await writeFile( + path.join(outputDir, "zod-crawler.status.json"), + JSON.stringify({ exitCode: 1, finishedAt: new Date().toISOString() }), + ); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + const code = await runCli(["status", "--output", outputDir]); + + expect(code).toBe(1); + expect( + consoleLog.mock.calls.some((call) => + String(call[0]).includes("failed"), + ), + ).toBe(true); + consoleLog.mockRestore(); + }); + + it("reports not running when the pid is dead and there's no status file", async () => { + await writeFile( + path.join(outputDir, "zod-crawler.pid"), + JSON.stringify({ + pid: 999999, + startedAt: new Date().toISOString(), + ids: 1, + }), + ); + const killSpy = vi + .spyOn(process, "kill") + .mockImplementation((): never => { + const error = new Error("no such process") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + }); + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + const code = await runCli(["status", "--output", outputDir]); + + expect(code).toBe(1); + expect( + consoleLog.mock.calls.some((call) => + String(call[0]).includes("not running"), + ), + ).toBe(true); + + killSpy.mockRestore(); + consoleLog.mockRestore(); + }); + }); }); diff --git a/apps/cli/src/runCli.ts b/apps/cli/src/runCli.ts index a641e3c..ecc071b 100644 --- a/apps/cli/src/runCli.ts +++ b/apps/cli/src/runCli.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { openSync, closeSync } from "node:fs"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -25,7 +25,27 @@ import { createNodeSampleCache, runBullmqFetchQueue, } from "@zod-crawler/pipeline-node"; -import { parseCliArgs, usage, type CliSettings } from "./cliArgs.js"; +import { + parseCliArgs, + type CliSettings, + type StatusSettings, +} from "./cliArgs.js"; + +const PID_FILENAME = "zod-crawler.pid"; +const STATUS_FILENAME = "zod-crawler.status.json"; +const LOG_FILENAME = "zod-crawler.log"; +const DETACHED_ENV_VAR = "ZOD_CRAWLER_DETACHED"; + +interface PidFile { + pid: number; + startedAt: string; + ids: number; +} + +interface StatusFile { + exitCode: number; + finishedAt: string; +} // Prints fetch/cache status lines to stderr; the web app renders the same FetchProgressEvent stream as SSE messages instead. function logFetchProgress(event: FetchProgressEvent): void { @@ -45,15 +65,40 @@ export async function runCli(argv: string[]): Promise { const parsed = parseCliArgs(argv); if (!parsed.ok) { console.error(parsed.message); - console.error(usage); + console.error(parsed.usage); return 1; } + + if (parsed.command === "status") { + return runStatus(parsed.settings); + } + const { settings } = parsed; if (settings.detach) { return runDetached(settings); } + if (process.env[DETACHED_ENV_VAR] === "1") { + let exitCode = 1; + try { + exitCode = await runCrawl(settings); + } finally { + await writeFile( + path.join(settings.outputDir, STATUS_FILENAME), + JSON.stringify({ + exitCode, + finishedAt: new Date().toISOString(), + } satisfies StatusFile), + ); + } + return exitCode; + } + + return runCrawl(settings); +} + +async function runCrawl(settings: CliSettings): Promise { await mkdir(settings.outputDir, { recursive: true }); const cache = createNodeSampleCache(settings.outputDir); @@ -143,7 +188,7 @@ export async function runCli(argv: string[]): Promise { return exitCode; } -// Re-invokes this same CLI as a detached child with --detach stripped (settings has no way to express it, so it can't round-trip back in), then returns immediately without waiting for the crawl itself. There's no status subcommand yet, so the log file and pid printed here are the only way to check on it later. +// Re-invokes this same CLI as a detached child with --detach stripped (settings has no way to express it, so it can't round-trip back in), then returns immediately without waiting for the crawl itself. The "status" subcommand reads the pidfile written here and the status file the child writes on completion. async function runDetached(settings: CliSettings): Promise { await mkdir(settings.outputDir, { recursive: true }); @@ -170,7 +215,7 @@ async function runDetached(settings: CliSettings): Promise { childArgs.push("--concurrency", String(settings.concurrency)); } - const logPath = path.join(settings.outputDir, "zod-crawler.log"); + const logPath = path.join(settings.outputDir, LOG_FILENAME); const logFd = openSync(logPath, "a"); // Resolved from this module's own location, not process.argv[1]: works the same whether invoked via the installed bin, npx, or a direct node call. @@ -178,6 +223,7 @@ async function runDetached(settings: CliSettings): Promise { const child = spawn(process.execPath, [scriptPath, ...childArgs], { detached: true, stdio: ["ignore", logFd, logFd], + env: { ...process.env, [DETACHED_ENV_VAR]: "1" }, }); const spawned = await new Promise<{ ok: true } | { ok: false; error: Error }>( @@ -194,12 +240,85 @@ async function runDetached(settings: CliSettings): Promise { } child.unref(); + + await writeFile( + path.join(settings.outputDir, PID_FILENAME), + JSON.stringify({ + // spawn() sets pid synchronously. It's undefined only when spawning + // fails outright, already ruled out by spawned.ok above. + pid: child.pid!, + startedAt: new Date().toISOString(), + ids: settings.ids.length, + } satisfies PidFile), + ); + console.log(`Started crawl in the background (pid ${child.pid}).`); console.log(`Logs: ${logPath}`); console.log(`Output: ${settings.outputDir}`); return 0; } +async function readJsonFile(filePath: string): Promise { + try { + return JSON.parse(await readFile(filePath, "utf8")) as T; + } catch { + return undefined; + } +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function runStatus(settings: StatusSettings): Promise { + const pidFile = await readJsonFile( + path.join(settings.outputDir, PID_FILENAME), + ); + if (pidFile === undefined) { + console.log( + `No detached crawl found in ${settings.outputDir} (no ${PID_FILENAME}).`, + ); + return 1; + } + + const lines = [ + `Detached crawl in ${settings.outputDir}`, + ` pid: ${pidFile.pid}`, + ` ids: ${pidFile.ids}`, + ` started: ${pidFile.startedAt}`, + ]; + + const statusFile = await readJsonFile( + path.join(settings.outputDir, STATUS_FILENAME), + ); + + let exitCode: number; + if (statusFile !== undefined) { + lines.push(` finished: ${statusFile.finishedAt}`); + lines.push( + ` status: ${statusFile.exitCode === 0 ? "succeeded" : "failed"} (exit code ${statusFile.exitCode})`, + ); + exitCode = statusFile.exitCode; + } else if (isProcessAlive(pidFile.pid)) { + lines.push(" status: running"); + exitCode = 0; + } else { + lines.push( + " status: not running (no completion record - it may have crashed)", + ); + exitCode = 1; + } + + lines.push(` log: ${path.join(settings.outputDir, LOG_FILENAME)}`); + console.log(lines.join("\n")); + return exitCode; +} + // Writes only new candidates (not a seed id, not already cached), grouped by field path like the web app's export. async function writeCrawlCandidates( samples: Awaited>, diff --git a/docs/cli.md b/docs/cli.md index ce9878b..b4815c8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -82,6 +82,25 @@ export const TodoSchema = z.object({ The CLI also prints a validation summary (how many of the cached samples parsed successfully against the generated schema) and exits non-zero if any didn't. +## Checking on a detached crawl + +A crawl started with `--detach` writes a `/zod-crawler.pid` file when it starts and a `/zod-crawler.status.json` file when it finishes. Point `status` at the same `--output` directory to read them back: + +```bash +npx @zod-crawler/cli status --output ./out +``` + +``` +Detached crawl in ./out + pid: 12345 + ids: 3 + started: 2026-08-14T10:00:00.000Z + status: running + log: out/zod-crawler.log +``` + +Once the crawl finishes, `status` reports its recorded exit code instead (`succeeded`/`failed`) and exits with that same code, so it's scriptable (`zod-crawler status --output ./out && echo done`). If the process died without writing a status file, it reports `not running` and exits non-zero. If `--output` has no `zod-crawler.pid` at all, it reports that no detached crawl was found there. + ## How it works This is a thin wrapper around [`@zod-crawler/core`](core-usage.md). It fetches each id, caching raw responses to disk and resuming from cache on re-runs, merges the samples' shapes to infer a Zod schema, validates every cached sample against that schema in-process, and writes the result. diff --git a/docs/flags.md b/docs/flags.md index 667f7ff..3f9051a 100644 --- a/docs/flags.md +++ b/docs/flags.md @@ -16,4 +16,4 @@ Full flag reference for [`@zod-crawler/cli`](cli.md). | `--stop-on-error` | no (default off) | By default, a failed fetch (a non-2xx response, a network error, etc.) is logged and skipped, and the crawl continues with the rest of the ids. Set this to abort the whole run on the first failed fetch instead. | | `--redis-url` | no (default off, or `REDIS_URL` env var) | Switches fetching to a [BullMQ](https://bullmq.io)-backed queue against this instance instead of the plain sequential queue. [Valkey](https://valkey.io) recommended - see [Advanced queuing](advanced-queuing.md). | | `--concurrency` | no (default `1`) | How many fetches run at once. Only valid alongside `--redis-url`/`REDIS_URL`. | -| `--detach`, `-d` | no (default off) | Starts the crawl as a background process and returns immediately, printing its pid, output directory, and a log file path (`/zod-crawler.log`) capturing everything it would otherwise print to the terminal. There's no subcommand yet to check on a detached run's status - use the pid/log/output directory printed at start. | +| `--detach`, `-d` | no (default off) | Starts the crawl as a background process and returns immediately, printing its pid, output directory, and a log file path (`/zod-crawler.log`) capturing everything it would otherwise print to the terminal. Check on it later with `zod-crawler status --output ` - see [CLI](cli.md#checking-on-a-detached-crawl). | diff --git a/package-lock.json b/package-lock.json index ebceb36..6b1237b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "typescript": "^5", "vitest": "^4", "zod": "^4.4.3", - "zod-commands": "^1.1.2" + "zod-commands": "^1.1.3" } }, "apps/cli": { @@ -34,7 +34,7 @@ "@zod-crawler/pipeline-node": "^1.1.0", "prettier": "^3.8.3", "zod": "^4.4.3", - "zod-commands": "^1.1.2", + "zod-commands": "^1.1.3", "zod-transformers": "^1.0.0" }, "bin": { @@ -12489,9 +12489,9 @@ } }, "node_modules/zod-commands": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/zod-commands/-/zod-commands-1.1.2.tgz", - "integrity": "sha512-bQEQU3FEwhITFQKbcxev4fjz3IZtXLapo4U4wzCqtcF9UYTMva3d3UtMiiI+cptw77M+ASsHJPoSF5fPYlvsMw==", + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/zod-commands/-/zod-commands-1.1.3.tgz", + "integrity": "sha512-rwMq3EW6PKLwgMsRkUcQ89BTTRwOxEEFjXCLAXJ3YiQ8XcmN5zVj8EzfM5xkIoQnUD2V97Wx1D4vWAjpJhS5Nw==", "license": "MIT", "peerDependencies": { "zod": "^4.0.0" diff --git a/package.json b/package.json index 5d15558..4b08836 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,6 @@ "typescript": "^5", "vitest": "^4", "zod": "^4.4.3", - "zod-commands": "^1.1.2" + "zod-commands": "^1.1.3" } }