Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
- `@zod-crawler/components`: a new shared shadcn/Tailwind UI package holding the UI and crawl components moved out of `apps/web`, so `apps/web` and `apps/web-demo` render from the same components.
- `tools/cors-proxy`: a self-hosted CORS proxy at `zodcrawler.figulus.dev/proxy` that the demo falls back to when a target API doesn't allow direct cross-origin fetches, rate-limited per IP and capped daily.
- `tools/site-router`: routes `zodcrawler.figulus.dev`'s docs, the new demo, and everything else to the right place.
- `@zod-crawler/cli`: a `status` subcommand (`zod-crawler status --output <dir>`) reports on a crawl started with `--detach` - running, finished (with its recorded exit code), or crashed - by reading `<output>/zod-crawler.pid` and `<output>/zod-crawler.status.json`, both written by the detached run itself.

### Changed

- CI workflow (`ci.yml`) and `apps/cli`/`apps/web`'s Docker images now run on Node 22/24/26 instead of 20/22/24.
- `@zod-crawler/cli`: upgraded from `zod-cli-flags@1.0.2` to its renamed successor, `zod-commands@1.1.2`. `--ids`/`--ids-file` now use `exclusiveGroups` to declare "exactly one of" instead of a hand-written check in the parse transform, so `cli.usage` renders it as `(--ids <value> | --ids-file <value>)`.
- `@zod-crawler/cli`: upgraded from `zod-cli-flags@1.0.2` to its renamed successor, `zod-commands@1.1.3`. `--ids`/`--ids-file` now use `exclusiveGroups` to declare "exactly one of" instead of a hand-written check in the parse transform, so `cli.usage` renders it as `(--ids <value> | --ids-file <value>)`.

## [1.1.0] - 2026-08-12

Expand Down
2 changes: 1 addition & 1 deletion apps/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
"@zod-crawler/pipeline-node": "^1.1.0",
"prettier": "^3.8.3",
"zod": "^4.4.3",
"zod-commands": "^1.1.2",
"zod-commands": "^1.1.3",
"zod-transformers": "^1.0.0"
},
"peerDependencies": {
Expand Down
69 changes: 52 additions & 17 deletions apps/cli/src/cliArgs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids", "a, b ,a,c", "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a", "b", "c"]);
expect(result.settings.delayMs).toBe(7500);
expect(result.settings.schemaName).toBe("InferredSchema");
Expand All @@ -31,7 +31,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.emitCrawlCandidates).toBe(true);
expect(result.settings.crawlCandidatesFormat).toBe("txt");
}
Expand All @@ -50,7 +50,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.crawlCandidatesFormat).toBe("yaml");
}
});
Expand Down Expand Up @@ -96,7 +96,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.useZodTransformers).toBe(true);
}
});
Expand All @@ -111,7 +111,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.stopOnError).toBe(true);
}
});
Expand All @@ -120,7 +120,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.detach).toBe(false);
}
});
Expand All @@ -130,7 +130,8 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out", flag]);

expect(result.ok, `${flag} should parse`).toBe(true);
if (result.ok) expect(result.settings.detach).toBe(true);
if (result.ok && result.command === "crawl")
expect(result.settings.detach).toBe(true);
}
});

Expand All @@ -149,7 +150,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.urlTemplate).toBe("https://example.com/{id}.json");
expect(result.settings.delayMs).toBe(100);
expect(result.settings.schemaName).toBe("MySchema");
Expand Down Expand Up @@ -264,7 +265,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.redisUrl).toBeUndefined();
expect(result.settings.concurrency).toBeUndefined();
}
Expand All @@ -282,7 +283,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.redisUrl).toBe("redis://localhost:6379");
expect(result.settings.concurrency).toBe(1);
}
Expand All @@ -293,7 +294,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids", "a", "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.redisUrl).toBe("redis://env-host:6379");
}
});
Expand All @@ -310,7 +311,7 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.redisUrl).toBe("redis://flag-host:6379");
}
});
Expand All @@ -328,7 +329,8 @@ describe("parseCliArgs", () => {
]);

expect(result.ok).toBe(true);
if (result.ok) expect(result.settings.concurrency).toBe(4);
if (result.ok && result.command === "crawl")
expect(result.settings.concurrency).toBe(4);
});

it("rejects --concurrency without a Redis URL (flag or env)", () => {
Expand Down Expand Up @@ -383,7 +385,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a", "b", "c"]);
}
});
Expand All @@ -410,7 +412,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a", "b", "c"]);
}
});
Expand All @@ -425,7 +427,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a", "b", "c"]);
}
});
Expand All @@ -437,7 +439,7 @@ describe("parseCliArgs", () => {
const result = parseCliArgs(["--ids-file", file, "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok) {
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a", "c", "b"]);
}
});
Expand Down Expand Up @@ -465,4 +467,37 @@ describe("parseCliArgs", () => {
if (!result.ok) expect(result.message).toMatch(/--ids-file/);
});
});

it('accepts an explicit leading "crawl" token, same as omitting it', () => {
const result = parseCliArgs([
"crawl",
"--ids",
"a",
"--output",
"/tmp/out",
]);

expect(result.ok).toBe(true);
if (result.ok && result.command === "crawl") {
expect(result.settings.ids).toEqual(["a"]);
}
});

describe("status", () => {
it("parses --output into outputDir", () => {
const result = parseCliArgs(["status", "--output", "/tmp/out"]);

expect(result.ok).toBe(true);
if (result.ok && result.command === "status") {
expect(result.settings.outputDir).toBe("/tmp/out");
}
});

it("rejects a missing --output", () => {
const result = parseCliArgs(["status"]);

expect(result.ok).toBe(false);
if (!result.ok) expect(result.message).toMatch(/--output/);
});
});
});
Loading
Loading