diff --git a/.gitignore b/.gitignore index afab37e..56abad4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .saga.toml +.env node_modules/ dist/ .astro/ @@ -6,4 +7,4 @@ dist/ artifacts/ .claude/settings.local.json CLAUDE.local.json -AGENTS.local.json \ No newline at end of file +AGENTS.local.json diff --git a/README.md b/README.md index d8c5b0c..aea948e 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,41 @@ pnpm preview # serve the built ./dist locally pnpm check # astro check (type checking) ``` +## Writing sync + +Blog posts are authored outside the repository and synced into Astro's generated +content collection. Configure the local source folder in an uncommitted `.env` file: + +```sh +DBTLR_WRITING_SOURCE_DIR=/absolute/path/to/local/writing/folder +``` + +Source articles are Markdown files with frontmatter: + +```yaml +title: "Article title" +blurb: "Short description for listings and RSS" +lede: "Optional article-page intro" +date: 2026-06-20 +tag: "systems" +status: published +slug: optional-manual-slug +``` + +Only `status: published` files are synced; drafts, stubs, and missing statuses are +skipped. The sync command writes generated Markdown to `src/content/writing/` and +tracks stable source-to-slug mappings in `src/content/writing/.sync-manifest.json`. + +```sh +pnpm sync:writing -- --dry-run # preview create/update/stale/delete candidates +pnpm sync:writing # create/update published articles +pnpm sync:writing -- --prune # remove generated articles whose source is gone or unpublished +``` + +Slugs are derived from `slug` when present, otherwise from `title` on first publish. +After that, the manifest preserves the published URL unless an explicit slug change +is synced with `--rename-slugs`. + ## Workflow Work happens on feature branches and lands through pull requests: diff --git a/package.json b/package.json index 238390a..db2059b 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "dev": "astro dev", "build": "astro build", "check": "astro check", - "preview": "astro preview" + "preview": "astro preview", + "sync:writing": "node scripts/sync-writing.ts" }, "dependencies": { "@astrojs/rss": "^4.0.18", @@ -22,6 +23,9 @@ }, "devDependencies": { "@astrojs/check": "^0.9.9", + "@types/node": "^25.9.3", + "dotenv": "^17.4.2", + "gray-matter": "^4.0.3", "typescript": "^6.0.3", "wrangler": "^4.103.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f5742a..a534bad 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,10 +19,10 @@ importers: version: 5.2.8 '@tailwindcss/vite': specifier: ^4.3.1 - version: 4.3.1(vite@7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)) + version: 4.3.1(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)) astro: specifier: ^6.4.8 - version: 6.4.8(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.0)(yaml@2.9.0) + version: 6.4.8(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.0)(yaml@2.9.0) tailwindcss: specifier: ^4.3.1 version: 4.3.1 @@ -30,6 +30,15 @@ importers: '@astrojs/check': specifier: ^0.9.9 version: 0.9.9(prettier@3.8.4)(typescript@6.0.3) + '@types/node': + specifier: ^25.9.3 + version: 25.9.3 + dotenv: + specifier: ^17.4.2 + version: 17.4.2 + gray-matter: + specifier: ^4.0.3 + version: 4.0.3 typescript: specifier: ^6.0.3 version: 6.0.3 @@ -985,6 +994,9 @@ packages: '@types/nlcst@2.0.3': resolution: {integrity: sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA==} + '@types/node@25.9.3': + resolution: {integrity: sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg==} + '@types/unist@3.0.3': resolution: {integrity: sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==} @@ -1043,6 +1055,9 @@ packages: anynum@1.0.0: resolution: {integrity: sha512-xjR9/zBVnUOP6ztMIIgShjsxui80nQUQH+5xJnvrYLs+90bF25/KJqaAi8mk+B4RDtX1Nspi6fmp4YTEts8SfA==} + argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -1199,6 +1214,10 @@ packages: domutils@3.2.2: resolution: {integrity: sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==} + dotenv@17.4.2: + resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} + engines: {node: '>=12'} + dset@3.1.4: resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} engines: {node: '>=4'} @@ -1245,12 +1264,21 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + estree-walker@2.0.2: resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==} eventemitter3@5.0.4: resolution: {integrity: sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==} + extend-shallow@2.0.1: + resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} + engines: {node: '>=0.10.0'} + extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} @@ -1315,6 +1343,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + gray-matter@4.0.3: + resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} + engines: {node: '>=6.0'} + h3@1.15.11: resolution: {integrity: sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==} @@ -1370,6 +1402,10 @@ packages: engines: {node: '>=20'} hasBin: true + is-extendable@0.1.1: + resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} + engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} @@ -1391,6 +1427,10 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + js-yaml@3.14.2: + resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==} + hasBin: true + js-yaml@4.2.0: resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true @@ -1404,6 +1444,10 @@ packages: jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + kind-of@6.0.3: + resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} + engines: {node: '>=0.10.0'} + kleur@4.1.5: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} @@ -1833,6 +1877,10 @@ packages: resolution: {integrity: sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==} engines: {node: '>=11.0.0'} + section-matter@1.0.0: + resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} + engines: {node: '>=4'} + semver@7.8.3: resolution: {integrity: sha512-wnilbGyMxzbY7dNOl7jpKbLSjcfeweJWU5j4+u5qW+6/wuGD9KzIGOyZnQVSBM9E7DtWaaH3CyHkppYrKYoxwg==} engines: {node: '>=10'} @@ -1865,6 +1913,9 @@ packages: space-separated-tokens@2.0.2: resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -1876,6 +1927,10 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-bom-string@1.0.0: + resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} + engines: {node: '>=0.10.0'} + strnum@2.4.0: resolution: {integrity: sha512-sHrVyWWdq28RbhjuJdZsA1SnGRJV6NiXbk6AXBxDOsgAcA+lmpUZCYjOdLBxkXMwis6RRe7dlZt4VlIWFVzkmg==} @@ -1939,6 +1994,9 @@ packages: uncrypto@0.1.3: resolution: {integrity: sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@7.28.0: resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} @@ -2956,12 +3014,12 @@ snapshots: '@tailwindcss/oxide-win32-arm64-msvc': 4.3.1 '@tailwindcss/oxide-win32-x64-msvc': 4.3.1 - '@tailwindcss/vite@4.3.1(vite@7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0))': + '@tailwindcss/vite@4.3.1(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0))': dependencies: '@tailwindcss/node': 4.3.1 '@tailwindcss/oxide': 4.3.1 tailwindcss: 4.3.1 - vite: 7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) + vite: 7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) '@types/debug@4.1.13': dependencies: @@ -2983,6 +3041,10 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/node@25.9.3': + dependencies: + undici-types: 7.24.6 + '@types/unist@3.0.3': {} '@ungap/structured-clone@1.3.1': {} @@ -3061,13 +3123,17 @@ snapshots: anynum@1.0.0: {} + argparse@1.0.10: + dependencies: + sprintf-js: 1.0.3 + argparse@2.0.1: {} aria-query@5.3.2: {} array-iterate@2.0.1: {} - astro@6.4.8(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.0)(yaml@2.9.0): + astro@6.4.8(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(rollup@4.62.0)(yaml@2.9.0): dependencies: '@astrojs/compiler': 4.0.0 '@astrojs/internal-helpers': 0.10.0 @@ -3119,8 +3185,8 @@ snapshots: unist-util-visit: 5.1.0 unstorage: 1.17.5 vfile: 6.0.3 - vite: 7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) - vitefu: 1.1.3(vite@7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)) + vite: 7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) + vitefu: 1.1.3(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)) xxhash-wasm: 1.1.0 yargs-parser: 22.0.0 zod: 4.4.3 @@ -3280,6 +3346,8 @@ snapshots: domelementtype: 2.3.0 domhandler: 5.0.3 + dotenv@17.4.2: {} + dset@3.1.4: {} emmet@2.4.11: @@ -3364,10 +3432,16 @@ snapshots: escape-string-regexp@5.0.0: {} + esprima@4.0.1: {} + estree-walker@2.0.2: {} eventemitter3@5.0.4: {} + extend-shallow@2.0.1: + dependencies: + is-extendable: 0.1.1 + extend@3.0.2: {} fast-deep-equal@3.1.3: {} @@ -3424,6 +3498,13 @@ snapshots: graceful-fs@4.2.11: {} + gray-matter@4.0.3: + dependencies: + js-yaml: 3.14.2 + kind-of: 6.0.3 + section-matter: 1.0.0 + strip-bom-string: 1.0.0 + h3@1.15.11: dependencies: cookie-es: 1.2.3 @@ -3535,6 +3616,8 @@ snapshots: is-docker@4.0.0: {} + is-extendable@0.1.1: {} + is-fullwidth-code-point@3.0.0: {} is-inside-container@1.0.0: @@ -3549,6 +3632,11 @@ snapshots: jiti@2.7.0: {} + js-yaml@3.14.2: + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + js-yaml@4.2.0: dependencies: argparse: 2.0.1 @@ -3559,6 +3647,8 @@ snapshots: jsonc-parser@3.3.1: {} + kind-of@6.0.3: {} + kleur@4.1.5: {} lightningcss-android-arm64@1.32.0: @@ -4199,6 +4289,11 @@ snapshots: sax@1.6.0: {} + section-matter@1.0.0: + dependencies: + extend-shallow: 2.0.1 + kind-of: 6.0.3 + semver@7.8.3: {} semver@7.8.4: {} @@ -4253,6 +4348,8 @@ snapshots: space-separated-tokens@2.0.2: {} + sprintf-js@1.0.3: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -4268,6 +4365,8 @@ snapshots: dependencies: ansi-regex: 5.0.1 + strip-bom-string@1.0.0: {} + strnum@2.4.0: dependencies: anynum: 1.0.0 @@ -4320,6 +4419,8 @@ snapshots: uncrypto@0.1.3: {} + undici-types@7.24.6: {} + undici@7.28.0: {} unenv@2.0.0-rc.24: @@ -4410,7 +4511,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0): + vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0): dependencies: esbuild: 0.27.7 fdir: 6.5.0(picomatch@4.0.4) @@ -4419,14 +4520,15 @@ snapshots: rollup: 4.62.0 tinyglobby: 0.2.17 optionalDependencies: + '@types/node': 25.9.3 fsevents: 2.3.3 jiti: 2.7.0 lightningcss: 1.32.0 yaml: 2.9.0 - vitefu@1.1.3(vite@7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)): + vitefu@1.1.3(vite@7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)): optionalDependencies: - vite: 7.3.5(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) + vite: 7.3.5(@types/node@25.9.3)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) volar-service-css@0.0.70(@volar/language-service@2.4.28): dependencies: diff --git a/scripts/sync-writing.ts b/scripts/sync-writing.ts new file mode 100644 index 0000000..77ed65f --- /dev/null +++ b/scripts/sync-writing.ts @@ -0,0 +1,448 @@ +import 'dotenv/config'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, mkdir, readFile, readdir, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import os from 'node:os'; +import matter from 'gray-matter'; + +type CliOptions = { + dryRun: boolean; + prune: boolean; + renameSlugs: boolean; + sourceDir: string; + outputDir: string; + manifestPath: string; +}; + +type ManifestArticle = { + slug: string; + outputPath: string; + sourceHash: string; + title: string; + lastSyncedAt: string; +}; + +type Manifest = { + version: 1; + articles: Record; +}; + +type SourceArticle = { + sourcePath: string; + sourceKey: string; + slug: string; + outputPath: string; + sourceHash: string; + title: string; + body: string; +}; + +const defaultOutputDir = 'src/content/writing'; +const manifestFileName = '.sync-manifest.json'; +const sourceDirEnvVar = 'DBTLR_WRITING_SOURCE_DIR'; + +const options = parseArgs(process.argv.slice(2)); +const result = await syncWriting(options); + +for (const line of result) { + console.log(line); +} + +async function syncWriting(options: CliOptions): Promise { + const lines: string[] = []; + const sourceDir = resolvePath(options.sourceDir); + const outputDir = resolvePath(options.outputDir); + const manifestPath = resolvePath(options.manifestPath); + + if (!(await exists(sourceDir))) { + throw new Error(`source directory not found: ${sourceDir}`); + } + + const manifest = await readManifest(manifestPath); + const markdownFiles = await listMarkdownFiles(sourceDir); + const articles: SourceArticle[] = []; + const errors: string[] = []; + + for (const sourcePath of markdownFiles) { + const parsed = await parseSourceArticle(sourcePath, sourceDir, outputDir, manifest, options.renameSlugs); + if (parsed.status === 'skip') { + lines.push(`skip ${path.relative(sourceDir, sourcePath)} (${parsed.reason})`); + continue; + } + if (parsed.status === 'error') { + errors.push(`${path.relative(sourceDir, sourcePath)}: ${parsed.reason}`); + continue; + } + articles.push(parsed.article); + } + + validateUniqueSlugs(articles, errors); + + if (errors.length > 0) { + throw new Error(`writing sync failed:\n${errors.map((err) => ` - ${err}`).join('\n')}`); + } + + const seenSources = new Set(); + const nextManifest: Manifest = { version: 1, articles: { ...manifest.articles } }; + let changed = false; + + for (const article of articles) { + seenSources.add(article.sourceKey); + + const previous = manifest.articles[article.sourceKey]; + const outputChanged = + previous?.sourceHash !== article.sourceHash || + previous?.slug !== article.slug || + !(await exists(article.outputPath)); + + if (!outputChanged) { + lines.push(`unchanged ${article.slug}`); + continue; + } + + changed = true; + lines.push(`${previous ? 'update' : 'create'} ${path.relative(process.cwd(), article.outputPath)}`); + + if (!options.dryRun) { + await mkdir(path.dirname(article.outputPath), { recursive: true }); + await writeFile(article.outputPath, article.body, 'utf8'); + } + + if (previous && previous.outputPath !== path.relative(process.cwd(), article.outputPath)) { + lines.push(`delete ${previous.outputPath}`); + if (!options.dryRun) { + const previousOutputPath = resolvePath(previous.outputPath); + if (await exists(previousOutputPath)) { + await unlink(previousOutputPath); + } + } + } + + nextManifest.articles[article.sourceKey] = { + slug: article.slug, + outputPath: path.relative(process.cwd(), article.outputPath), + sourceHash: article.sourceHash, + title: article.title, + lastSyncedAt: new Date().toISOString(), + }; + } + + const staleEntries = Object.entries(manifest.articles).filter(([sourceKey]) => !seenSources.has(sourceKey)); + + for (const [sourceKey, entry] of staleEntries) { + const outputPath = resolvePath(entry.outputPath); + + if (!options.prune) { + lines.push(`stale ${entry.outputPath} (run with --prune to remove)`); + continue; + } + + delete nextManifest.articles[sourceKey]; + changed = true; + lines.push(`delete ${entry.outputPath}`); + + if (!options.dryRun && (await exists(outputPath))) { + await unlink(outputPath); + } + } + + if (changed || staleEntries.length > 0) { + if (options.dryRun) { + lines.push('dry run: no files changed'); + } else { + await mkdir(path.dirname(manifestPath), { recursive: true }); + await writeFile(manifestPath, `${JSON.stringify(nextManifest, null, 2)}\n`, 'utf8'); + } + } + + if (articles.length === 0) { + lines.push('no published articles found'); + } + + return lines; +} + +async function parseSourceArticle( + sourcePath: string, + sourceDir: string, + outputDir: string, + manifest: Manifest, + renameSlugs: boolean +): Promise< + | { status: 'article'; article: SourceArticle } + | { status: 'skip'; reason: string } + | { status: 'error'; reason: string } +> { + const raw = await readFile(sourcePath, 'utf8'); + const sourceKey = normalizePath(path.relative(sourceDir, sourcePath)); + const parsed = matter(raw); + const data = parsed.data as Record; + const status = asOptionalString(data.status); + + if (status !== 'published') { + return { status: 'skip', reason: status ? `status=${status}` : 'missing status=published' }; + } + + const title = asOptionalString(data.title); + const blurb = asOptionalString(data.blurb); + const lede = asOptionalString(data.lede); + const tag = asOptionalString(data.tag); + const date = formatDate(data.date); + const explicitSlug = asOptionalString(data.slug); + + const missing = [ + ['title', title], + ['blurb', blurb], + ['date', date], + ['tag', tag], + ] + .filter(([, value]) => !value) + .map(([field]) => field); + + if (missing.length > 0) { + return { status: 'error', reason: `missing required field(s): ${missing.join(', ')}` }; + } + + const previous = manifest.articles[sourceKey]; + const derivedSlug = slugify(explicitSlug ?? title!); + + if (!derivedSlug) { + return { status: 'error', reason: 'title/slug does not produce a valid slug' }; + } + + if (previous && explicitSlug && previous.slug !== derivedSlug && !renameSlugs) { + return { + status: 'error', + reason: `slug change from "${previous.slug}" to "${derivedSlug}" requires --rename-slugs`, + }; + } + + const slug = previous && !renameSlugs ? previous.slug : derivedSlug; + const publicData = { title: title!, blurb: blurb!, lede, date: date!, tag: tag! }; + const content = normalizeBody(parsed.content); + const body = buildMarkdown(publicData, content); + const sourceHash = sha256(JSON.stringify(publicData) + '\n' + content); + + return { + status: 'article', + article: { + sourcePath, + sourceKey, + slug, + outputPath: path.join(outputDir, `${slug}.md`), + sourceHash, + title: title!, + body, + }, + }; +} + +function validateUniqueSlugs(articles: SourceArticle[], errors: string[]): void { + const bySlug = new Map(); + + for (const article of articles) { + const existing = bySlug.get(article.slug); + if (existing) { + errors.push( + `slug collision "${article.slug}" between ${path.relative(process.cwd(), existing.sourcePath)} and ${path.relative( + process.cwd(), + article.sourcePath + )}` + ); + } + bySlug.set(article.slug, article); + } +} + +function buildMarkdown(data: { title: string; blurb: string; lede?: string; date: string; tag: string }, content: string): string { + const frontmatter = [ + '---', + `title: ${yamlString(data.title)}`, + `blurb: ${yamlString(data.blurb)}`, + data.lede ? `lede: ${yamlString(data.lede)}` : undefined, + `date: ${data.date}`, + `tag: ${yamlString(data.tag)}`, + '---', + ].filter(Boolean); + + return `${frontmatter.join('\n')}\n\n${content}\n`; +} + +function parseArgs(argv: string[]): CliOptions { + const options: CliOptions = { + dryRun: false, + prune: false, + renameSlugs: false, + sourceDir: process.env[sourceDirEnvVar] ?? '', + outputDir: defaultOutputDir, + manifestPath: path.join(defaultOutputDir, manifestFileName), + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + + if (arg === '--') { + continue; + } + if (arg === '--dry-run') { + options.dryRun = true; + continue; + } + if (arg === '--prune') { + options.prune = true; + continue; + } + if (arg === '--rename-slugs') { + options.renameSlugs = true; + continue; + } + if (arg === '--source') { + options.sourceDir = readValue(argv, ++i, arg); + continue; + } + if (arg === '--output') { + options.outputDir = readValue(argv, ++i, arg); + options.manifestPath = path.join(options.outputDir, manifestFileName); + continue; + } + if (arg === '--manifest') { + options.manifestPath = readValue(argv, ++i, arg); + continue; + } + if (arg === '--help' || arg === '-h') { + printHelp(); + process.exit(0); + } + + throw new Error(`unknown option: ${arg}`); + } + + if (!options.sourceDir) { + throw new Error(`${sourceDirEnvVar} must be set in .env or passed with --source`); + } + + return options; +} + +function readValue(argv: string[], index: number, option: string): string { + const value = argv[index]; + if (!value) { + throw new Error(`${option} requires a value`); + } + return value; +} + +function printHelp(): void { + console.log(`Usage: pnpm sync:writing [options] + +Sync published Markdown articles from a local source folder into Astro's writing collection. + +Options: + --dry-run show planned changes without writing files + --prune delete generated posts whose source is gone or unpublished + --rename-slugs allow explicit slug changes to rename generated files + --source source folder (default: ${sourceDirEnvVar} from .env) + --output output folder (default: ${defaultOutputDir}) + --manifest manifest path (default: ${defaultOutputDir}/${manifestFileName}) +`); +} + +async function readManifest(manifestPath: string): Promise { + if (!(await exists(manifestPath))) { + return { version: 1, articles: {} }; + } + + const raw = await readFile(manifestPath, 'utf8'); + const parsed = JSON.parse(raw) as Manifest; + + if (parsed.version !== 1 || typeof parsed.articles !== 'object' || parsed.articles === null) { + throw new Error(`unsupported writing sync manifest: ${manifestPath}`); + } + + return parsed; +} + +async function listMarkdownFiles(dir: string): Promise { + const entries = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + entries.map(async (entry) => { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + return listMarkdownFiles(fullPath); + } + if (entry.isFile() && entry.name.endsWith('.md')) { + return [fullPath]; + } + return []; + }) + ); + + return files.flat().sort((a, b) => a.localeCompare(b)); +} + +async function exists(filePath: string): Promise { + try { + await access(filePath, fsConstants.F_OK); + return true; + } catch { + return false; + } +} + +function resolvePath(filePath: string): string { + const expanded = filePath.startsWith('~/') ? path.join(os.homedir(), filePath.slice(2)) : filePath; + return path.resolve(expanded); +} + +function asOptionalString(value: unknown): string | undefined { + if (typeof value !== 'string') { + return undefined; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +function formatDate(value: unknown): string | undefined { + if (value instanceof Date && !Number.isNaN(value.valueOf())) { + return value.toISOString().slice(0, 10); + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) { + return trimmed; + } + const parsed = new Date(trimmed); + if (!Number.isNaN(parsed.valueOf())) { + return parsed.toISOString().slice(0, 10); + } + } + return undefined; +} + +function slugify(value: string): string { + return value + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .trim() + .replace(/['\u2019]/g, '') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + +function normalizeBody(content: string): string { + return content.replace(/\r\n/g, '\n').trim(); +} + +function normalizePath(filePath: string): string { + return filePath.split(path.sep).join('/'); +} + +function yamlString(value: string): string { + return JSON.stringify(value); +} + +function sha256(value: string): string { + return createHash('sha256').update(value).digest('hex'); +} diff --git a/src/pages/search.json.ts b/src/pages/search.json.ts index 991df31..fe69033 100644 --- a/src/pages/search.json.ts +++ b/src/pages/search.json.ts @@ -6,8 +6,15 @@ import { getCollection } from 'astro:content'; // by type transitions. export const GET: APIRoute = async () => { const projects = (await getCollection('projects')).sort((a, b) => a.data.order - b.data.order); + const posts = (await getCollection('writing')).sort((a, b) => b.data.date.valueOf() - a.data.date.valueOf()); const items = [ + ...posts.map((p) => ({ + type: 'post', + title: p.data.title, + sub: `${p.data.tag} ยท ${p.data.blurb}`, + href: `/writing/${p.id}/`, + })), ...projects.map((p) => ({ type: 'project', title: p.data.name,