From 421e529e7ecb3ed6d5c9f534e3408259bab80ff1 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 6 Aug 2026 12:59:36 -0500 Subject: [PATCH 1/2] Add Kotlin docs DB pipeline + Build Kotlin Docs GitHub Action (ADFA-4739) Adds the code that loads converted Kotlin website content (build_nav.py, populate_db.py, media insertion) and kotlin-stdlib/-reflect/-test JSON content (sync_kdoc_json_to_db.py) into documentation.db, the local e2e test script for that pipeline, and the CI workflow that runs it end-to-end against a Drive-hosted copy of the database. Split out of the larger Kotlin-docs pipeline PR (#21) so the DB-manipulation side (this ticket) can be reviewed separately from producing the raw JSON data for the Kotlin website (ADFA-5039, PR #23). This PR depends on ADFA-5039 merging first - populate_db.py, build_nav.py, and find_missing_assets.py all import md_to_json.py, which isn't included here. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build-kotlin-docs.yaml | 378 +++ .gitignore | 2 + CLAUDE.md | 192 ++ .../scripts/kotlin/build-stdlib-json-docs.sh | 95 + .../scripts/kotlin/build.gradle.kts | 9 +- .../ProcessKotlinWebsiteJSON/README.md | 133 + .../ProcessKotlinWebsiteJSON/assets/docs.css | 227 ++ .../assets/sidebar.js | 172 ++ .../ProcessKotlinWebsiteJSON/assets/tabs.js | 71 + .../ProcessKotlinWebsiteJSON/build_nav.py | 216 ++ .../find_missing_assets.py | 142 ++ .../insert_optimized_media.py | 457 ++++ .../optimize_media.py | 570 +++++ .../ProcessKotlinWebsiteJSON/populate_db.py | 616 +++++ .../templates/nav.html | 2133 +++++++++++++++++ .../templates/nav.peb | 50 + .../templates/page.peb | 138 ++ .../run_e2e_pipeline_test.sh | 220 ++ .../sync_kdoc_json_to_db.py | 204 ++ 19 files changed, 6024 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-kotlin-docs.yaml create mode 100644 CLAUDE.md create mode 100755 Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/docs.css create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/sidebar.js create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/tabs.js create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/templates/nav.html create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/templates/nav.peb create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/templates/page.peb create mode 100755 ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh create mode 100755 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml new file mode 100644 index 000000000..fa981350b --- /dev/null +++ b/.github/workflows/build-kotlin-docs.yaml @@ -0,0 +1,378 @@ +name: Build Kotlin Docs + +# CI counterpart of ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh - +# same five steps (find_missing_assets -> populate_db -> insert_optimized_media +# -> build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but sourcing its inputs from fresh git checkouts instead of a developer's +# local machine, and reading/writing the real database on Google Drive +# (GOOGLE_DRIVE_FILE_ID) instead of a local SOURCE_DB copy. +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow downloads it from +# Google Drive rather than generating it itself; someone has to run that IDE +# export, upload the zip to Drive, and supply its file ID (see +# images_zip_file_id / GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID below) before +# triggering a run that touches the website docs. Use skip_website_docs to +# bypass this entirely and only refresh the kotlin-stdlib/-reflect/-test +# JSON content. +# +# Required secrets (already configured - see docdb-regression-test.yaml for +# their other use in this repo): +# GCP_WIF_PROVIDER - Workload Identity Federation provider name +# GCP_WIF_SERVICE_ACCOUNT - Service account email for WIF (needs read +# access to the images-zip file below, and +# write access - not just view - to the +# database file, since this workflow +# overwrites it) +# GOOGLE_DRIVE_FILE_ID - File ID of the production documentation.db +# (stored on Drive as a zip) +# +# Optional secret (falls back to the images_zip_file_id input if unset; see +# also the hard-coded TEST_*_FILE_ID overrides below for one-off testing): +# GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID - File ID of Writerside's webHelpImages.zip export +# +# Optional secret (Slack notifications are skipped with a warning if unset): +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared Drive file). + +permissions: + contents: read + id-token: write + +# This workflow overwrites a single shared Drive file - never let two runs +# race to upload against each other. +concurrency: + group: build-kotlin-docs + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + images_zip_file_id: + description: >- + Google Drive file ID for Writerside's webHelpImages.zip export + matching kotlin_web_site_ref (see KNOWN LIMITATION above). Falls + back to the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret if left empty. + Ignored if skip_website_docs is true. + required: false + default: '' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT upload the result + back to Google Drive - the production database is left untouched. + Set to false only once you trust a given ref/URL combination (see + this workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }} + IMAGES_ZIP_FILE_ID_INPUT: ${{ inputs.images_zip_file_id }} + IMAGES_ZIP_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID }} + # --- Hard-coded overrides for one-off manual testing ----------------- + # Fill in either of these with a literal Google Drive file ID to + # bypass the secret/input resolution above for a quick, repeatable + # test run (e.g. against scratch copies of the database/images zip on + # Drive). Leave both empty ('') for normal operation. + TEST_DB_FILE_ID: '' + TEST_IMAGES_ZIP_FILE_ID: '' + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve Google Drive file IDs + run: | + DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" + IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}" + if [ -z "$DB_FILE_ID" ]; then + echo "Error: no database file ID resolved - set the GOOGLE_DRIVE_FILE_ID secret, or TEST_DB_FILE_ID above for a test run" >&2 + exit 1 + fi + echo "Resolved DB_FILE_ID: ${DB_FILE_ID:+(set)}" + echo "Resolved IMG_FILE_ID: ${IMG_FILE_ID:+(set)}" + echo "DB_FILE_ID=$DB_FILE_ID" >> "$GITHUB_ENV" + echo "IMG_FILE_ID=$IMG_FILE_ID" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y pngquant unzip zip sqlite3 + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py/scour/cairosvg: ProcessKotlinWebsiteJSON's own + # requirements (see its README), not in the root requirements.txt. + # google-api-python-client & friends: Drive download/upload, same + # libraries check-tools/download_database.py already depends on. + pip install markdown-it-py scour cairosvg \ + google-api-python-client google-auth-httplib2 google-auth-oauthlib + + - name: Authenticate to Google Cloud using Workload Identity Federation + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} + service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} + access_token_scopes: | + https://www.googleapis.com/auth/drive.file + + - name: Download current documentation.db from Google Drive + run: | + python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip + unzip -o documentation.zip + if [ ! -f documentation.db ]; then + found="$(find . -maxdepth 2 -name documentation.db | head -n1)" + [ -n "$found" ] && mv "$found" documentation.db + fi + test -f documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + rm -f documentation.zip + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Download Writerside image export from Google Drive + if: ${{ !inputs.skip_website_docs }} + run: | + if [ -z "$IMG_FILE_ID" ]; then + echo "Error: no images-zip file ID resolved - set images_zip_file_id, the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret, or TEST_IMAGES_ZIP_FILE_ID above (see KNOWN LIMITATION in this workflow's header comment). Required unless skip_website_docs is true." >&2 + exit 1 + fi + # download_database.py is a generic Drive-file-by-ID downloader + # despite its name - reused here rather than duplicating the + # WIF/Drive-API download logic for a second file type. + python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Upload missing-assets report + if: ${{ !inputs.skip_website_docs }} + uses: actions/upload-artifact@v4 + with: + name: missing-assets-report + path: missing-assets-report.md + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # Same three blacklist entries as run_e2e_pipeline_test.sh + # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree + # if its nav structure has changed since this was written. + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row, which this + # database doesn't ship with by default (idempotent). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + id: stdlib_docs + run: | + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Upload built database as workflow artifact + uses: actions/upload-artifact@v4 + with: + name: documentation-db-${{ github.run_number }} + path: documentation.db + retention-days: 14 + + - name: Zip updated database for upload + if: ${{ !inputs.dry_run }} + run: zip -j documentation.zip documentation.db + + - name: Upload updated database to Google Drive + if: ${{ !inputs.dry_run }} + run: | + python3 - <<'PYEOF' + import os + from google.auth import default + from googleapiclient.discovery import build + from googleapiclient.http import MediaFileUpload + + file_id = os.environ["DB_FILE_ID"] + credentials, _ = default() + service = build("drive", "v3", credentials=credentials) + media = MediaFileUpload("documentation.zip", mimetype="application/zip", resumable=True) + updated = service.files().update( + fileId=file_id, media_body=media, fields="id, modifiedTime, md5Checksum" + ).execute() + print(f"Uploaded new revision of {file_id}: {updated}") + PYEOF + + - name: 'Notify Slack: build complete' + if: ${{ !inputs.dry_run }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.gitignore b/.gitignore index 59cc3acb6..42fa9ce9c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ __pycache__/ *$py.class *.db *.sqlite +run_e2e_pipeline_test.local.sh +grep_content_blobs.local.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..93b9b24bb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,192 @@ +# CLAUDE.md + +Guidance for Claude (and anyone else) working in this repository. + +## What this repository is + +App Dev For All builds **Code on the Go**, an Android IDE aimed at users with no or limited +internet access +(code: [appdevforall/CodeOnTheGo](https://github.com/appdevforall/CodeOnTheGo)). To support that, +Java/Kotlin/Android API documentation is bundled into the app as a single SQLite file — the +**documentation database** — rather than fetched from the web. + +The documentation database serves two distinct features in the IDE: + +1. **Tooltips (Tier 1/2).** When a user selects a keyword/symbol in the code editor, a dialog + shows short (Tier 1) and detailed (Tier 2) tooltip text if the selection matches an entry in + the DB. This lookup happens elsewhere in the CodeOnTheGo Android code (not in this repo, and + not in `WebServer.kt` — see below). +2. **Content pages (Tier 3).** From a tooltip, the user can click through to a full documentation + page. Those pages (and other static content — HTML, images, PDFs) are served over HTTP by + **`WebServer.kt`** + ([CodeOnTheGo/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt](https://github.com/appdevforall/CodeOnTheGo/blob/stage/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt)), + which runs inside the app and reads directly from the `Content` (and, as of recently, + `Templates`/`Bookshelf`/`BookCategories`) tables of the same database. + +**This repository (`OfflineDocumentationTools`) is the collection of offline tools that build and +edit that database** — it contains no part of the production Android app itself. + +> **Alex's standing caveat, worth repeating at the top of every session:** nothing in this +> repository is guaranteed to work against the *current* production database. The schema has moved +> forward (in the app / by hand) faster than the tooling in this repo has been updated. See +> "Schema: current vs. what this repo expects" below — that gap is the most important thing to +> understand before making changes here. + +## Schema: current vs. what this repo expects + +The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, +as of 2026-08-05: + +```sql +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL); +CREATE TABLE TooltipButtonNumbers (id INTEGER UNIQUE); -- manually assigned display order +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE('path') +); +CREATE TABLE Tooltips ( + id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL, + summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag), + FOREIGN KEY(categoryId) REFERENCES TooltipCategories(id) +); +CREATE TABLE TooltipButtons ( + tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT, + FOREIGN KEY(tooltipId) REFERENCES Tooltips(id), FOREIGN KEY(buttonNumberId) REFERENCES TooltipButtonNumbers(id) +); +CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE('name')); +CREATE TABLE BookCategories (id INTEGER PRIMARY KEY AUTOINCREMENT, category STRING, description STRING DEFAULT '', UNIQUE('category')); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT '', + bookCategoryID INTEGER, FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), UNIQUE(title, bookCategoryId)); +-- Triggers keep Bookshelf in sync when a .pdf row is added to/removed from Content. +CREATE TABLE PUCC_Students (...), PUCC_Classes (...), PUCC_Sections (...), PUCC_Professors (...), + PUCC_StudentAssignments (...), PUCC_ProfessorAssignments (...) +-- Unrelated to documentation tooling (confirmed by Alex) — ignore, leave as-is, do not +-- document or maintain further in this repo. +``` + +**`Templates`, `BookCategories`, and `Bookshelf` are not documentation cruft — `WebServer.kt` +actively depends on them.** Its `/pr/bs` endpoint builds a JSON "bookshelf" payload straight from +`Content` + `Bookshelf` + `BookCategories`, looks up a template named `'bookshelf'` in `Templates`, +and renders it with the Pebble template engine. More generally, any `Content` row with a non-zero +`templateId` gets its stored (decompressed) content run through the matching row in `Templates` as +a Pebble template before being served. This is a real, current feature of the shipped server, not +a placeholder. + +**Nothing that currently builds or writes to the database in this repository knows about any of +that — and that's expected.** `Templates`/`Bookshelf`/`BookCategories` are populated by a separate +plugin system, not by anything in this repo: App Dev For All supports plugins that write into the +documentation database, including the bookshelf feature specifically — +[appdevforall/bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin). So the absence +of any `Templates`/`Bookshelf`/`BookCategories` handling here is not a gap to fill; it's out of +scope for this repo. (A repo-wide search for `templateId`, `Templates`, `Bookshelf`, +`BookCategories`, or `PUCC` turns up zero matches outside `WebServer.kt` itself, which is +consistent with that division of responsibility.) Concretely, relative to the schema above: + +| Piece | What it thinks the schema is | Consequence | +| --- | --- | --- | +| `scripts/DocumentationDatabase.py` (used by `scripts/ingest.py`, and hence by `.github/workflows/publish-doc-db.yaml`) | `Content` / `Languages` / `ContentTypes` only, plus an optional `ide_tooltip_table`. Its constructor explicitly **raises `ValueError`** if it opens a DB containing any table outside that whitelist. | **This will refuse to open the current production `documentation.db` at all** — it will list `Tooltips`, `TooltipCategories`, `TooltipButtons`, `TooltipButtonNumbers`, `LastChange`, `Templates`, `BookCategories`, `Bookshelf`, and every `PUCC_*` table as "unexpected." This is the single biggest blocker to reusing this script as-is. | +| `docdb-studio/SCHEMA.md` / `AGENTS.md` (states the schema is "locked," no migrations) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, `LastChange` (with a *different* shape: `documentationSet`/`changeTime`/`who` — this part does match current), plus a legacy `ide_tooltip_table`. Missing `templateId`, `Templates`, `BookCategories`, `Bookshelf`, `PUCC_*`. | Closest of the three documented schemas to reality, but still out of date. `docdb_studio.py`'s own "never change the schema" policy is itself now stale, since the live schema has already changed underneath it. | +| `check-tools/README.md`'s embedded schema (and by extension the mental model behind `check-tools/db_health_checker.py`) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, and a *third* variant of `LastChange` (`now`/`who`). No `Templates`/`Bookshelf`/`BookCategories`/`PUCC_*`. | The health checker's required-table check still passes (it only checks that its known tables exist, not that no others do). Since `Templates`/`Bookshelf`/`BookCategories` are out of scope for this repo (see above), this is not being treated as something to fix right now. | + +There also appear to be **two unrelated tooltip storage formats** in this repo's history, and it's +worth being deliberate about which one is current: + +- The **normalized** format (`Tooltips` + `TooltipCategories` + `TooltipButtons` + + `TooltipButtonNumbers`) — this is what's in the live schema above, what `docdb-studio` edits, + what `check-tools/db_health_checker.py` validates, and what `scripts/TooltipManager.py` + dumps/rebuilds via CSV. +- A **legacy flat** format, a single `ide_tooltip_table(tooltipCategory, tooltipTag, + tooltipSummary, tooltipDetail, tooltipButtons)` table (button data packed as a JSON string in + one column) — written by `scripts/tooltips.py` (`TooltipDatabase`, driven by + `scripts/import_tooltips.py` from `SourceDocs/Tooltips/tooltips.xlsx`) and by + `scripts/load_android_data.py` (fed by pickle files that `scripts/android_tooltips.py` / + `scripts/java_tooltips.py` scrape from Android/Java HTML doc trees). **`ide_tooltip_table` does + not exist in the current production schema at all.** + +**`ide_tooltip_table` is officially dead (confirmed by Alex).** That means the entire chain that +targets it — `scripts/tooltips.py`, `scripts/import_tooltips.py`, `scripts/android_tooltips.py`, +`scripts/java_tooltips.py`, `scripts/android_html_page.py`, and `scripts/load_android_data.py` — is +**deprecated legacy code**. It's left in the repo for reference/history, but none of it should be +extended or relied on, and none of it writes to a table the shipped app or `docdb-studio` actually +uses. Any future Android/Java tooltip work should target the normalized `Tooltips` / +`TooltipCategories` / `TooltipButtons` / `TooltipButtonNumbers` tables instead (the same ones +`docdb-studio` and `scripts/TooltipManager.py` already use for Kotlin tooltips). + +## Repository tour + +- **`docdb-studio/`** — a Flet (Flutter-for-Python) desktop GUI for browsing/editing `Tooltips` / + `TooltipCategories` / `TooltipButtons` and importing `Content`. Has its own `CLAUDE.md`, + `AGENTS.md`, `SCHEMA.md`, and a real pytest suite. Actively maintained (most recent commits in + the repo touch this tool), but per the table above, its documented schema is behind the live one. + That's an accepted state, not an active problem: schema evolution happens outside + `docdb-studio` (and outside this repo, e.g. via plugins — see below), and `docdb-studio` is + expected to catch up after the fact rather than lead. Its `AGENTS.md`/`SCHEMA.md` "never migrate + the schema" language should be read as "don't migrate it from in here," not as a claim that the + schema never changes. +- **`check-tools/`** — `db_health_checker.py` (schema/integrity/referential checks against the + *old* normalized schema) plus `download_database.py`, a working Google Drive downloader + authenticated via GCP Workload Identity Federation (no long-lived keys). Wired into + `.github/workflows/docdb-regression-test.yaml`, which runs it daily against the production DB on + Drive. +- **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content + ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by + `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), + `create_empty_database.py`, `list_database_documents.py`. **Deprecated/dead** (target the + removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, + `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, + `load_android_data.py`. +- **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates + `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli + decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and + knows nothing about compression-aware content types, templates, or fragmentation. **This is not + what ships in the app** — treat it as historical/reference only, not as documentation of current + server behavior. `WebServer.kt` is the real thing. +- **`Dokka-plugin-kdoc2json/`** — on `main`, this is just a `README.md` describing the intended + design plus a flowchart image; there is no code here yet. The actual implementation (the Dokka + `JsonRenderer`/`ModelMapper`/`LinkPostProcessor` plugin, its test suite, and the + `kotlin-stdlib-docs` build scripts) exists only on the unmerged branch **`fix/ADFA-4514`**. That + branch's diff against `main` also shows it removing recent `docdb-studio` work and all of + `scripts/pdfjs/` — almost certainly because the branch was cut before those were added and hasn't + been rebased, not because it intends to delete them. **Flagged: rebase `fix/ADFA-4514` onto + current `main` before merging**, to avoid actually deleting that work. +- **`ProcessDocs/`** — HTML-processing pipelines that predate the "build docs as JSON" goal: + `ProcessKotlinDocs/` (turns Kotlin's HTML doc export into a self-contained HTML set + table of + contents, used by `.github/workflows/automate-kotlin.yaml`), `ProcessAndroidDevSite/`, `AndroidDocs/` + (holds `android-tooltips.pkl`, the pickle consumed by the now-deprecated `load_android_data.py`), + `ProcessPDFs/`. +- **`SourceDocs/`** — raw inputs: `KotlinDocs/html`, `JavaDocs/html` + `java_keywords.html`, + `Tooltips/tooltips.xlsx`, `KotlinDocs/kotlin-spec.pdf`. +- **`DocumentationAnalysis/`, `DocAnalysis/`, `png_optimization/`, `androidxtooltips/`** — Jupyter + notebooks and one-off scripts for doc-set size analysis, image/PNG compression experiments, and a + one-time AndroidX tooltip import (ADFA-1419). Not part of the critical build path. +- **`.github/workflows/`** — three workflows: `automate-kotlin.yaml` (tag-triggered, builds the + Kotlin HTML doc bundle as a GitHub release asset), `publish-doc-db.yaml` (tag-triggered, runs the + `scripts/ingest.py` pipeline and releases the resulting `.sqlite`), `docdb-regression-test.yaml` + (daily cron, downloads the production DB from Google Drive via WIF and runs + `check-tools/main.py` against it). None of these have any Slack integration yet. + +## Decisions log + +Settled with Alex on 2026-08-05, folded into the sections above; recorded here so the reasoning +isn't lost: + +- `ide_tooltip_table` and everything that targets it are dead. Treat as deprecated, not as a gap. +- `Templates`/`Bookshelf`/`BookCategories` are populated by App Dev For All's plugin system + (e.g. [bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin)), not by this repo. + Not a gap to fill here. +- `PUCC_*` tables are unrelated to documentation tooling. Ignore; leave as-is. +- `fix/ADFA-4514` needs a rebase onto `main` before merge — noted above. +- `docdb-studio`'s schema is expected to lag the live schema and catch up after the fact; that's + fine, no urgent update needed. +- `check-tools/db_health_checker.py` is not being extended with `Templates`/`Bookshelf` checks + right now — deliberately out of scope for the moment. + +The one piece of this document that still describes an *active* problem rather than a settled +scope boundary is `scripts/DocumentationDatabase.py`'s hard failure on unrecognized tables (see the +table above) — that will need to be addressed before `scripts/ingest.py` / +`publish-doc-db.yaml` can run against a current-schema database. diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh new file mode 100755 index 000000000..8d160825d --- /dev/null +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Builds the kotlin-stdlib/kotlin-test/kotlin-reflect API docs as JSON via the +# kdoc-to-json Dokka plugin, against a full kotlin/ (https://github.com/JetBrains/kotlin) +# repo checkout - freshly compiling and publishing the plugin from source +# first, so every run picks up whatever's currently in +# Dokka-plugin-kdoc2json/kdoc-to-json/src, not a jar left over from an +# earlier run. +# +# Only generates the JSON output (dokkaGenerateModuleJson), not the default +# HTML - JSON/latest/all-libs is the only thing this project's pipeline +# (sync_kdoc_json_to_db.py) consumes. Use build-kotlin-stdlib.sh directly, +# against libraries/tools/kotlin-stdlib-docs, if you also want the HTML +# comparison output that test_kotlin_stdlib.sh checks against. +# +# The target kotlin-stdlib-docs project's build.gradle.kts is swapped out +# for this directory's own (JSON-plugin-enabled) copy for the duration of +# the build, then restored automatically on exit - the kotlin checkout is +# left exactly as it was found, whether the build succeeds or fails. +# +# Only the final output path is written to stdout; every other message goes +# to stderr, so this composes as: +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh )" +set -euo pipefail + +log() { echo "$@" >&2; } + +if [ $# -lt 1 ]; then + log "Usage: $0 [output-dir]" + exit 1 +fi + +KOTLIN_ROOT="$(cd "$1" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/../../kdoc-to-json" && pwd)" +STDLIB_DOCS_DIR="$KOTLIN_ROOT/libraries/tools/kotlin-stdlib-docs" +OUTPUT_ROOT="$(mkdir -p "${2:-$SCRIPT_DIR/build-output}" && cd "${2:-$SCRIPT_DIR/build-output}" && pwd)" +JSON_OUTPUT_DIR="$OUTPUT_ROOT/json" + +if [ ! -f "$KOTLIN_ROOT/gradle.properties" ]; then + log "error: '$KOTLIN_ROOT' doesn't look like a kotlin repo checkout (missing gradle.properties)." + exit 1 +fi +if [ ! -f "$STDLIB_DOCS_DIR/settings.gradle.kts" ] || [ ! -x "$STDLIB_DOCS_DIR/gradlew" ]; then + log "error: '$STDLIB_DOCS_DIR' doesn't look like a kotlin-stdlib-docs project (missing settings.gradle.kts or gradlew)." + exit 1 +fi +if [ ! -x "$PLUGIN_DIR/gradlew" ]; then + log "error: kdoc-to-json plugin project not found at '$PLUGIN_DIR' (missing gradlew)." + exit 1 +fi + +# The JSON-plugin-enabled build.gradle.kts we're about to install reads +# dokka_version as a plain Gradle project property (-Pdokka_version=...) +# rather than through this repo's own version catalog, so it has to be +# supplied explicitly - pulled from the same catalog entry the rest of the +# kotlin repo's Dokka usage is pinned to, so it never drifts out of sync. +DOKKA_VERSION="$(grep -m1 '^dokka[[:space:]]*=' "$KOTLIN_ROOT/gradle/libs.versions.toml" | sed -E 's/^dokka[[:space:]]*=[[:space:]]*"([^"]*)".*/\1/')" +if [ -z "$DOKKA_VERSION" ]; then + log "error: couldn't find a 'dokka = \"...\"' entry in $KOTLIN_ROOT/gradle/libs.versions.toml" + exit 1 +fi + +log "==> [1/2] Building and publishing a fresh copy of the kdoc-to-json plugin..." +# Sent to stderr (fd 2), not left on stdout - a caller doing +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh ...)" must only capture the +# final path this script echoes, not gradlew's own build console output. +( cd "$PLUGIN_DIR" && ./gradlew clean publishToMavenLocal ) >&2 + +log "==> Installing kdoc-to-json-enabled build.gradle.kts into $STDLIB_DOCS_DIR" +ORIGINAL_BUILD_GRADLE="$(mktemp)" +cp "$STDLIB_DOCS_DIR/build.gradle.kts" "$ORIGINAL_BUILD_GRADLE" +restore_build_gradle() { + cp "$ORIGINAL_BUILD_GRADLE" "$STDLIB_DOCS_DIR/build.gradle.kts" + rm -f "$ORIGINAL_BUILD_GRADLE" +} +trap restore_build_gradle EXIT +cp "$SCRIPT_DIR/build.gradle.kts" "$STDLIB_DOCS_DIR/build.gradle.kts" + +log "==> [2/2] Generating JSON documentation via kdoc-to-json (dokka $DOKKA_VERSION)..." +# --refresh-dependencies forces Gradle to re-resolve the just-published +# SNAPSHOT jar from mavenLocal() rather than serving a same-GAV copy it +# cached from an earlier run of this same script. +( cd "$STDLIB_DOCS_DIR" && ./gradlew dokkaGenerateModuleJson \ + "-PdocsBuildDir=$JSON_OUTPUT_DIR" \ + "-Pdokka_version=$DOKKA_VERSION" \ + --refresh-dependencies ) >&2 + +ALL_LIBS_DIR="$JSON_OUTPUT_DIR/latest/all-libs" +if [ ! -d "$ALL_LIBS_DIR" ]; then + log "error: expected output at '$ALL_LIBS_DIR' but it wasn't created." + exit 1 +fi + +log "==> Done." +echo "$ALL_LIBS_DIR" diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts index db4c72983..48f68c3f2 100644 --- a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts @@ -46,7 +46,14 @@ allprojects { // 3. Maven Central (Keep this for standard standard stable libraries like Gson/Coroutines) mavenCentral() - // ALL REMOTE JETBRAINS SNAPSHOT SERVERS HAVE BEEN REMOVED! + // 4. Dokka's own dev-snapshot server - required by plugins:dokka-samples-transformer-plugin + // and plugins:dokka-version-filter-plugin (both included by kotlin-stdlib-docs' + // settings.gradle.kts and pulled onto the build graph by its dokka-convention plugin), + // which pin to a Dokka dev build rather than a Maven Central release. Same property + + // default kotlin-stdlib-docs' own settings.gradle.kts uses, so this only ever points + // wherever that project already expects it to. + maven(url = providers.gradleProperty("dokka_repository") + .getOrElse("https://redirector.kotlinlang.org/maven/dokka-dev")) } // --- ADDED THIS EXCLUSION BLOCK --- diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..fe6ffbcee --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,133 @@ +# Process Kotlin Website JSON + +Scripts for loading converted Kotlin website JSON, its navigation tree, and +its media straight into a `documentation.db`-schema SQLite database. + +The JSON conversion itself (`md_to_json.py`) is a separate ticket/PR +(ADFA-5039); `build_nav.py` and `populate_db.py` below import it directly, +so it needs to already be merged (or otherwise present in this directory) +for anything here to run. + +## Scripts + +| Script | Purpose | +|---|---| +| `md_to_json.py` (ADFA-5039, not in this PR) | Converts every `topics/**/*.md` page into one JSON file. Writes `theme.json` and copies `images/` into the output directory. See that ticket's README for the page JSON schema. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | + +## Requirements + +- Python 3.10+ +- `pip install markdown-it-py Pillow scour brotli` +- `cairosvg` (only needed if an optimized SVG exceeds `--svg-rasterize-threshold`): `pip install cairosvg` +- `pngquant` on `PATH` (e.g. `apt install pngquant`) — required by `optimize_media.py`/`insert_optimized_media.py`, and by `populate_db.py` for the images it inserts directly from the Writerside export. + +`populate_db.py` also expects, relative to its own location, and already +included in this directory: + +- `templates/page.peb`, `templates/nav.peb` — Pebble templates upserted into the `Templates` table. +- `assets/docs.css`, `assets/tabs.js`, `assets/sidebar.js` — static assets inserted at `assets/`. + +## Inputs you need before starting + +- A checkout of `kotlin-web-site/docs` (the `` argument below) — contains `topics/`, `images/`, `v.list`, and `kr.tree`. +- A config JSON with theming colors, e.g.: + ```json + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} + ``` +- Writerside's own image export zip (e.g. `webHelpImages.zip`, found next to `kr.tree`) if you're using `populate_db.py`. + +## Workflow: generate JSON + nav for a static/templated preview + +Use this to produce standalone JSON pages and nav data (not the database) +for local inspection or a different renderer. Step 1 is `md_to_json.py` +(ADFA-5039) — see that ticket for its own usage and the page JSON schema it +produces. + +```bash +# 1. Convert every topic .md into JSON, one file per page (ADFA-5039) +python3 md_to_json.py config.json + +# 2. Build the sidebar nav from kr.tree against that JSON output +python3 build_nav.py + +# 3. (optional) Check for broken links/images/includes in the source tree +python3 find_missing_assets.py missing-assets-report.md +``` + +`` ends up containing everything `md_to_json.py` writes, plus: +- `nav.json` / `nav.html` — sidebar tree and a pre-rendered static copy + +## Workflow: generate + insert directly into the documentation database + +This is the path that actually populates `documentation.db`. It performs +the same conversion as `md_to_json.py`/`build_nav.py` internally — you don't +run those scripts first. + +```bash +python3 populate_db.py config.json [db-path] +``` + +- `db-path` defaults to `documentation.db` in the current directory, and must already exist with the expected schema (`Languages`, `ContentTypes`, `Templates` tables populated). +- A timestamped backup (`.backup-`) is written before any changes, via SQLite's `VACUUM INTO`. +- Everything under `k/html/` and `assets/` is deleted and re-inserted in a single transaction (rolled back on error), then the database is `VACUUM`ed. + +### Pruning documentation you don't want (ADFA-4737) + +To leave a whole `kr.tree` subtree out of the database entirely — nav +entry, converted pages, and all — pass `--blacklisted-element-titles` with +the full `toc-title` path from a top-level element down to the one you want +to drop. Levels are joined with `\/` (backslash-slash), not a bare `/`, +since a bare `/` commonly appears inside a real title. The example below is +illustrative only — open `/kr.tree` and copy the actual +`toc-title` chain for whatever section you're dropping (e.g. Kotlin/Wasm): + +```bash +python3 populate_db.py config.json documentation.db \ + --blacklisted-element-titles \ + "\/" +``` + +Any other page's in-content link to a pruned topic renders as a styled +"broken" link (via `broken-ext-link-color`) rather than a dead link with no +indication anything changed. Run with `--blacklisted-element-titles` first +against a scratch copy of the database and check the warnings on stderr for +any path that didn't match — that usually means the toc-title or ancestor +chain was copied wrong. + +## Workflow: optimizing and inserting media + +Two options, depending on whether the database already has pages loaded: + +**Standalone optimization only** (no database involved): + +```bash +python3 optimize_media.py [--max-width 500] [--webp] [...] +``` + +**Optimize and update an existing database's images in place:** + +```bash +python3 insert_optimized_media.py [work-dir] [options] +``` + +This re-runs `optimize_media.py`'s pipeline, backs up the database first, +replaces each `k/html/images/` row with the optimized bytes, rewrites +any page/nav reference to a file that got renamed during optimization (e.g. +`--webp` conversion or SVG rasterization), and deletes any image no page +references anymore. Both scripts share the same tuning flags +(`--max-width`, `--jpeg-quality`, `--webp`, `--webp-quality`, +`--pngquant-speed`, `--svg-precision`, `--svg-rasterize-threshold`, +`--verbose`, `--log-file`), settable via `--config ` instead of the +command line — see either script's module docstring for the full option +reference. + +## Recommended order for a full refresh + +1. `find_missing_assets.py` against the new `` — fix anything broken in the source before converting it. +2. `populate_db.py`, with `--blacklisted-element-titles` for anything you don't want documented (e.g. Kotlin/Wasm per ADFA-4737). +3. `insert_optimized_media.py` against the raw media directory, if you want optimized (resized/compressed) images rather than Writerside's own export as-is. diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/docs.css b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/docs.css new file mode 100644 index 000000000..b8eef9286 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/docs.css @@ -0,0 +1,227 @@ +html, body { + background: #ffffff; + font-family: Arial, Helvetica, sans-serif; +} + +.docs-layout { + display: flex; + align-items: flex-start; +} + +.docs-sidebar { + width: 280px; + flex-shrink: 0; + box-sizing: border-box; + overflow-y: auto; + max-height: 100vh; + position: sticky; + top: 0; +} + +.docs-content { + flex: 1; + min-width: 0; + padding: 0 24px; +} + +/* Source images/videos carry explicit width="..." attributes (from + Writerside's `{width="800"}` sizing hints, see md_to_json.py), which is an + intrinsic pixel size the img/video would otherwise render at even when + that's wider than the viewport. max-width: 100% lets it shrink to fit + instead of overflowing on narrow screens, while height: auto keeps its + aspect ratio as it scales down. */ +.docs-content img, +.docs-content video, +.docs-content iframe { + max-width: 100%; + height: auto; +} + +/* Long code lines can't shrink like an image can without breaking the code's + formatting, so let the block itself scroll horizontally instead - without + this the unconstrained width pushes out past the viewport and the whole + page grows a horizontal scrollbar on narrow screens. */ +.docs-content pre.code-block { + overflow-x: auto; + box-sizing: border-box; + max-width: 100%; +} + +/* Sidebar tree:
    /
  • are only used for their document structure here, + not as a bulleted list, so strip the browser's default marker/indent/ + margin on every level (root .nav-tree and each nested .nav-subtree share + the class) and build indentation and disclosure icons ourselves below. */ +.docs-nav { + font-size: 14px; + line-height: 1.4; + color: #333333; +} + +.docs-nav ul { + list-style: none; + margin: 0; + padding: 0; +} + +.docs-nav .nav-subtree { + padding-left: 14px; +} + +/* Per-topic collapse/expand (nav.peb: every node with children renders a + sibling .nav-toggle-group button next to its label/link, leaf nodes render + a same-sized .nav-spacer instead so labels still line up in a column). + Collapsed is the CSS default so the tree stays usable without JS; + assets/sidebar.js toggles .nav-expanded on click and auto-expands the + current page's ancestors. */ +.nav-row { + display: flex; + align-items: center; + gap: 4px; + padding: 4px 8px; + border-radius: 4px; + cursor: default; +} + +.nav-row:hover { + background: #f0f1f2; +} + +.nav-item > .nav-subtree { + display: none; +} + +.nav-item.nav-expanded > .nav-subtree { + display: block; +} + +.nav-toggle-group, +.nav-spacer { + width: 14px; + height: 14px; + flex-shrink: 0; +} + +.nav-toggle-group { + display: flex; + align-items: center; + justify-content: center; + background: none; + border: none; + padding: 0; + cursor: pointer; + color: #666666; +} + +.nav-toggle-group::before { + content: "\25B8"; /* ▸ */ + display: inline-block; + font-size: 10px; + transition: transform 0.15s ease; +} + +.nav-item.nav-expanded > .nav-row > .nav-toggle-group::before { + transform: rotate(90deg); +} + +.nav-link, +.nav-group-title { + flex: 1; + min-width: 0; + padding: 2px 0; + color: inherit; + text-decoration: none; +} + +.nav-link:hover { + text-decoration: underline; +} + +.nav-group-title { + cursor: default; +} + +.nav-link--active { + font-weight: 700; + color: #0b57d0; +} + +.nav-item--current > .nav-row { + background: #e8f0fe; + border-left: 3px solid #0b57d0; + padding-left: 5px; /* 8px base padding - 3px border, so text doesn't shift */ +} + +/* Mobile off-canvas drawer (assets/sidebar.js: hamburger button, backdrop, + and edge-swipe open/close all toggle the classes below). Above the + breakpoint the sidebar stays a normal static column and these are inert. */ +.nav-toggle { + display: none; +} + +.nav-backdrop { + display: none; +} + +@media (max-width: 900px) { + .docs-layout { + display: block; + } + + .nav-toggle { + display: block; + position: fixed; + top: 10px; + left: 10px; + z-index: 1001; + width: 40px; + height: 40px; + border: 1px solid #ccc; + border-radius: 4px; + background: #ffffff; + } + + .nav-toggle::before { + content: "\2630"; /* ☰ */ + font-size: 1.3em; + } + + .docs-sidebar { + position: fixed; + top: 0; + left: 0; + height: 100%; + max-height: 100%; + width: 85vw; + max-width: 320px; + background: #ffffff; + z-index: 1000; + box-shadow: 2px 0 12px rgba(0, 0, 0, 0.25); + transform: translateX(-100%); + transition: transform 0.25s ease; + padding-top: 56px; + } + + .docs-sidebar.sidebar-open { + transform: translateX(0); + } + + .nav-backdrop { + display: block; + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.4); + opacity: 0; + pointer-events: none; + transition: opacity 0.25s ease; + z-index: 999; + } + + .nav-backdrop--visible { + opacity: 1; + pointer-events: auto; + } + + .docs-content { + padding-top: 56px; + } +} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/sidebar.js b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/sidebar.js new file mode 100644 index 000000000..68b480c61 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/sidebar.js @@ -0,0 +1,172 @@ +// Drives the sidebar templates/nav.peb renders: the mobile off-canvas drawer +// (hamburger button, backdrop, edge swipe-to-open / swipe-to-close) and the +// per-topic collapse/expand toggles (.nav-toggle-group, one per node with +// children). Also auto-expands and highlights whichever nav-link matches the +// current page on load, so a reader landing deep in the tree doesn't see an +// entirely collapsed sidebar with no indication of where they are. +(function () { + var sidebar = document.getElementById("sidebar"); + if (!sidebar) return; + + var menuButton = document.getElementById("nav-toggle"); + var backdrop = document.getElementById("nav-backdrop"); + var MOBILE_QUERY = "(max-width: 900px)"; + + function isMobile() { + return window.matchMedia && window.matchMedia(MOBILE_QUERY).matches; + } + + function openSidebar() { + sidebar.classList.add("sidebar-open"); + if (backdrop) backdrop.classList.add("nav-backdrop--visible"); + if (menuButton) menuButton.setAttribute("aria-expanded", "true"); + } + + function closeSidebar() { + sidebar.classList.remove("sidebar-open"); + if (backdrop) backdrop.classList.remove("nav-backdrop--visible"); + if (menuButton) menuButton.setAttribute("aria-expanded", "false"); + } + + function toggleSidebar() { + if (sidebar.classList.contains("sidebar-open")) { + closeSidebar(); + } else { + openSidebar(); + } + } + + if (menuButton) menuButton.addEventListener("click", toggleSidebar); + if (backdrop) backdrop.addEventListener("click", closeSidebar); + + function toggleItem(item) { + var toggle = item.querySelector(":scope > .nav-row > .nav-toggle-group"); + var expanded = item.classList.toggle("nav-expanded"); + if (toggle) toggle.setAttribute("aria-expanded", expanded ? "true" : "false"); + } + + // Per-topic collapse/expand: every node with children renders a sibling + // .nav-toggle-group button next to its label/link (see nav.peb) so this + // never has to guess which
  • a click landed in beyond closest(). A + // group header with no page of its own (.nav-group-title, not a link) also + // toggles on click, since clicking it can't navigate anywhere anyway. + sidebar.addEventListener("click", function (event) { + var toggle = event.target.closest(".nav-toggle-group"); + if (toggle) { + toggleItem(toggle.closest(".nav-item")); + return; + } + var groupTitle = event.target.closest(".nav-group-title"); + if (groupTitle) { + toggleItem(groupTitle.closest(".nav-item")); + return; + } + // Navigating on mobile should close the drawer instead of leaving it + // covering the page the reader just opened. + if (event.target.closest(".nav-link") && isMobile()) { + closeSidebar(); + } + }); + + function expandAncestors(item) { + while (item) { + item.classList.add("nav-expanded"); + var toggle = item.querySelector(":scope > .nav-row > .nav-toggle-group"); + if (toggle) toggle.setAttribute("aria-expanded", "true"); + var parentList = item.parentElement; + item = parentList ? parentList.closest(".nav-item") : null; + } + } + + // Scrolls only the sidebar's own internal scrollbox to reveal el, without + // touching window scroll - the sidebar is position:fixed on mobile (and + // translated off-canvas until opened), so a plain el.scrollIntoView() + // would have the browser scroll the whole page trying to "reveal" an + // element that fixed positioning makes it unable to actually bring into + // view that way. + function scrollWithinSidebar(el) { + var elRect = el.getBoundingClientRect(); + var boxRect = sidebar.getBoundingClientRect(); + sidebar.scrollTop += elRect.top - boxRect.top - sidebar.clientHeight / 2; + } + + function highlightCurrentPage() { + var path = window.location.pathname.replace(/^\//, "").replace(/\.html$/, ""); + var current = sidebar.querySelector('.nav-link[data-nav-id="' + path + '"]'); + if (!current) return; + current.classList.add("nav-link--active"); + current.setAttribute("aria-current", "page"); + current.closest(".nav-item").classList.add("nav-item--current"); + expandAncestors(current.closest(".nav-item")); + scrollWithinSidebar(current); + } + + // The static-site pipeline (RenderDocs.java) has page.peb include a + // pre-rendered nav.html at request/build time, so the sidebar already has + // its content by the time this script runs. The database-backed server + // has no such file-based template include (see layout.pebble - templates + // there are fully self-contained), so its page template instead renders + // empty and expects the + // client to fetch and inject the nav markup itself. Handle both the same + // way: run the content-dependent init once the markup actually exists. + var navSrc = sidebar.getAttribute("data-nav-src"); + if (navSrc) { + fetch(navSrc) + .then(function (response) { + return response.text(); + }) + .then(function (html) { + sidebar.insertAdjacentHTML("beforeend", html); + highlightCurrentPage(); + }) + .catch(function (err) { + console.error("Failed to load navigation from " + navSrc + ":", err); + }); + } else { + highlightCurrentPage(); + } + + // Edge swipe-right opens the drawer, swipe-left (anywhere) closes it. + var touchStartX = null; + var touchStartY = null; + var EDGE_ZONE = 24; + var SWIPE_THRESHOLD = 60; + + document.addEventListener( + "touchstart", + function (event) { + var t = event.touches[0]; + touchStartX = t.clientX; + touchStartY = t.clientY; + }, + { passive: true } + ); + + document.addEventListener( + "touchend", + function (event) { + if (touchStartX === null || !isMobile()) { + touchStartX = null; + touchStartY = null; + return; + } + var startX = touchStartX; + var startY = touchStartY; + touchStartX = null; + touchStartY = null; + + var t = event.changedTouches[0]; + var dx = t.clientX - startX; + var dy = t.clientY - startY; + if (Math.abs(dx) < SWIPE_THRESHOLD || Math.abs(dx) < Math.abs(dy)) return; + + var isOpen = sidebar.classList.contains("sidebar-open"); + if (!isOpen && dx > 0 && startX <= EDGE_ZONE) { + openSidebar(); + } else if (isOpen && dx < 0) { + closeSidebar(); + } + }, + { passive: true } + ); +})(); diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/tabs.js b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/tabs.js new file mode 100644 index 000000000..aa6916a53 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/assets/tabs.js @@ -0,0 +1,71 @@ +// Makes the
    ...
    blocks page.peb renders (for +// md_to_json.py's "tabs" block type) switchable, including linking every +// tabs block that shares the same data-group on a page - e.g. picking +// "Groovy" in one Kotlin/Groovy/Maven build-script tabs block switches every +// other build-script tabs block on the page - and remembering the choice +// across pages via localStorage, mirroring Writerside's data-sync-tabs. +(function () { + var STORAGE_PREFIX = "docs-tab-group:"; + + function activate(container, groupKey) { + var buttons = container.querySelectorAll(".tab-button"); + var panels = container.querySelectorAll(".tab-panel"); + var hasMatch = false; + for (var i = 0; i < buttons.length; i++) { + if (buttons[i].dataset.groupKey === groupKey) { + hasMatch = true; + break; + } + } + if (!hasMatch) return; + + buttons.forEach(function (btn) { + var active = btn.dataset.groupKey === groupKey; + btn.classList.toggle("tab-button--active", active); + btn.setAttribute("aria-selected", active ? "true" : "false"); + }); + panels.forEach(function (panel) { + var active = panel.dataset.groupKey === groupKey; + panel.classList.toggle("tab-panel--active", active); + panel.hidden = !active; + }); + } + + function switchGroup(group, groupKey) { + document.querySelectorAll('.tabs[data-group="' + group + '"]').forEach(function (container) { + activate(container, groupKey); + }); + try { + localStorage.setItem(STORAGE_PREFIX + group, groupKey); + } catch (e) { + // localStorage unavailable (private browsing, etc.) - selection just won't persist. + } + } + + document.addEventListener("click", function (event) { + var button = event.target.closest(".tab-button"); + if (!button) return; + var container = button.closest(".tabs"); + if (!container) return; + var groupKey = button.dataset.groupKey; + var group = container.dataset.group; + if (group) { + switchGroup(group, groupKey); + } else { + activate(container, groupKey); + } + }); + + document.addEventListener("DOMContentLoaded", function () { + document.querySelectorAll(".tabs[data-group]").forEach(function (container) { + var group = container.dataset.group; + var saved = null; + try { + saved = localStorage.getItem(STORAGE_PREFIX + group); + } catch (e) { + // ignore + } + if (saved) activate(container, saved); + }); + }); +})(); diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py new file mode 100644 index 000000000..ce3084d75 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/build_nav.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Builds sidebar navigation data/HTML from a JetBrains Writerside .tree file +(e.g. kotlin-web-site/docs/kr.tree), for use with templates/nav.peb. + +Usage: + python3 build_nav.py [--tree-file kr.tree] + + is the checkout of kotlin-web-site/docs (contains kr.tree). + is the output of md_to_json.py, used to resolve each + to the page's "id" (its path + relative to docs-root, e.g. "topics/ksp/ksp-overview") and, + when a has no toc-title, its rendered "title". + receives nav.json (the tree, for feeding into nav.peb) and + nav.html (a pre-rendered static copy of what nav.peb outputs). + +Writerside lets a both link to its own page (topic="...") and +contain nested children (sub-pages) at once, and topic +references are bare filenames resolved by uniqueness across the whole +topics/ subtree, not by path - this script relies on that same uniqueness. + +A topic="foo.md" that doesn't resolve to any converted page is treated as +manually deleted (rather than e.g. a typo) and dropped from the nav: if the +toc-element has no children either, build_node() returns None for it and it's +filtered out of its parent's children entirely; if it still has children +(sub-topics that do still exist), the element is kept as a link-less category +header for them instead of losing those children too. This only applies to +topic="*.md" references - kr.tree itself is never modified, so a purged +topic's (and any of its still-live children) stays in the tree +forever; this is what re-derives "not actually present" from the docs-json +output on every run instead. + +If /theme.json (written by md_to_json.py from its own config +argument) has a "menu-no-link-color", each node that doesn't actually lead to +a generated page - a pure category header with no topic="...", a +topic="foo.topic" that isn't a converted .md page (e.g. home.topic, +api-references.topic), or a topic="foo.md" whose page was deleted but which +still has live children - gets a "noLinkColor" of that value baked directly +into its entry in nav.json (null otherwise), which nav.peb turns into an +inline style. A toc-element using kr.tree's other, unrelated href="https://..." +attribute (e.g. "Test library (kotlin.test)" under API reference) is not +treated any differently here - it has no topic="...", so it's already a +no-link category header like any other; this script does not read that +external href at all, so no link is added for it. +""" +import argparse +import json +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +HUMANIZE_RE = re.compile(r"[-_]+") + + +def humanize(stem: str) -> str: + return HUMANIZE_RE.sub(" ", stem).strip().title() + + +def load_page_index(docs_json_dir: Path) -> tuple[dict, dict]: + """Returns (stem -> id, id -> title) built from every generated page JSON.""" + stem_to_id = {} + id_to_title = {} + for json_path in docs_json_dir.rglob("*.json"): + page = json.loads(json_path.read_text(encoding="utf-8")) + page_id = page.get("id") + if not page_id: + continue + stem = Path(page_id).name + stem_to_id[stem] = page_id + id_to_title[page_id] = page.get("title") + return stem_to_id, id_to_title + + +def build_node(el: ET.Element, stem_to_id: dict, id_to_title: dict, warnings: list, no_link_color: str, + id_prefix: str = "") -> dict | None: + """Returns None when this element's own topic="*.md" no longer resolves + to a converted page (deleted) and it has no children left worth keeping - + callers must filter these out of whatever list they collect build_node() + results into (both build_node() itself, for its own children, and the + top-level toc-element loop in main()).""" + topic = el.get("topic") + toc_title = el.get("toc-title") + hidden = el.get("hidden") == "true" + + children = [ + build_node(c, stem_to_id, id_to_title, warnings, no_link_color, id_prefix) + for c in el.findall("toc-element") + ] + children = [c for c in children if c is not None] + + page_id = None + title = toc_title + no_link = True + deleted = False + if topic: + stem = Path(topic).stem + page_id = stem_to_id.get(stem) + if page_id is None: + if topic.endswith(".md"): + # Was a real page at some point, isn't anymore - the .md was + # manually deleted. Drop this node; if children still resolve + # to real pages, keep it around as a no-link category header + # for them rather than losing those children too. + deleted = True + if children: + warnings.append(f"topic={topic!r} has no converted page (deleted); " + f"keeping as a header for its {len(children)} remaining child page(s)") + else: + warnings.append(f"topic={topic!r} has no converted page (deleted); dropping from nav") + return None + else: + # Not a converted .md page (e.g. home.topic, api-references.topic): + # still rendered as a link (for consistency, and it's still the + # best URL guess) but it doesn't lead anywhere real, so it's + # colored the same as a no-topic-at-all category header. Prefixed + # the same way as a resolved stem_to_id hit would be (e.g. + # populate_db.py passes id_prefix="k/html/" since stem_to_id + # there already maps every real page to "k/html/"), so + # this fallback id follows the same URL convention as everything + # else on the page rather than silently reverting to a bare one. + page_id = f"{id_prefix}{stem}" + warnings.append(f"no converted page for topic={topic!r}; using id={page_id!r}") + else: + no_link = False + if not title: + title = (id_to_title.get(page_id) if page_id else None) or humanize(stem) + elif not title: + title = "Untitled" + + return { + "title": title, + "id": None if deleted else page_id, + "hidden": hidden, + "noLinkColor": no_link_color if (no_link or deleted) else None, + "children": children, + } + + +def render_node(node: dict, indent: int = 0) -> str: + # Kept byte-identical to templates/nav.peb's renderNavNode macro, since + # RenderDocs.java re-renders nav.peb over nav.json for the live site and + # this function only produces a standalone preview copy of the same HTML. + classes = "nav-item" + (" nav-hidden" if node["hidden"] else "") + has_children = bool(node["children"]) + style = f' style="color: {node["noLinkColor"]};"' if node["noLinkColor"] else "" + if node["id"]: + label = f'{node["title"]}' + else: + label = f'{node["title"]}' + toggle = ( + f'' + if has_children else '' + ) + parts = [f'
  • ', '
    ', toggle, label, "
    "] + if has_children: + parts.append('
      ') + for child in node["children"]: + parts.append(render_node(child)) + parts.append("
    ") + parts.append("
    • ") + return "".join(parts) + + +def render_html(tree: list) -> str: + body = "".join(render_node(node) for node in tree) + return ( + '" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("docs_json_dir", type=Path, help="Path to md_to_json.py output directory") + parser.add_argument("output_dir", type=Path, help="Directory to write nav.json and nav.html into") + parser.add_argument("--tree-file", default="kr.tree", help="Filename of the Writerside tree file under docs_root") + args = parser.parse_args() + + tree_path = args.docs_root / args.tree_file + if not tree_path.is_file(): + print(f"error: {tree_path} does not exist", file=sys.stderr) + sys.exit(1) + + stem_to_id, id_to_title = load_page_index(args.docs_json_dir) + + theme_path = args.docs_json_dir / "theme.json" + no_link_color = None + if theme_path.is_file(): + no_link_color = json.loads(theme_path.read_text(encoding="utf-8")).get("menu-no-link-color") + else: + print(f"warning: {theme_path} not found (run md_to_json.py first); " + f"no-link menu items won't be colored", file=sys.stderr) + + root = ET.parse(tree_path).getroot() + warnings = [] + nav_tree = [build_node(el, stem_to_id, id_to_title, warnings, no_link_color) for el in root.findall("toc-element")] + nav_tree = [node for node in nav_tree if node is not None] + + for w in warnings: + print(f"warning: {w}", file=sys.stderr) + + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "nav.json").write_text( + json.dumps(nav_tree, separators=(",", ":"), ensure_ascii=False), encoding="utf-8" + ) + (args.output_dir / "nav.html").write_text(render_html(nav_tree), encoding="utf-8") + + print(f"Wrote nav.json and nav.html ({len(warnings)} warning(s)) into {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py new file mode 100644 index 000000000..653982113 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +Documents every reference in the docs source (kotlin-web-site/docs) that +points at an asset - another topic page, an image, or an target - +which doesn't actually exist, so broken source content can be found and +fixed without having to read every generated page. + +Usage: + python3 find_missing_assets.py [report-path] [--topics-subdir topics] [--images-subdir images] + +Reuses md_to_json.py's own link/image resolution (build_topic_index, +build_image_index, Converter) instead of re-parsing links with regexes, so +this reports exactly what would end up broken in the rendered site - e.g. a +"foo.md" written inside a fenced code sample (showing readers what Writerside +markup looks like) is correctly ignored, since it's never tokenized as a +real link in the first place. + + targets are a separate check: those +aren't links/images so md_to_json.py's Converter never resolves them, but a +missing include is still a broken asset reference worth surfacing. This is +checked with a small standalone regex scan (fenced code blocks stripped +first) against every filename that exists anywhere under /, +regardless of extension (include targets can be ".md" or ".topic"). +""" +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path + +from md_to_json import Converter, build_image_index, build_topic_index, load_variables, make_markdown_it + +INCLUDE_RE = re.compile(r']*\bfrom\s*=\s*"([^"]+)"') +FENCE_RE = re.compile(r"```.*?```", re.S) + + +def find_include_warnings(topics_dir: Path) -> list: + all_filenames = {p.name for p in topics_dir.rglob("*") if p.is_file()} + warnings = [] + for md_path in sorted(topics_dir.rglob("*.md")): + text_no_fences = FENCE_RE.sub("", md_path.read_text(encoding="utf-8")) + source_rel = str(md_path.relative_to(topics_dir.parent)) + for target in INCLUDE_RE.findall(text_no_fences): + if target not in all_filenames: + warnings.append({"kind": "include", "source": source_rel, "reference": target}) + return warnings + + +def group_by_reference(warnings: list) -> dict: + grouped = defaultdict(set) + for w in warnings: + grouped[w["reference"]].add(w["source"]) + return {ref: sorted(sources) for ref, sources in sorted(grouped.items())} + + +def render_section(title: str, grouped: dict) -> str: + lines = [f"## {title} ({len(grouped)})", ""] + if not grouped: + lines.append("None.") + for ref, sources in grouped.items(): + lines.append(f"- `{ref}`") + for src in sources: + lines.append(f" - {src}") + lines.append("") + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("report_path", type=Path, nargs="?", default=Path("missing-assets-report.md"), + help="Where to write the Markdown report (default: ./missing-assets-report.md)") + parser.add_argument("--topics-subdir", default="topics", help="Subdirectory of docs_root holding .md files") + parser.add_argument("--images-subdir", default="images", help="Subdirectory of docs_root holding image files") + args = parser.parse_args() + + docs_root: Path = args.docs_root + topics_dir = docs_root / args.topics_subdir + images_dir = docs_root / args.images_subdir + if not topics_dir.is_dir(): + print(f"error: {topics_dir} is not a directory", file=sys.stderr) + sys.exit(1) + + variables = load_variables(docs_root) + topic_index = build_topic_index(topics_dir) + image_index, image_collisions = build_image_index(images_dir) + md = make_markdown_it() + converter = Converter(md, variables, topic_index, image_index) + + md_files = sorted(topics_dir.rglob("*.md")) + for md_path in md_files: + rel = md_path.relative_to(topics_dir) + page_id = str(rel.with_suffix("")) + source_rel = str(Path(args.topics_subdir) / rel) + try: + converter.convert_file(md_path, page_id, source_rel) + except Exception as exc: # noqa: BLE001 - surface which file broke, keep auditing the rest + print(f"error scanning {md_path}: {exc}", file=sys.stderr) + + link_warnings = [w for w in converter.warnings if w["kind"] == "link"] + image_warnings = [w for w in converter.warnings if w["kind"] == "image"] + include_warnings = find_include_warnings(topics_dir) + + links = group_by_reference(link_warnings) + images = group_by_reference(image_warnings) + includes = group_by_reference(include_warnings) + collisions = {name: rels for name, rels in image_collisions} + + report = [ + "# Missing Assets Report", + "", + f"Scanned {len(md_files)} Markdown files under `{topics_dir}`.", + "", + "## Summary", + "", + f"- {len(links)} unresolved cross-page link target(s)", + f"- {len(images)} missing image(s)", + f"- {len(collisions)} ambiguous image filename(s) (exist in more than one place under images/)", + f"- {len(includes)} unresolved `` target(s)", + "", + render_section("Unresolved cross-page links", links), + render_section("Missing images", images), + render_section("Unresolved targets", includes), + f"## Ambiguous image filenames ({len(collisions)})", + "", + ] + if not collisions: + report.append("None.") + for name, rels in collisions.items(): + report.append(f"- `{name}`") + report.append(f" - used: images/{rels[0]}") + for rel in rels[1:]: + report.append(f" - ignored: images/{rel}") + report.append("") + + args.report_path.write_text("\n".join(report), encoding="utf-8") + print(f"Wrote {args.report_path} " + f"({len(links)} links, {len(images)} images, {len(collisions)} ambiguous, {len(includes)} includes)") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py new file mode 100644 index 000000000..3b08078ee --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +""" +insert_optimized_media.py + +Runs optimize_media.py's image optimizer over a directory of raw media, +then updates an existing documentation.db-schema database (as +populate_db.py produces) with the optimized results, fixing up every page +that referenced a file under its old name. + +What this does, inside a single transaction (rolled back on any error): + 1. Backs up first, same as populate_db.py (VACUUM INTO a + timestamped sibling file). + 2. Optimizes every file under into a staging directory + (--work-dir, or a temporary one removed afterwards) via + optimize_media.py's own pipeline - see its own module docstring for + what "optimized" means (resize, pngquant, Scour, optional WEBP + conversion / SVG rasterization). Aborts before touching the database + if any file fails to optimize. + 3. Replaces every "k/html/images/" Content row with the optimized + bytes, deleting the old row (and any leftover chunked fragments) first + - Content.path is UNIQUE, so a stale row has to go before its + replacement can be inserted. Images are addressed by bare filename + only, matching populate_db.py's own flat "k/html/images/*" convention: + subdirectories are flattened to their basename, and a + basename collision across two different subdirectories is a warning + (keeping the first, sorted, skipping the rest), not an error. + 4. Wherever optimization renamed a file (webp conversion, or an oversized + SVG rasterized to PNG/WEBP), rewrites every "/k/html/images/" + reference still pointing at the old name, in every k/html/*.html page + and the nav row, to the new name - so a page doesn't end up linking to + a filename that no longer exists. + 5. Deletes every remaining "k/html/images/" row (base row and any + chunked fragments) that, after the rename rewriting above, no + k/html/*.html page or the nav row references even once - not just ones + touched by this run's rename_map, but every currently-stored image, + so media that fell out of use in an earlier run (e.g. a topic's .md + was deleted, or an reference was removed by hand) gets cleaned + up too, not just this run's renames. + 6. VACUUMs the database afterwards (outside the transaction - SQLite + refuses to VACUUM inside one), same as populate_db.py. + +Usage: + python3 insert_optimized_media.py [work_dir] [options] + python3 insert_optimized_media.py --config myjob.config + + are optimize_media.py's own tuning flags (--max-width, +--jpeg-quality, --webp, --webp-quality, --pngquant-speed, --svg-precision, +--svg-rasterize-threshold, --verbose, --log-file, --config) - see +optimize_media.py's own docstring for what each one does. media-dir/db-path/ +work-dir can also be set via --config (as "input-dir"/"db-path"/ +"output-dir"), the same as optimize_media.py's own options. + +Note: --webp requires this database's ContentTypes table to already have an +"image/webp" row (checked up front, before any optimization work starts) - +this project's documentation.db doesn't ship with one. +""" +import argparse +import re +import shutil +import sqlite3 +import sys +import tempfile +from pathlib import Path + +import brotli + +from optimize_media import ( + BUILTIN_DEFAULTS, Logger, OPTION_SPECS, add_optimize_arguments, find_pngquant, optimize_directory, + resolve_config, +) +from populate_db import ( + CHUNK_SIZE, EXTENSION_TO_CONTENT_TYPE, IMAGES_DB_PATH_PREFIX, IMAGES_URL_PREFIX, LANGUAGE, PAGE_CONTENT_TYPE, + backup_database, get_content_type, get_id, insert_chunked_content, +) + +WEBP_CONTENT_TYPE = "image/webp" +# populate_db.py's own EXTENSION_TO_CONTENT_TYPE has no ".webp" entry - its +# image source (a Writerside export) never produces one, but +# optimize_media.py's --webp does, so it's added here rather than touching +# that shared dict. +IMAGE_EXTENSION_TO_CONTENT_TYPE = {**EXTENSION_TO_CONTENT_TYPE, ".webp": WEBP_CONTENT_TYPE} + +# This script's own options, layered on top of optimize_media.py's (db-path +# has no equivalent there) - passed to resolve_config/load_config_file so +# --config can set any of them, the same mechanism optimize_media.py uses +# for its own options. +OWN_OPTION_SPECS = {**OPTION_SPECS, "db-path": ("db_path", Path)} + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("input_dir", type=Path, nargs="?", default=None, metavar="media_dir", + help="Directory of raw media to optimize, recursively (or set input-dir in --config)") + parser.add_argument("db_path", type=Path, nargs="?", default=None, + help="SQLite database to update, e.g. documentation.db (or set db-path in --config)") + parser.add_argument("output_dir", type=Path, nargs="?", default=None, metavar="work_dir", + help="Staging directory for optimized files; default: a temporary directory removed " + "afterwards (or set output-dir in --config)") + add_optimize_arguments(parser) + return parser + + +def delete_content(conn, path: str) -> None: + """Deletes a Content row and any chunked continuation fragments for it + (see insert_chunked_content/CHUNK_SIZE) - safe to call even if nothing + exists yet at that path. Content.path is UNIQUE, so this has to run + before any re-insert at the same path.""" + conn.execute("DELETE FROM Content WHERE path = ? OR path LIKE ?", (path, f"{path}-%")) + + +def insert_optimized_file(conn, data: bytes, name: str, db_path: str, language_id: int, content_type_cache: dict, + chunked_log: list) -> bool: + """Inserts one already-optimized file's bytes as-is. Unlike + populate_db.py's own insert_file, this does not run pngquant itself - + optimize_media.py already did, and running it again here would just + re-quantize an already-quantized image for no benefit. Returns False + (skipping the file, with a warning) for an extension with no known + content type.""" + content_type_value = IMAGE_EXTENSION_TO_CONTENT_TYPE.get(Path(name).suffix.lower()) + if content_type_value is None: + print(f"warning: no known content type for {name!r}; skipping", file=sys.stderr) + return False + if content_type_value not in content_type_cache: + content_type_cache[content_type_value] = get_content_type(conn, content_type_value) + content_type_id, compress = content_type_cache[content_type_value] + + if compress: + data = brotli.compress(data) + delete_content(conn, db_path) + insert_chunked_content(conn, db_path, language_id, content_type_id, 0, data, chunked_log) + return True + + +def build_rename_map(manifest: dict, logger: Logger) -> dict: + """Flattens optimize_directory's {relative_src: relative_dst} manifest + to {old_basename: new_basename}, matching k/html/images/*'s bare-filename + addressing. Warns (keeping the first) if two different renames collide + on the same old basename - e.g. two identically-named files in + different subdirectories of media_dir.""" + rename_map = {} + for old_rel, new_rel in sorted(manifest.items()): + old_name = Path(old_rel).name + new_name = Path(new_rel).name + if old_name == new_name: + continue + if old_name in rename_map and rename_map[old_name] != new_name: + logger.error( + f"warning: {old_rel!r} and an earlier file both renamed from {old_name!r}, to different names " + f"({rename_map[old_name]!r} vs {new_name!r}); keeping the first" + ) + continue + rename_map[old_name] = new_name + return rename_map + + +def reassemble_content(conn, path: str, first_content: bytes) -> bytes: + """Reassembles a possibly-chunked row's full bytes - mirrors + WebServer.kt's own reassembly protocol (see CHUNK_SIZE's docstring in + populate_db.py): a row is fragmented purely when its content is exactly + CHUNK_SIZE bytes, in which case "-1", "-2", ... are + concatenated until a missing or shorter-than-CHUNK_SIZE row is hit.""" + if len(first_content) < CHUNK_SIZE: + return first_content + parts = [first_content] + n = 1 + while True: + row = conn.execute("SELECT content FROM Content WHERE path = ?", (f"{path}-{n}",)).fetchone() + if row is None: + break + parts.append(row[0]) + if len(row[0]) < CHUNK_SIZE: + break + n += 1 + return b"".join(parts) + + +def rewrite_pages(conn, rename_map: dict, language_id: int, page_content_type_id: int, logger: Logger, + chunked_log: list) -> int: + """Rewrites every k/html/*.html page (and the nav row) that references a + renamed image, replacing "/k/html/images/" with + "/k/html/images/" wherever it appears. Operates directly on + each row's decompressed JSON text rather than parsing it: every image + reference is a literal IMAGES_URL_PREFIX+filename substring, baked in at + conversion time by md_to_json.py's Converter (resolve_image_src), so a + plain text substitution finds it correctly regardless of which block + type it ends up nested inside - no need to understand that nested block + schema here. The match is anchored on the escaped quote (\\") that + always immediately follows a rewritten src="..." attribute in the + stored JSON (see resolve_image_src/rewrite_urls - image references are + only ever embedded as HTML attributes, never as a bare JSON field on + their own), so a renamed file's name can't accidentally match as a + prefix of some other, unrelated, longer filename. Returns the number of + rows changed. + + ".html" is the exact literal suffix populate_db.py gives every base + page/nav row; fragment continuation rows are named "-" (the + "-N" appended after the ".html" already in path), so the path filter + below naturally excludes them without needing to detect chunking up + front. + + Substitutes in a single pass over each row's original text (one regex + covering every old_name, dispatched through `replacements` by exact + match) rather than N sequential str.replace calls on a mutating buffer. + Sequential replaces would risk a chain rename: if one rename's new_name + equals another rename's old_name (e.g. foo.png -> foo.webp and, + unrelated, foo.webp -> foo-2.webp), a later replace could re-match text + an earlier replace just wrote, sending an original foo.png reference to + foo-2.webp instead of foo.webp. Scanning the untouched original text + once makes that impossible.""" + if not rename_map: + return 0 + + rows = conn.execute( + "SELECT path, content, templateId FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? " + "AND templateId != 0", + (page_content_type_id,), + ).fetchall() + + replacements = { + f'{IMAGES_URL_PREFIX}{old_name}\\"': f'{IMAGES_URL_PREFIX}{new_name}\\"' + for old_name, new_name in rename_map.items() + } + old_ref_pattern = re.compile("|".join(re.escape(old_ref) for old_ref in replacements)) + + changed = 0 + for path, first_content, template_id in rows: + full = reassemble_content(conn, path, first_content) + text = brotli.decompress(full).decode("utf-8") + hits = len(old_ref_pattern.findall(text)) + if not hits: + continue + new_text = old_ref_pattern.sub(lambda m: replacements[m.group(0)], text) + blob = brotli.compress(new_text.encode("utf-8")) + delete_content(conn, path) + insert_chunked_content(conn, path, language_id, page_content_type_id, template_id, blob, chunked_log) + changed += 1 + logger.info(f"[URL FIX] {path}: updated {hits} image reference(s)") + return changed + + +# Matches a rewritten image src's filename, anchored the same way +# rewrite_pages' own known-rename substitutions are: resolve_image_src/ +# rewrite_urls only ever embed an image reference as an HTML src="..." +# attribute, which - JSON-encoded - always has the escaped quote (\") right +# after it, so this can't accidentally swallow past the end of the filename. +IMAGE_REF_RE = re.compile(re.escape(IMAGES_URL_PREFIX) + r'([^\\"]+)\\"') + + +def collect_referenced_media(conn, page_content_type_id: int) -> set: + """Bare filenames (e.g. "mascot.png") referenced by at least one + src="/k/html/images/" anywhere across current k/html/*.html page + content and the nav row - the same row selection/reassembly + rewrite_pages uses, just extracting every image reference found instead + of only substituting the ones in a known rename_map.""" + rows = conn.execute( + "SELECT path, content FROM Content WHERE path LIKE 'k/html/%.html' AND contentTypeID = ? AND templateId != 0", + (page_content_type_id,), + ).fetchall() + referenced = set() + for path, first_content in rows: + full = reassemble_content(conn, path, first_content) + text = brotli.decompress(full).decode("utf-8") + referenced.update(IMAGE_REF_RE.findall(text)) + return referenced + + +def list_stored_media(conn) -> dict: + """Bare filename -> Content.path (e.g. "mascot.png" -> "k/html/images/ + mascot.png") for every image currently stored under IMAGES_DB_PATH_PREFIX, + collapsing chunked continuation fragments ("-1", "-2", ...) + back into their base row, since deleting the base via delete_content + already takes its fragments with it (see CHUNK_SIZE's docstring in + populate_db.py for that fragmentation convention). A path is treated as + a fragment when stripping a trailing "-" yields another path + that's also present - the same convention this whole pipeline already + relies on elsewhere, ambiguous only for a base filename that itself + looks like "-", which no real optimized + media filename does.""" + paths = {row[0] for row in conn.execute( + "SELECT path FROM Content WHERE path LIKE ?", (f"{IMAGES_DB_PATH_PREFIX}%",) + )} + + def is_fragment(path: str) -> bool: + prefix, sep, suffix = path.rpartition("-") + return sep == "-" and suffix.isdigit() and prefix in paths + + return {path[len(IMAGES_DB_PATH_PREFIX):]: path for path in paths if not is_fragment(path)} + + +def delete_unreferenced_media(conn, page_content_type_id: int, logger: Logger) -> int: + """Deletes every currently-stored k/html/images/ row (base row and + any chunked fragments) that no page or the nav row references even once. + Must run after insertion and rename-rewriting, so it sees the final, + up-to-date state of both stored media and in-content references - a file + renamed this run is only "unreferenced" under its stale old name, which + rewrite_pages will have already fixed up by the time this runs. Returns + the number of images removed.""" + stored = list_stored_media(conn) + referenced = collect_referenced_media(conn, page_content_type_id) + removed = 0 + for name, path in sorted(stored.items()): + if name in referenced: + continue + delete_content(conn, path) + removed += 1 + logger.info(f"[UNUSED] removed {path} (not referenced by any page)") + return removed + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + try: + cfg = resolve_config(args, OWN_OPTION_SPECS, BUILTIN_DEFAULTS) + except RuntimeError as exc: + parser.error(str(exc)) + return + + if cfg["input_dir"] is None or cfg["db_path"] is None: + parser.error("media_dir and db_path must be given either as positional arguments or in --config") + + log_file_handle = open(cfg["log_file"], "w", encoding="utf-8") if cfg["log_file"] else None + logger = Logger(log_file_handle) + work_dir_is_temp = cfg["output_dir"] is None + work_dir = cfg["output_dir"] or Path(tempfile.mkdtemp(prefix="insert_optimized_media_")) + + try: + if not cfg["input_dir"].is_dir(): + logger.error(f"error: {cfg['input_dir']} is not a directory") + sys.exit(1) + if not cfg["db_path"].is_file(): + logger.error(f"error: {cfg['db_path']} does not exist") + sys.exit(1) + + if cfg["verbose"]: + logger.info("Config parameters:") + for key, (dest, _converter) in OWN_OPTION_SPECS.items(): + logger.info(f" {key} = {cfg.get(dest)}") + logger.info(f" work-dir = {work_dir}{' (temporary)' if work_dir_is_temp else ''}") + if args.config: + logger.info(f" (loaded from {args.config})") + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + + # Fail fast on a schema this database doesn't support - before + # spending time optimizing every file - rather than discovering it + # partway through the (rolled-back, but still wasted) DB transaction. + preflight_conn = sqlite3.connect(cfg["db_path"]) + try: + get_id(preflight_conn, "Languages", LANGUAGE) + get_id(preflight_conn, "ContentTypes", PAGE_CONTENT_TYPE) + if cfg["webp"]: + get_content_type(preflight_conn, WEBP_CONTENT_TYPE) + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + finally: + preflight_conn.close() + + work_dir.mkdir(parents=True, exist_ok=True) + stats = {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, "original_bytes": 0, + "optimized_bytes": 0} + logger.info(f"Optimizing media from {cfg['input_dir']} into {work_dir}...") + manifest = optimize_directory(cfg["input_dir"], work_dir, cfg=cfg, pngquant_path=pngquant_path, + logger=logger, stats=stats) + if stats["errors"]: + logger.error( + f"error: {stats['errors']} file(s) failed to optimize; aborting before touching the database" + ) + sys.exit(1) + rename_map = build_rename_map(manifest, logger) + + logger.info(f"Backing up {cfg['db_path']}...") + backup_path = backup_database(cfg["db_path"]) + logger.info(f"Backup written to {backup_path}") + + conn = sqlite3.connect(cfg["db_path"]) + try: + conn.execute("BEGIN") + language_id = get_id(conn, "Languages", LANGUAGE) + page_content_type_id = get_id(conn, "ContentTypes", PAGE_CONTENT_TYPE) + + content_type_cache = {} + chunked_log = [] + inserted = 0 + seen_names = {} + for out_path in sorted(work_dir.rglob("*")): + if out_path.is_dir(): + continue + name = out_path.name + if name in seen_names: + logger.error( + f"warning: {out_path} has the same filename as {seen_names[name]}; keeping the first, " + "skipping this one" + ) + continue + seen_names[name] = out_path + db_path = f"{IMAGES_DB_PATH_PREFIX}{name}" + if insert_optimized_file(conn, out_path.read_bytes(), name, db_path, language_id, content_type_cache, + chunked_log): + inserted += 1 + if cfg["verbose"]: + logger.info(f"[OK] {out_path} -> {db_path}") + + # A renamed file's old basename no longer appears anywhere under + # work_dir (that's what makes it a rename), so the loop above + # never visits its old db_path to replace it - it'd otherwise + # linger forever as an orphaned, no-longer-referenced row. + removed = 0 + for old_name in rename_map: + old_db_path = f"{IMAGES_DB_PATH_PREFIX}{old_name}" + delete_content(conn, old_db_path) + removed += 1 + if cfg["verbose"]: + logger.info(f"[REMOVED] {old_db_path} (renamed to {IMAGES_DB_PATH_PREFIX}{rename_map[old_name]})") + + changed_pages = rewrite_pages(conn, rename_map, language_id, page_content_type_id, logger, chunked_log) + + unreferenced_removed = delete_unreferenced_media(conn, page_content_type_id, logger) + + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + logger.info("Vacuuming database to reclaim freed space...") + vacuum_conn = sqlite3.connect(cfg["db_path"]) + try: + vacuum_conn.execute("VACUUM") + finally: + vacuum_conn.close() + + logger.info( + f"Done: inserted/updated {inserted} image(s) in {cfg['db_path']}, {removed} stale renamed-away row(s) " + f"removed, {changed_pages} page(s)/nav row(s) updated to match {len(rename_map)} renamed file(s), " + f"{unreferenced_removed} unreferenced image(s) deleted." + ) + if chunked_log: + logger.info(f"Chunked {len(chunked_log)} file(s) over {CHUNK_SIZE:,} bytes:") + for path, total_size, chunk_count in chunked_log: + logger.info(f" {path}: {total_size:,} bytes -> {chunk_count} chunks") + finally: + if log_file_handle is not None: + log_file_handle.close() + if work_dir_is_temp: + shutil.rmtree(work_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py new file mode 100644 index 000000000..df1a538f8 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/optimize_media.py @@ -0,0 +1,570 @@ +#!/usr/bin/env python3 +""" +optimize_media.py + +Recursively mirrors an input directory into an output directory, optimizing +image files along the way and copying everything else unchanged. + + - Raster images (png, jpg/jpeg, gif, bmp, tif/tiff, webp) are downscaled to + a maximum width (default 500px, preserving aspect ratio, never upscaled), + then optimized: PNGs through pngquant (at a configurable --pngquant-speed + trade-off - see https://pngquant.org/), other formats through Pillow's + own encoder (quality/optimize flags). With --webp, the resized image is + saved as WEBP instead of its original format. Animated GIFs get every + frame resized the same way (preserving frame count/duration/loop count) + rather than being copied through unchanged; --webp is not applied to + them, since animated WEBP re-encoding isn't implemented here. Any other + animated format (e.g. an animated WEBP given as input) is still copied + through unchanged, since per-frame resizing isn't implemented for it. + - SVGs (.svg) are aggressively optimized with the Scour library: metadata, + comments and editor cruft stripped, ids shortened, whitespace collapsed, + and every number rounded to --svg-precision decimal places. If the + optimized SVG still exceeds --svg-rasterize-threshold bytes, it's + rasterized (via cairosvg) and run through the same raster pipeline above + instead of being kept as a vector. If rasterizing fails for any reason + (e.g. cairosvg isn't installed), that's logged as a warning and the + optimized (still oversized) SVG is written instead - every input file + always ends up with something at its mirrored output path. + - Every other file is copied through unchanged. + +Usage: + python3 optimize_media.py [options] + python3 optimize_media.py --config myjob.config + +All options may instead be set in a .config file (one `key = value` per +line, '#' for comments) passed via --config; see OPTION_SPECS below for the +recognized keys, which are the same as the long-form CLI flags. Values +explicitly given on the command line always take precedence over the config +file. + +Every media file processed logs a line with its original and optimized +locations, regardless of --verbose (which adds byte sizes to that line and +prints all resolved config parameters up front). --log-file redirects all +log output (those per-file lines, warnings, and errors) to that file +instead of stdout/stderr. + +Requires the "pngquant" binary on PATH, the Pillow and scour Python packages +(pip install Pillow scour), and - only if any SVG actually needs rasterizing +- the cairosvg package (pip install cairosvg). +""" +import argparse +import io +import shutil +import subprocess +import sys +from pathlib import Path + +from PIL import Image + +try: + RESAMPLE = Image.Resampling.LANCZOS +except AttributeError: # Pillow < 9.1 + RESAMPLE = Image.LANCZOS + +RASTER_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tif", ".tiff", ".webp"} +SVG_EXTENSION = ".svg" + + +class Logger: + """Routes both info-level and error-level messages to a single + destination: the --log-file path if one was given (so a redirected run + still has one coherent, chronological log to inspect afterwards), or + stdout/stderr otherwise (so a normal terminal run keeps its usual + split - errors are still visible even if stdout is piped elsewhere).""" + + def __init__(self, file_handle): + self._fh = file_handle + + def info(self, msg: str) -> None: + print(msg, file=self._fh if self._fh is not None else sys.stdout, flush=self._fh is not None) + + def error(self, msg: str) -> None: + print(msg, file=self._fh if self._fh is not None else sys.stderr, flush=self._fh is not None) + + +def parse_bool(value: str) -> bool: + v = value.strip().lower() + if v in ("1", "true", "yes", "on", "y", "t"): + return True + if v in ("0", "false", "no", "off", "n", "f"): + return False + raise ValueError(f"not a boolean: {value!r}") + + +# Maps a config-file key (and, with dashes, a --long-form CLI flag) to the +# argparse dest it corresponds to and the converter used to parse its value +# out of the config file's plain-text "key = value" form. Order here is also +# the order config parameters get listed in when --verbose is on. +OPTION_SPECS = { + "input-dir": ("input_dir", Path), + "output-dir": ("output_dir", Path), + "max-width": ("max_width", int), + "jpeg-quality": ("jpeg_quality", int), + "webp": ("webp", parse_bool), + "webp-quality": ("webp_quality", int), + "pngquant-speed": ("pngquant_speed", int), + "svg-precision": ("svg_precision", int), + "svg-rasterize-threshold": ("svg_rasterize_threshold", int), + "verbose": ("verbose", parse_bool), + "log-file": ("log_file", Path), +} + +# Used for any option left unset by both the CLI and (if given) --config. +BUILTIN_DEFAULTS = { + "max_width": 500, + "jpeg_quality": 82, + "webp": False, + "webp_quality": 80, + "pngquant_speed": 4, # pngquant's own default; 1 = slow/best, 11 = fast/rough + "svg_precision": 4, + "svg_rasterize_threshold": 300 * 1024, # 300KB + "verbose": False, +} + + +def load_config_file(path: Path, option_specs: dict = OPTION_SPECS) -> dict: + """Parses a simple `key = value` (or `key: value`) config file, one + option per line; blank lines and lines starting with '#' or ';' are + ignored. A bare key with no value means true (for boolean options). + Keys match `option_specs` (case-insensitive, dashes or underscores) - + defaulting to this module's own OPTION_SPECS, but overridable so a + caller layering its own options on top (e.g. insert_optimized_media.py + adding "db-path") can reuse this same parser for its extended key set. + Returns {dest: converted_value}.""" + if not path.is_file(): + raise RuntimeError(f"config file not found: {path}") + + overrides = {} + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#") or line.startswith(";"): + continue + if "=" in line: + key, _, value = line.partition("=") + elif ":" in line: + key, _, value = line.partition(":") + else: + key, value = line, "true" + key = key.strip().lower().replace("_", "-") + value = value.strip() + + spec = option_specs.get(key) + if spec is None: + raise RuntimeError(f"{path}:{lineno}: unknown config option {key!r}") + dest, converter = spec + try: + overrides[dest] = converter(value) + except ValueError as exc: + raise RuntimeError(f"{path}:{lineno}: invalid value for {key!r}: {exc}") from exc + return overrides + + +def find_pngquant() -> str: + path = shutil.which("pngquant") + if path is None: + raise RuntimeError("pngquant not found on PATH; install it (e.g. `apt install pngquant`) and retry") + return path + + +def quantize_png_bytes(data: bytes, pngquant_path: str, speed: int, name: str, logger: Logger) -> bytes: + """Runs pngquant on raw PNG bytes (stdin -> stdout, no temp files) at the + given --speed trade-off (1 = slow/best quality, 11 = fast/rough - see + https://pngquant.org/), returning the compressed bytes. Falls back to + the original bytes if pngquant declines (e.g. exit 99: result would fall + below --quality's floor) or otherwise fails - a slightly larger PNG + beats a missing one.""" + result = subprocess.run( + [pngquant_path, "--quality", "65-95", "--speed", str(speed), "--strip", "--force", "--output", "-", "-"], + input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0 or not result.stdout: + logger.error( + f"warning: pngquant declined to compress {name!r} " + f"(exit {result.returncode}: {result.stderr.decode(errors='replace').strip()}); keeping original" + ) + return data + return result.stdout + + +def resize_if_needed(img: Image.Image, max_width: int) -> Image.Image: + if img.width <= max_width: + return img + new_height = max(1, round(img.height * (max_width / img.width))) + return img.resize((max_width, new_height), RESAMPLE) + + +def normalize_mode(img: Image.Image) -> Image.Image: + """Flattens palette/CMYK modes to something every downstream encoder + (JPEG, WEBP, PNG) can handle directly, preserving alpha where present.""" + if img.mode == "P": + return img.convert("RGBA") if img.info.get("transparency") is not None else img.convert("RGB") + if img.mode == "CMYK": + return img.convert("RGB") + return img + + +def encode_raster(img: Image.Image, dst: Path, *, suffix: str, max_width: int, jpeg_quality: int, webp: bool, + webp_quality: int, pngquant_path: str, pngquant_speed: int, logger: Logger) -> Path: + """Resizes an already-loaded image and writes it under dst (whose suffix + may be swapped to .webp), choosing the encoder by `suffix` (the source + file's extension, or ".png" for a freshly rasterized SVG). Returns the + path actually written. Shared by optimize_raster and optimize_svg's + rasterize-on-oversize fallback so both go through identical resize/ + encode logic.""" + img = normalize_mode(img) + + if suffix == ".png" and not webp: + # pngquant runs against the full-resolution pixels here, before any + # downscaling - its palette selection and dithering have the whole + # original image's color detail to work from, rather than the + # coarser, already-blended pixels a resize would leave it with. The + # quantized result is then decoded back and resized down below, same + # as any other image. + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + quantized = quantize_png_bytes(buf.getvalue(), pngquant_path, pngquant_speed, str(dst), logger) + img = Image.open(io.BytesIO(quantized)) + img.load() + img = normalize_mode(img) # pngquant's output PNG is palette ("P") mode + + img = resize_if_needed(img, max_width) + + if webp: + dst = dst.with_suffix(".webp") + img.save(dst, "WEBP", quality=webp_quality, method=6) + return dst + + if suffix == ".png": + # Resizing the first quantize pass's palette image back down blended + # it back into full RGB(A) - re-quantize at the final size to + # restore a compact palette PNG, now informed by both the full- + # resolution pass above and the actual delivered dimensions. + buf = io.BytesIO() + img.save(buf, "PNG", optimize=True) + dst.write_bytes(quantize_png_bytes(buf.getvalue(), pngquant_path, pngquant_speed, str(dst), logger)) + elif suffix in (".jpg", ".jpeg"): + if img.mode != "RGB": + img = img.convert("RGB") + img.save(dst, "JPEG", quality=jpeg_quality, optimize=True, progressive=True) + else: + img.save(dst, optimize=True) + return dst + + +def resize_animated_gif(img: Image.Image, dst: Path, max_width: int) -> Path: + """Resizes every frame of an animated GIF down to max_width, preserving + frame count, each frame's own duration, and the loop count - a naive + single-frame resize (or the old copy-through-unchanged behavior) would + otherwise silently drop the animation entirely. seek()+convert("RGBA") + composites each frame the way Pillow's GIF plugin normally displays it + (accounting for the previous frame's disposal method), so every frame + saved below is a complete, standalone image rather than a partial + update relying on its predecessor - hence disposal=2 (restore to + background) on save, rather than trying to preserve each original + frame's own disposal method. --webp is intentionally not honored here: + animated WEBP re-encoding is a separate feature this doesn't attempt.""" + n_frames = getattr(img, "n_frames", 1) + loop = img.info.get("loop", 0) + frames = [] + durations = [] + for i in range(n_frames): + img.seek(i) + frames.append(resize_if_needed(img.convert("RGBA"), max_width)) + durations.append(img.info.get("duration", 100)) + frames[0].save( + dst, save_all=True, append_images=frames[1:], duration=durations, loop=loop, disposal=2, optimize=True, + ) + return dst + + +def optimize_raster(src: Path, dst: Path, **encode_kwargs) -> Path: + with Image.open(src) as img: + if getattr(img, "is_animated", False): + if src.suffix.lower() == ".gif": + return resize_animated_gif(img, dst, encode_kwargs["max_width"]) + # Animated non-GIF (e.g. webp): per-frame resizing/re-encoding is + # out of scope here - copy through unchanged rather than + # flattening it to a single frame and silently breaking the + # animation. + shutil.copy2(src, dst) + return dst + return encode_raster(img, dst, suffix=src.suffix.lower(), **encode_kwargs) + + +def rasterize_svg(svg_text: str, max_width: int) -> Image.Image: + """Renders SVG markup to a raster image at exactly `max_width` pixels + wide (cairosvg computes the proportional height from the SVG's own + viewBox/aspect ratio), for SVGs too large to keep as vector output.""" + try: + import cairosvg + except ImportError as exc: + raise RuntimeError( + "cairosvg is required to rasterize oversized SVGs; install it with `pip install cairosvg`" + ) from exc + png_bytes = cairosvg.svg2png(bytestring=svg_text.encode("utf-8"), output_width=max_width) + img = Image.open(io.BytesIO(png_bytes)) + img.load() + return img + + +def optimize_svg(src: Path, dst: Path, *, precision: int, rasterize_threshold: int, max_width: int, + jpeg_quality: int, webp: bool, webp_quality: int, pngquant_path: str, pngquant_speed: int, + logger: Logger) -> tuple: + """Optimizes one SVG with Scour, rounding numbers to `precision` decimal + places. If the optimized markup is still over `rasterize_threshold` + bytes, rasterizes it and runs it through the raster pipeline instead of + writing it as a (still large) vector. Returns (final_path, was_rasterized).""" + from scour import scour + + options = scour.generateDefaultOptions() + # Aggressive settings, roughly equivalent to: + # scour --enable-viewboxing --enable-id-stripping --enable-comment-stripping + # --shorten-ids --indent=none --strip-xml-prolog --set-precision= + options.remove_metadata = True + options.remove_descriptive_elements = True + options.remove_titles = True + options.remove_descriptions = True + options.strip_comments = True + options.strip_ids = True + options.shorten_ids = True + options.keep_editor_data = False + options.strip_xml_prolog = True + options.enable_viewboxing = True + options.simple_colors = True + options.style_to_xml = True + options.group_collapse = True + options.group_create = True + options.indent_type = "none" + options.newlines = False + options.digits = precision + + in_string = src.read_text(encoding="utf-8") + out_string = scour.scourString(in_string, options) + out_bytes = out_string.encode("utf-8") + + if len(out_bytes) > rasterize_threshold: + try: + img = rasterize_svg(out_string, max_width) + final = encode_raster( + img, dst.with_suffix(".png"), suffix=".png", max_width=max_width, jpeg_quality=jpeg_quality, + webp=webp, webp_quality=webp_quality, pngquant_path=pngquant_path, pngquant_speed=pngquant_speed, + logger=logger, + ) + logger.info( + f"note: rasterized {src} -> {final} (optimized SVG was {len(out_bytes):,} bytes, " + f"over the {rasterize_threshold:,} byte threshold)" + ) + return final, True + except Exception as exc: # noqa: BLE001 - fall through to writing the vector below instead + logger.error(f"warning: failed to rasterize {src} ({exc}); keeping optimized SVG instead") + + dst.write_bytes(out_bytes) + return dst, False + + +def process_file(src: Path, dst: Path, *, cfg: dict, pngquant_path: str, stats: dict, logger: Logger) -> Path: + """Optimizes (or copies through) one file. Returns the path actually + written on success (which may differ from `dst` - webp conversion or + SVG rasterization changes the extension), or None on error (already + logged; `stats["errors"]` is incremented so callers can tell without + inspecting the return value).""" + dst.parent.mkdir(parents=True, exist_ok=True) + suffix = src.suffix.lower() + original_size = src.stat().st_size + + try: + if suffix == SVG_EXTENSION: + dst_final, rasterized = optimize_svg( + src, dst, precision=cfg["svg_precision"], rasterize_threshold=cfg["svg_rasterize_threshold"], + max_width=cfg["max_width"], jpeg_quality=cfg["jpeg_quality"], webp=cfg["webp"], + webp_quality=cfg["webp_quality"], pngquant_path=pngquant_path, pngquant_speed=cfg["pngquant_speed"], + logger=logger, + ) + if rasterized: + stats["svg_rasterized"] += 1 + kind = "svg, rasterized" + else: + stats["svg"] += 1 + kind = "svg" + elif suffix in RASTER_EXTENSIONS: + dst_final = optimize_raster( + src, dst, max_width=cfg["max_width"], jpeg_quality=cfg["jpeg_quality"], webp=cfg["webp"], + webp_quality=cfg["webp_quality"], pngquant_path=pngquant_path, pngquant_speed=cfg["pngquant_speed"], + logger=logger, + ) + stats["raster"] += 1 + kind = "raster" + else: + shutil.copy2(src, dst) + dst_final = dst + stats["copied"] += 1 + kind = "copied" + except Exception as exc: # noqa: BLE001 - keep processing the rest of the tree + stats["errors"] += 1 + logger.error(f"error: failed to process {src}: {exc}") + return None + + optimized_size = dst_final.stat().st_size + stats["original_bytes"] += original_size + stats["optimized_bytes"] += optimized_size + if kind != "copied": + # Always shown (not just under --verbose) - a per-file record of + # where the optimized copy of each media file actually landed, + # since that's not otherwise derivable once optimization has + # renamed a file (webp conversion, SVG rasterization). + message = f"Optimized {src} -> {dst_final}" + if cfg["verbose"]: + saved = original_size - optimized_size + pct = (saved / original_size * 100) if original_size else 0.0 + message = ( + f"[OK][{kind}] {src} -> {dst_final}: {original_size:,} -> {optimized_size:,} bytes " + f"(saved {saved:,} bytes, {pct:.1f}%)" + ) + logger.info(message) + return dst_final + + +def optimize_directory(input_dir: Path, output_dir: Path, *, cfg: dict, pngquant_path: str, logger: Logger, + stats: dict) -> dict: + """Walks input_dir recursively, optimizing every file into the mirrored + location under output_dir (see process_file). Returns + {relative_src_path: relative_dst_path} for every file whose output path + ended up different from its input path (webp conversion, or an SVG + rasterized to PNG/WEBP) - callers that also maintain references to these + files elsewhere (e.g. insert_optimized_media.py, fixing up image URLs + stored in a database) use this to know what changed.""" + renamed = {} + for src in sorted(input_dir.rglob("*")): + if src.is_dir(): + continue + rel = src.relative_to(input_dir) + dst = output_dir / rel + dst_final = process_file(src, dst, cfg=cfg, pngquant_path=pngquant_path, stats=stats, logger=logger) + if dst_final is None: + continue + rel_final = dst_final.relative_to(output_dir) + if rel_final != rel: + renamed[str(rel)] = str(rel_final) + return renamed + + +def add_optimize_arguments(parser: argparse.ArgumentParser) -> None: + """Adds every --tuning-flag (everything except the input/output + positionals) to `parser` - split out from build_parser() so a caller + with its own positional arguments (e.g. insert_optimized_media.py, which + also needs a database path) can still get these for free instead of + redeclaring them.""" + parser.add_argument("--config", type=Path, default=None, + help="Path to a .config file providing any of the options below") + parser.add_argument("--max-width", type=int, default=None, + help=f"Max width in pixels for raster images (default: {BUILTIN_DEFAULTS['max_width']})") + parser.add_argument("--jpeg-quality", type=int, default=None, + help=f"JPEG output quality, 0-95 (default: {BUILTIN_DEFAULTS['jpeg_quality']})") + parser.add_argument("--webp", action="store_true", default=None, + help="Convert optimized raster images (and rasterized SVGs) to WEBP") + parser.add_argument("--webp-quality", type=int, default=None, + help=f"WEBP output quality, 0-100 (default: {BUILTIN_DEFAULTS['webp_quality']})") + parser.add_argument("--pngquant-speed", type=int, default=None, + help="pngquant speed/quality trade-off, 1 (slow/best) - 11 (fast/rough); " + f"see https://pngquant.org/ (default: {BUILTIN_DEFAULTS['pngquant_speed']})") + parser.add_argument("--svg-precision", type=int, default=None, + help="Decimal places to round SVG numbers to via Scour " + f"(default: {BUILTIN_DEFAULTS['svg_precision']})") + parser.add_argument("--svg-rasterize-threshold", type=int, default=None, + help="Rasterize an optimized SVG if it's still over this many bytes " + f"(default: {BUILTIN_DEFAULTS['svg_rasterize_threshold']:,})") + parser.add_argument("--verbose", action="store_true", default=None, + help="Add original/optimized byte sizes to the per-file log line every media file " + "already gets, plus all resolved config parameters up front") + parser.add_argument("--log-file", type=Path, default=None, + help="Write all log output here instead of stdout/stderr") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("input_dir", type=Path, nargs="?", default=None, + help="Directory to read files from, recursively (or set input-dir in --config)") + parser.add_argument("output_dir", type=Path, nargs="?", default=None, + help="Directory to mirror optimized output into (or set output-dir in --config)") + add_optimize_arguments(parser) + return parser + + +def resolve_config(args: argparse.Namespace, option_specs: dict = OPTION_SPECS, + builtin_defaults: dict = BUILTIN_DEFAULTS) -> dict: + """Merges CLI args over --config file values over builtin_defaults (in + that precedence order) into one dict keyed by dest name. option_specs/ + builtin_defaults default to this module's own, but are overridable for a + caller (e.g. insert_optimized_media.py) extending them with its own + extra options (like "db-path").""" + file_overrides = load_config_file(args.config, option_specs) if args.config else {} + + cfg = {} + for dest, _converter in option_specs.values(): + cli_value = getattr(args, dest, None) + if cli_value is not None: + cfg[dest] = cli_value + elif dest in file_overrides: + cfg[dest] = file_overrides[dest] + elif dest in builtin_defaults: + cfg[dest] = builtin_defaults[dest] + else: + cfg[dest] = None # input_dir/output_dir: no built-in default + return cfg + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + try: + cfg = resolve_config(args) + except RuntimeError as exc: + parser.error(str(exc)) + return + + if cfg["input_dir"] is None or cfg["output_dir"] is None: + parser.error("input_dir and output_dir must be given either as positional arguments or in --config") + + log_file_handle = open(cfg["log_file"], "w", encoding="utf-8") if cfg["log_file"] else None + logger = Logger(log_file_handle) + try: + if not cfg["input_dir"].is_dir(): + logger.error(f"error: {cfg['input_dir']} is not a directory") + sys.exit(1) + + if cfg["verbose"]: + logger.info("Config parameters:") + for key, (dest, _converter) in OPTION_SPECS.items(): + logger.info(f" {key} = {cfg[dest]}") + if args.config: + logger.info(f" (loaded from {args.config})") + + try: + pngquant_path = find_pngquant() + except RuntimeError as exc: + logger.error(f"error: {exc}") + sys.exit(1) + + stats = {"raster": 0, "svg": 0, "svg_rasterized": 0, "copied": 0, "errors": 0, "original_bytes": 0, + "optimized_bytes": 0} + optimize_directory(cfg["input_dir"], cfg["output_dir"], cfg=cfg, pngquant_path=pngquant_path, logger=logger, + stats=stats) + + saved = stats["original_bytes"] - stats["optimized_bytes"] + pct = (saved / stats["original_bytes"] * 100) if stats["original_bytes"] else 0.0 + logger.info( + f"Done: {stats['raster']} raster image(s) optimized, {stats['svg']} SVG(s) optimized, " + f"{stats['svg_rasterized']} SVG(s) rasterized, {stats['copied']} other file(s) copied, " + f"{stats['errors']} error(s). Total size: {stats['original_bytes']:,} -> {stats['optimized_bytes']:,} " + f"bytes (saved {saved:,} bytes, {pct:.1f}%)." + ) + if stats["errors"]: + sys.exit(1) + finally: + if log_file_handle is not None: + log_file_handle.close() + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py new file mode 100644 index 000000000..c338b9dc0 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +""" +Populates a documentation.db-schema SQLite database with the same content +templates/page.peb and this project's md_to_json.py conversion pipeline +produce for the static site, replacing what's currently at k/html/*. + +Usage: + python3 populate_db.py [db-path] + [--tree-file kr.tree] [--topics-subdir topics] + [--blacklisted-element-titles "Ancestor\\/.../Element Title" ...] + +--blacklisted-element-titles names element(s) +to drop from kr.tree entirely before anything else below reads it: the +element and its whole subtree get no nav entry, none of their .md sub-topics +get converted or inserted, and any *other*, non-blacklisted page's in-content +link to one of those .md files renders broken/styled (same as any other +unresolved link - see broken-ext-link-color) rather than pointing somewhere +that no longer exists. + +Each value is the *full* toc-title path from a top-level down +to the one being blacklisted, since toc-title alone is not unique across +kr.tree (e.g. plenty of "Overview"s). Levels are joined by the two-character +sequence "\\/" (backslash then slash) rather than a bare "/", because a bare +"/" routinely appears *within* a single real toc-title (e.g. "Swift/ +Objective-C and C interop") and this way that overwhelmingly common case +needs no escaping at all - only the rare level separator does. So to +blacklist the "Swift/Objective-C and C interop" element nested under the +top-level "Interoperability" element, pass +"Interoperability\\/Swift/Objective-C and C interop": split on "\\/" that's +["Interoperability", "Swift/Objective-C and C interop"], matching kr.tree's +actual nesting - the inner "/" is left untouched since it wasn't preceded by +a backslash. + + defaults to "documentation.db". A safety backup (via SQLite's +"VACUUM INTO", which is safe even against a live/WAL-mode database) is +written next to it before any changes: ".backup-". + + is Writerside's own official image output for this doc set +(e.g. "webHelpImages.zip", found next to kr.tree) - a flat archive with no +subdirectories, one entry per image, already exactly as Writerside itself +would serve them. Rather than re-deriving image content/sizing ourselves +from the raw source tree (which is a plain, uncompressed truecolor export - +several times larger than what a real Writerside build actually ships, +since it applies its own image optimization we have no easy way to +replicate faithfully), this script just copies that zip's entries in +directly, so k/html/images/ ends up byte-for-byte what Writerside +itself produces. + +What this does, inside a single transaction (rolled back on any error): + 1. Deletes every Content row with path LIKE 'k/html/%' or 'assets/%' - the + former includes the existing *.html doc pages AND everything else + parked there (images, the old Writerside JS bundle under + k/html/frontend/, none of which this script replaces); the latter is + wherever a previous run of this script put images/CSS/JS, all of + which get freshly re-inserted below. + 2. Upserts templates/page.peb and templates/nav.peb into Templates. + page.peb's