diff --git a/.github/workflows/build-kotlin-docs.yaml b/.github/workflows/build-kotlin-docs.yaml new file mode 100644 index 000000000..fa981350b --- /dev/null +++ b/.github/workflows/build-kotlin-docs.yaml @@ -0,0 +1,378 @@ +name: Build Kotlin Docs + +# CI counterpart of ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh - +# same five steps (find_missing_assets -> populate_db -> insert_optimized_media +# -> build-stdlib-json-docs -> sync_kdoc_json_to_db), same ADFA-4737 blacklist, +# but sourcing its inputs from fresh git checkouts instead of a developer's +# local machine, and reading/writing the real database on Google Drive +# (GOOGLE_DRIVE_FILE_ID) instead of a local SOURCE_DB copy. +# +# KNOWN LIMITATION: populate_db.py requires Writerside's own image export +# ("webHelpImages.zip"), which JetBrains only produces via IntelliJ IDEA's +# Writerside plugin build/export action - there is no headless/CLI way to +# generate it (see ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md, +# "Inputs you need before starting"). So this workflow downloads it from +# Google Drive rather than generating it itself; someone has to run that IDE +# export, upload the zip to Drive, and supply its file ID (see +# images_zip_file_id / GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID below) before +# triggering a run that touches the website docs. Use skip_website_docs to +# bypass this entirely and only refresh the kotlin-stdlib/-reflect/-test +# JSON content. +# +# Required secrets (already configured - see docdb-regression-test.yaml for +# their other use in this repo): +# GCP_WIF_PROVIDER - Workload Identity Federation provider name +# GCP_WIF_SERVICE_ACCOUNT - Service account email for WIF (needs read +# access to the images-zip file below, and +# write access - not just view - to the +# database file, since this workflow +# overwrites it) +# GOOGLE_DRIVE_FILE_ID - File ID of the production documentation.db +# (stored on Drive as a zip) +# +# Optional secret (falls back to the images_zip_file_id input if unset; see +# also the hard-coded TEST_*_FILE_ID overrides below for one-off testing): +# GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID - File ID of Writerside's webHelpImages.zip export +# +# Optional secret (Slack notifications are skipped with a warning if unset): +# SLACK_WEBHOOK_URL - Incoming Webhook URL for the "Notify Slack" steps +# below ("Grabbing baton" on start, "...Dropping +# baton" on finish - org shorthand for lock +# acquire/release, since this workflow mutates a +# single shared Drive file). + +permissions: + contents: read + id-token: write + +# This workflow overwrites a single shared Drive file - never let two runs +# race to upload against each other. +concurrency: + group: build-kotlin-docs + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + kotlin_web_site_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin-web-site to check out for the + "docs" tree (topics/, images/, kr.tree, v.list). Leave empty to use + the repo's default branch. + required: false + default: '' + kotlin_ref: + description: >- + Branch/tag/commit of JetBrains/kotlin to check out for the + kotlin-stdlib-docs build. Leave empty to use the repo's default + branch. Pin this to a real release tag for a reproducible build. + required: false + default: '' + images_zip_file_id: + description: >- + Google Drive file ID for Writerside's webHelpImages.zip export + matching kotlin_web_site_ref (see KNOWN LIMITATION above). Falls + back to the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret if left empty. + Ignored if skip_website_docs is true. + required: false + default: '' + skip_website_docs: + description: 'Skip the kotlin-web-site steps and only refresh kotlin-stdlib/-reflect/-test JSON content.' + required: false + default: false + type: boolean + dry_run: + description: >- + If true, build and verify everything but do NOT upload the result + back to Google Drive - the production database is left untouched. + Set to false only once you trust a given ref/URL combination (see + this workflow's testing notes). + required: false + default: true + type: boolean + +jobs: + build-kotlin-docs: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + KOTLIN_WEB_SITE_REF: ${{ inputs.kotlin_web_site_ref }} + KOTLIN_REF: ${{ inputs.kotlin_ref }} + DB_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_FILE_ID }} + IMAGES_ZIP_FILE_ID_INPUT: ${{ inputs.images_zip_file_id }} + IMAGES_ZIP_FILE_ID_SECRET: ${{ secrets.GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID }} + # --- Hard-coded overrides for one-off manual testing ----------------- + # Fill in either of these with a literal Google Drive file ID to + # bypass the secret/input resolution above for a quick, repeatable + # test run (e.g. against scratch copies of the database/images zip on + # Drive). Leave both empty ('') for normal operation. + TEST_DB_FILE_ID: '' + TEST_IMAGES_ZIP_FILE_ID: '' + steps: + - name: Checkout OfflineDocumentationTools + uses: actions/checkout@v4 + + - name: Resolve Google Drive file IDs + run: | + DB_FILE_ID="${TEST_DB_FILE_ID:-$DB_FILE_ID_SECRET}" + IMG_FILE_ID="${TEST_IMAGES_ZIP_FILE_ID:-${IMAGES_ZIP_FILE_ID_INPUT:-$IMAGES_ZIP_FILE_ID_SECRET}}" + if [ -z "$DB_FILE_ID" ]; then + echo "Error: no database file ID resolved - set the GOOGLE_DRIVE_FILE_ID secret, or TEST_DB_FILE_ID above for a test run" >&2 + exit 1 + fi + echo "Resolved DB_FILE_ID: ${DB_FILE_ID:+(set)}" + echo "Resolved IMG_FILE_ID: ${IMG_FILE_ID:+(set)}" + echo "DB_FILE_ID=$DB_FILE_ID" >> "$GITHUB_ENV" + echo "IMG_FILE_ID=$IMG_FILE_ID" >> "$GITHUB_ENV" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Set up JDK (for the kdoc-to-json / kotlin-stdlib-docs Gradle builds) + uses: actions/setup-java@v4 + with: + distribution: temurin + # kdoc-to-json's own Gradle wrapper is pinned to Gradle 9.1.0, which + # needs JDK 17+. Bump this if the kotlin checkout's own wrapper + # (invoked by build-stdlib-json-docs.sh against kotlin-stdlib-docs) + # turns out to need something newer - verify on first real run. + java-version: '17' + + - name: Install system dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y pngquant unzip zip sqlite3 + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + # markdown-it-py/scour/cairosvg: ProcessKotlinWebsiteJSON's own + # requirements (see its README), not in the root requirements.txt. + # google-api-python-client & friends: Drive download/upload, same + # libraries check-tools/download_database.py already depends on. + pip install markdown-it-py scour cairosvg \ + google-api-python-client google-auth-httplib2 google-auth-oauthlib + + - name: Authenticate to Google Cloud using Workload Identity Federation + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} + service_account: ${{ secrets.GCP_WIF_SERVICE_ACCOUNT }} + access_token_scopes: | + https://www.googleapis.com/auth/drive.file + + - name: Download current documentation.db from Google Drive + run: | + python3 check-tools/download_database.py "$DB_FILE_ID" documentation.zip + unzip -o documentation.zip + if [ ! -f documentation.db ]; then + found="$(find . -maxdepth 2 -name documentation.db | head -n1)" + [ -n "$found" ] && mv "$found" documentation.db + fi + test -f documentation.db + sqlite3 documentation.db "SELECT 1;" > /dev/null + rm -f documentation.zip + echo "DB_SIZE=$(stat -c%s documentation.db 2>/dev/null || stat -f%z documentation.db)" >> "$GITHUB_ENV" + + - name: 'Notify Slack: build started' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Grabbing baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi + + - name: Clone kotlin-web-site + if: ${{ !inputs.skip_website_docs }} + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_WEB_SITE_REF" ] && ARGS+=(--branch "$KOTLIN_WEB_SITE_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin-web-site.git kotlin-web-site + + - name: Download Writerside image export from Google Drive + if: ${{ !inputs.skip_website_docs }} + run: | + if [ -z "$IMG_FILE_ID" ]; then + echo "Error: no images-zip file ID resolved - set images_zip_file_id, the GOOGLE_DRIVE_IMAGES_ZIP_FILE_ID secret, or TEST_IMAGES_ZIP_FILE_ID above (see KNOWN LIMITATION in this workflow's header comment). Required unless skip_website_docs is true." >&2 + exit 1 + fi + # download_database.py is a generic Drive-file-by-ID downloader + # despite its name - reused here rather than duplicating the + # WIF/Drive-API download logic for a second file type. + python3 check-tools/download_database.py "$IMG_FILE_ID" webHelpImages.zip + + - name: 'Step 1/5: find_missing_assets.py (source QA report)' + if: ${{ !inputs.skip_website_docs }} + run: | + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/find_missing_assets.py \ + kotlin-web-site/docs missing-assets-report.md + + - name: Upload missing-assets report + if: ${{ !inputs.skip_website_docs }} + uses: actions/upload-artifact@v4 + with: + name: missing-assets-report + path: missing-assets-report.md + + - name: 'Step 2/5: populate_db.py (convert docs, prune blacklist, insert into db)' + if: ${{ !inputs.skip_website_docs }} + run: | + # Same three blacklist entries as run_e2e_pipeline_test.sh + # (ADFA-4737) - re-derive these from kotlin-web-site/docs/kr.tree + # if its nav structure has changed since this was written. + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/populate_db.py \ + kotlin-web-site/docs \ + ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json \ + webHelpImages.zip \ + documentation.db \ + --blacklisted-element-titles \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' + + - name: 'Step 3/5: insert_optimized_media.py (re-optimize + reinsert images)' + if: ${{ !inputs.skip_website_docs }} + run: | + # --webp requires an "image/webp" ContentTypes row, which this + # database doesn't ship with by default (idempotent). + sqlite3 documentation.db \ + "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + mkdir -p media + unzip -q webHelpImages.zip -d media + python3 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/insert_optimized_media.py \ + media documentation.db \ + --jpeg-quality 85 --webp --webp-quality 90 --verbose + + - name: Clone kotlin (for kotlin-stdlib-docs) + run: | + ARGS=(--depth 1) + [ -n "$KOTLIN_REF" ] && ARGS+=(--branch "$KOTLIN_REF") + git clone "${ARGS[@]}" https://github.com/JetBrains/kotlin.git kotlin-repo + + - name: 'Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON)' + id: stdlib_docs + run: | + OUTPUT="$(Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh kotlin-repo stdlib-json-build)" + echo "Generated JSON docs at $OUTPUT" + echo "all_libs_dir=$OUTPUT" >> "$GITHUB_OUTPUT" + + - name: 'Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content)' + run: | + python3 scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py \ + "${{ steps.stdlib_docs.outputs.all_libs_dir }}" --db documentation.db + + - name: Summary + run: | + python3 - documentation.db <<'PYEOF' + import sqlite3 + import sys + + conn = sqlite3.connect(sys.argv[1]) + + def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + print(f"Database: {sys.argv[1]}") + print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") + print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") + print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") + print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") + print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") + print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + conn.close() + PYEOF + + - name: Blacklist pruning verification + if: ${{ !inputs.skip_website_docs }} + run: | + python3 - ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON kotlin-web-site/docs documentation.db \ + 'Development\/Web development' \ + 'Interoperability\/Swift/Objective-C and C interop' \ + 'Interoperability\/JavaScript interop' <<'PYEOF' + import sqlite3 + import sys + import xml.etree.ElementTree as ET + from pathlib import Path + + process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] + sys.path.insert(0, process_dir) + import populate_db # noqa: E402 + + root = ET.parse(Path(docs_root) / "kr.tree").getroot() + blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} + blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + + conn = sqlite3.connect(db_path) + leftover = [] + for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) + conn.close() + + print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") + for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") + print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + + if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a .") + sys.exit(1) + if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + + print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") + PYEOF + + - name: Upload built database as workflow artifact + uses: actions/upload-artifact@v4 + with: + name: documentation-db-${{ github.run_number }} + path: documentation.db + retention-days: 14 + + - name: Zip updated database for upload + if: ${{ !inputs.dry_run }} + run: zip -j documentation.zip documentation.db + + - name: Upload updated database to Google Drive + if: ${{ !inputs.dry_run }} + run: | + python3 - <<'PYEOF' + import os + from google.auth import default + from googleapiclient.discovery import build + from googleapiclient.http import MediaFileUpload + + file_id = os.environ["DB_FILE_ID"] + credentials, _ = default() + service = build("drive", "v3", credentials=credentials) + media = MediaFileUpload("documentation.zip", mimetype="application/zip", resumable=True) + updated = service.files().update( + fileId=file_id, media_body=media, fields="id, modifiedTime, md5Checksum" + ).execute() + print(f"Uploaded new revision of {file_id}: {updated}") + PYEOF + + - name: 'Notify Slack: build complete' + if: ${{ !inputs.dry_run }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "SLACK_WEBHOOK_URL not set - skipping Slack notification" >&2 + else + curl -sS -X POST -H 'Content-type: application/json' \ + --data '{"text": "Updated Kotlin documentation. Dropping baton"}' \ + "$SLACK_WEBHOOK_URL" || echo "warning: Slack notification failed" >&2 + fi diff --git a/.gitignore b/.gitignore index 59cc3acb6..42fa9ce9c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ __pycache__/ *$py.class *.db *.sqlite +run_e2e_pipeline_test.local.sh +grep_content_blobs.local.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..e6c8d384f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,191 @@ +# CLAUDE.md + +Guidance for Claude (and anyone else) working in this repository. + +## What this repository is + +App Dev For All builds **Code on the Go**, an Android IDE aimed at users with no or limited +internet access +(code: [appdevforall/CodeOnTheGo](https://github.com/appdevforall/CodeOnTheGo)). To support that, +Java/Kotlin/Android API documentation is bundled into the app as a single SQLite file — the +**documentation database** — rather than fetched from the web. + +The documentation database serves two distinct features in the IDE: + +1. **Tooltips (Tier 1/2).** When a user selects a keyword/symbol in the code editor, a dialog + shows short (Tier 1) and detailed (Tier 2) tooltip text if the selection matches an entry in + the DB. This lookup happens elsewhere in the CodeOnTheGo Android code (not in this repo, and + not in `WebServer.kt` — see below). +2. **Content pages (Tier 3).** From a tooltip, the user can click through to a full documentation + page. Those pages (and other static content — HTML, images, PDFs) are served over HTTP by + **`WebServer.kt`** + ([CodeOnTheGo/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt](https://github.com/appdevforall/CodeOnTheGo/blob/stage/app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt)), + which runs inside the app and reads directly from the `Content` (and, as of recently, + `Templates`/`Bookshelf`/`BookCategories`) tables of the same database. + +**This repository (`OfflineDocumentationTools`) is the collection of offline tools that build and +edit that database** — it contains no part of the production Android app itself. + +> **Alex's standing caveat, worth repeating at the top of every session:** nothing in this +> repository is guaranteed to work against the *current* production database. The schema has moved +> forward (in the app / by hand) faster than the tooling in this repo has been updated. See +> "Schema: current vs. what this repo expects" below — that gap is the most important thing to +> understand before making changes here. + +## Schema: current vs. what this repo expects + +The schema below is what `~/documentation.db` (Alex's current production copy) actually contains, +as of 2026-08-05: + +```sql +CREATE TABLE Languages (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE); +CREATE TABLE ContentTypes (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL UNIQUE, compression TEXT NOT NULL); +CREATE TABLE TooltipCategories (id INTEGER PRIMARY KEY, category TEXT NOT NULL); +CREATE TABLE TooltipButtonNumbers (id INTEGER UNIQUE); -- manually assigned display order +CREATE TABLE Content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT NOT NULL, languageID INTEGER NOT NULL, + content BLOB NOT NULL, contentTypeID INTEGER NOT NULL, templateId INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY (languageID) REFERENCES Languages(id), FOREIGN KEY (contentTypeID) REFERENCES ContentTypes(id), + UNIQUE('path') +); +CREATE TABLE Tooltips ( + id INTEGER PRIMARY KEY AUTOINCREMENT, categoryId INTEGER NOT NULL, tag TEXT NOT NULL, + summary TEXT NOT NULL, detail TEXT NOT NULL, UNIQUE (categoryId, tag), + FOREIGN KEY(categoryId) REFERENCES TooltipCategories(id) +); +CREATE TABLE TooltipButtons ( + tooltipId INTEGER, buttonNumberId INTEGER, description TEXT, uri TEXT, + FOREIGN KEY(tooltipId) REFERENCES Tooltips(id), FOREIGN KEY(buttonNumberId) REFERENCES TooltipButtonNumbers(id) +); +CREATE TABLE LastChange (documentationSet TEXT, changeTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP, who TEXT); +CREATE TABLE Templates (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, content BLOB NOT NULL, UNIQUE('name')); +CREATE TABLE BookCategories (id INTEGER PRIMARY KEY AUTOINCREMENT, category STRING, description STRING DEFAULT '', UNIQUE('category')); +CREATE TABLE Bookshelf (contentID INTEGER NOT NULL, title STRING DEFAULT '', description STRING DEFAULT '', + bookCategoryID INTEGER, FOREIGN KEY (bookCategoryID) REFERENCES BookCategories(id), UNIQUE(title, bookCategoryId)); +-- Triggers keep Bookshelf in sync when a .pdf row is added to/removed from Content. +CREATE TABLE PUCC_Students (...), PUCC_Classes (...), PUCC_Sections (...), PUCC_Professors (...), + PUCC_StudentAssignments (...), PUCC_ProfessorAssignments (...) +-- Unrelated to documentation tooling (confirmed by Alex) — ignore, leave as-is, do not +-- document or maintain further in this repo. +``` + +**`Templates`, `BookCategories`, and `Bookshelf` are not documentation cruft — `WebServer.kt` +actively depends on them.** Its `/pr/bs` endpoint builds a JSON "bookshelf" payload straight from +`Content` + `Bookshelf` + `BookCategories`, looks up a template named `'bookshelf'` in `Templates`, +and renders it with the Pebble template engine. More generally, any `Content` row with a non-zero +`templateId` gets its stored (decompressed) content run through the matching row in `Templates` as +a Pebble template before being served. This is a real, current feature of the shipped server, not +a placeholder. + +**Nothing that currently builds or writes to the database in this repository knows about any of +that — and that's expected.** `Templates`/`Bookshelf`/`BookCategories` are populated by a separate +plugin system, not by anything in this repo: App Dev For All supports plugins that write into the +documentation database, including the bookshelf feature specifically — +[appdevforall/bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin). So the absence +of any `Templates`/`Bookshelf`/`BookCategories` handling here is not a gap to fill; it's out of +scope for this repo. (A repo-wide search for `Templates`, `Bookshelf`, `BookCategories`, or `PUCC` +turns up zero matches outside `WebServer.kt` itself, which is consistent with that division of +responsibility. `templateId` itself is a different story - `populate_db.py` and +`insert_optimized_media.py` both read/write it directly, since it's a plain column on `Content` +they populate; it's only the `Templates` table and the plugin system that reference it that stay +out of scope.) Concretely, relative to the schema above: + +| Piece | What it thinks the schema is | Consequence | +| --- | --- | --- | +| `scripts/DocumentationDatabase.py` (used by `scripts/ingest.py`, and hence by `.github/workflows/publish-doc-db.yaml`) | `Content` / `Languages` / `ContentTypes` only, plus an optional `ide_tooltip_table`. Its constructor explicitly **raises `ValueError`** if it opens a DB containing any table outside that whitelist. | **This will refuse to open the current production `documentation.db` at all** — it will list `Tooltips`, `TooltipCategories`, `TooltipButtons`, `TooltipButtonNumbers`, `LastChange`, `Templates`, `BookCategories`, `Bookshelf`, and every `PUCC_*` table as "unexpected." This is the single biggest blocker to reusing this script as-is. | +| `docdb-studio/SCHEMA.md` / `AGENTS.md` (states the schema is "locked," no migrations) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, `LastChange` (with a *different* shape: `documentationSet`/`changeTime`/`who` — this part does match current), plus a legacy `ide_tooltip_table`. Missing `templateId`, `Templates`, `BookCategories`, `Bookshelf`, `PUCC_*`. | Closest of the three documented schemas to reality, but still out of date. `docdb_studio.py`'s own "never change the schema" policy is itself now stale, since the live schema has already changed underneath it. | +| `check-tools/README.md`'s embedded schema (and by extension the mental model behind `check-tools/db_health_checker.py`) | `Content` (no `templateId`, no `UNIQUE(path)`), `Tooltips`, `TooltipButtons`, `TooltipCategories`, `TooltipButtonNumbers`, and a *third* variant of `LastChange` (`now`/`who`). No `Templates`/`Bookshelf`/`BookCategories`/`PUCC_*`. | The health checker's required-table check still passes (it only checks that its known tables exist, not that no others do). Since `Templates`/`Bookshelf`/`BookCategories` are out of scope for this repo (see above), this is not being treated as something to fix right now. | + +There also appear to be **two unrelated tooltip storage formats** in this repo's history, and it's +worth being deliberate about which one is current: + +- The **normalized** format (`Tooltips` + `TooltipCategories` + `TooltipButtons` + + `TooltipButtonNumbers`) — this is what's in the live schema above, what `docdb-studio` edits, + what `check-tools/db_health_checker.py` validates, and what `scripts/TooltipManager.py` + dumps/rebuilds via CSV. +- A **legacy flat** format, a single `ide_tooltip_table(tooltipCategory, tooltipTag, + tooltipSummary, tooltipDetail, tooltipButtons)` table (button data packed as a JSON string in + one column) — written by `scripts/tooltips.py` (`TooltipDatabase`, driven by + `scripts/import_tooltips.py` from `SourceDocs/Tooltips/tooltips.xlsx`) and by + `scripts/load_android_data.py` (fed by pickle files that `scripts/android_tooltips.py` / + `scripts/java_tooltips.py` scrape from Android/Java HTML doc trees). **`ide_tooltip_table` does + not exist in the current production schema at all.** + +**`ide_tooltip_table` is officially dead (confirmed by Alex).** That means the entire chain that +targets it — `scripts/tooltips.py`, `scripts/import_tooltips.py`, `scripts/android_tooltips.py`, +`scripts/java_tooltips.py`, `scripts/android_html_page.py`, and `scripts/load_android_data.py` — is +**deprecated legacy code**. It's left in the repo for reference/history, but none of it should be +extended or relied on, and none of it writes to a table the shipped app or `docdb-studio` actually +uses. Any future Android/Java tooltip work should target the normalized `Tooltips` / +`TooltipCategories` / `TooltipButtons` / `TooltipButtonNumbers` tables instead (the same ones +`docdb-studio` and `scripts/TooltipManager.py` already use for Kotlin tooltips). + +## Repository tour + +- **`docdb-studio/`** — a Flet (Flutter-for-Python) desktop GUI for browsing/editing `Tooltips` / + `TooltipCategories` / `TooltipButtons` and importing `Content`. Has its own `CLAUDE.md`, + `AGENTS.md`, `SCHEMA.md`, and a real pytest suite. Actively maintained (most recent commits in + the repo touch this tool), but per the table above, its documented schema is behind the live one. + That's an accepted state, not an active problem: schema evolution happens outside + `docdb-studio` (and outside this repo, e.g. via plugins — see below), and `docdb-studio` is + expected to catch up after the fact rather than lead. Its `AGENTS.md`/`SCHEMA.md` "never migrate + the schema" language should be read as "don't migrate it from in here," not as a claim that the + schema never changes. +- **`check-tools/`** — `db_health_checker.py` (schema/integrity/referential checks against the + *old* normalized schema) plus `download_database.py`, a working Google Drive downloader + authenticated via GCP Workload Identity Federation (no long-lived keys). Wired into + `.github/workflows/docdb-regression-test.yaml`, which runs it daily against the production DB on + Drive. +- **`scripts/`** — the original CLI toolbox. Live/current: `DocumentationDatabase.py` (Content + ingestion — see whitelist issue above), `ingest.py` (thin CLI over it, used by + `publish-doc-db.yaml`), `TooltipManager.py` (CSV ⇄ normalized-Tooltips round-trip), + `create_empty_database.py`, `list_database_documents.py`. **Deprecated/dead** (target the + removed `ide_tooltip_table` — see above, kept for reference only): `tooltips.py`, + `import_tooltips.py`, `android_tooltips.py`, `java_tooltips.py`, `android_html_page.py`, + `load_android_data.py`. +- **`scripts/myServer.py`** — a minimal Python `http.server` reference implementation that predates + `WebServer.kt`. It queries a differently-cased `Documentation.db`, doesn't implement Brotli + decompression (there's a literal `TODO: Replace this function with Brotli decompression`), and + knows nothing about compression-aware content types, templates, or fragmentation. **This is not + what ships in the app** — treat it as historical/reference only, not as documentation of current + server behavior. `WebServer.kt` is the real thing. +- **`Dokka-plugin-kdoc2json/`** — the Dokka `JsonRenderer`/`ModelMapper`/`LinkPostProcessor` plugin, + its test suite, and the `kotlin-stdlib-docs` build scripts, merged to `main` via `fix/ADFA-4514` + (`4c6b8aef`). Consumed by `scripts/kotlin/build-stdlib-json-docs.sh` and + `scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py` (ADFA-4739) to generate and load + kotlin-stdlib/-reflect/-test JSON docs. +- **`ProcessDocs/`** — HTML-processing pipelines that predate the "build docs as JSON" goal: + `ProcessKotlinDocs/` (turns Kotlin's HTML doc export into a self-contained HTML set + table of + contents, used by `.github/workflows/automate-kotlin.yaml`), `ProcessAndroidDevSite/`, `AndroidDocs/` + (holds `android-tooltips.pkl`, the pickle consumed by the now-deprecated `load_android_data.py`), + `ProcessPDFs/`. +- **`SourceDocs/`** — raw inputs: `KotlinDocs/html`, `JavaDocs/html` + `java_keywords.html`, + `Tooltips/tooltips.xlsx`, `KotlinDocs/kotlin-spec.pdf`. +- **`DocumentationAnalysis/`, `DocAnalysis/`, `png_optimization/`, `androidxtooltips/`** — Jupyter + notebooks and one-off scripts for doc-set size analysis, image/PNG compression experiments, and a + one-time AndroidX tooltip import (ADFA-1419). Not part of the critical build path. +- **`.github/workflows/`** — three workflows: `automate-kotlin.yaml` (tag-triggered, builds the + Kotlin HTML doc bundle as a GitHub release asset), `publish-doc-db.yaml` (tag-triggered, runs the + `scripts/ingest.py` pipeline and releases the resulting `.sqlite`), `docdb-regression-test.yaml` + (daily cron, downloads the production DB from Google Drive via WIF and runs + `check-tools/main.py` against it). None of these have any Slack integration yet. + +## Decisions log + +Settled with Alex on 2026-08-05, folded into the sections above; recorded here so the reasoning +isn't lost: + +- `ide_tooltip_table` and everything that targets it are dead. Treat as deprecated, not as a gap. +- `Templates`/`Bookshelf`/`BookCategories` are populated by App Dev For All's plugin system + (e.g. [bookshelf-plugin](https://github.com/appdevforall/bookshelf-plugin)), not by this repo. + Not a gap to fill here. +- `PUCC_*` tables are unrelated to documentation tooling. Ignore; leave as-is. +- `docdb-studio`'s schema is expected to lag the live schema and catch up after the fact; that's + fine, no urgent update needed. +- `check-tools/db_health_checker.py` is not being extended with `Templates`/`Bookshelf` checks + right now — deliberately out of scope for the moment. + +The one piece of this document that still describes an *active* problem rather than a settled +scope boundary is `scripts/DocumentationDatabase.py`'s hard failure on unrecognized tables (see the +table above) — that will need to be addressed before `scripts/ingest.py` / +`publish-doc-db.yaml` can run against a current-schema database. diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh new file mode 100755 index 000000000..8d160825d --- /dev/null +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Builds the kotlin-stdlib/kotlin-test/kotlin-reflect API docs as JSON via the +# kdoc-to-json Dokka plugin, against a full kotlin/ (https://github.com/JetBrains/kotlin) +# repo checkout - freshly compiling and publishing the plugin from source +# first, so every run picks up whatever's currently in +# Dokka-plugin-kdoc2json/kdoc-to-json/src, not a jar left over from an +# earlier run. +# +# Only generates the JSON output (dokkaGenerateModuleJson), not the default +# HTML - JSON/latest/all-libs is the only thing this project's pipeline +# (sync_kdoc_json_to_db.py) consumes. Use build-kotlin-stdlib.sh directly, +# against libraries/tools/kotlin-stdlib-docs, if you also want the HTML +# comparison output that test_kotlin_stdlib.sh checks against. +# +# The target kotlin-stdlib-docs project's build.gradle.kts is swapped out +# for this directory's own (JSON-plugin-enabled) copy for the duration of +# the build, then restored automatically on exit - the kotlin checkout is +# left exactly as it was found, whether the build succeeds or fails. +# +# Only the final output path is written to stdout; every other message goes +# to stderr, so this composes as: +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh )" +set -euo pipefail + +log() { echo "$@" >&2; } + +if [ $# -lt 1 ]; then + log "Usage: $0 [output-dir]" + exit 1 +fi + +KOTLIN_ROOT="$(cd "$1" && pwd)" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/../../kdoc-to-json" && pwd)" +STDLIB_DOCS_DIR="$KOTLIN_ROOT/libraries/tools/kotlin-stdlib-docs" +OUTPUT_ROOT="$(mkdir -p "${2:-$SCRIPT_DIR/build-output}" && cd "${2:-$SCRIPT_DIR/build-output}" && pwd)" +JSON_OUTPUT_DIR="$OUTPUT_ROOT/json" + +if [ ! -f "$KOTLIN_ROOT/gradle.properties" ]; then + log "error: '$KOTLIN_ROOT' doesn't look like a kotlin repo checkout (missing gradle.properties)." + exit 1 +fi +if [ ! -f "$STDLIB_DOCS_DIR/settings.gradle.kts" ] || [ ! -x "$STDLIB_DOCS_DIR/gradlew" ]; then + log "error: '$STDLIB_DOCS_DIR' doesn't look like a kotlin-stdlib-docs project (missing settings.gradle.kts or gradlew)." + exit 1 +fi +if [ ! -x "$PLUGIN_DIR/gradlew" ]; then + log "error: kdoc-to-json plugin project not found at '$PLUGIN_DIR' (missing gradlew)." + exit 1 +fi + +# The JSON-plugin-enabled build.gradle.kts we're about to install reads +# dokka_version as a plain Gradle project property (-Pdokka_version=...) +# rather than through this repo's own version catalog, so it has to be +# supplied explicitly - pulled from the same catalog entry the rest of the +# kotlin repo's Dokka usage is pinned to, so it never drifts out of sync. +DOKKA_VERSION="$(grep -m1 '^dokka[[:space:]]*=' "$KOTLIN_ROOT/gradle/libs.versions.toml" | sed -E 's/^dokka[[:space:]]*=[[:space:]]*"([^"]*)".*/\1/')" +if [ -z "$DOKKA_VERSION" ]; then + log "error: couldn't find a 'dokka = \"...\"' entry in $KOTLIN_ROOT/gradle/libs.versions.toml" + exit 1 +fi + +log "==> [1/2] Building and publishing a fresh copy of the kdoc-to-json plugin..." +# Sent to stderr (fd 2), not left on stdout - a caller doing +# STDLIB_ALL_LIBS="$(build-stdlib-json-docs.sh ...)" must only capture the +# final path this script echoes, not gradlew's own build console output. +( cd "$PLUGIN_DIR" && ./gradlew clean publishToMavenLocal ) >&2 + +log "==> Installing kdoc-to-json-enabled build.gradle.kts into $STDLIB_DOCS_DIR" +ORIGINAL_BUILD_GRADLE="$(mktemp)" +cp "$STDLIB_DOCS_DIR/build.gradle.kts" "$ORIGINAL_BUILD_GRADLE" +restore_build_gradle() { + cp "$ORIGINAL_BUILD_GRADLE" "$STDLIB_DOCS_DIR/build.gradle.kts" + rm -f "$ORIGINAL_BUILD_GRADLE" +} +trap restore_build_gradle EXIT +cp "$SCRIPT_DIR/build.gradle.kts" "$STDLIB_DOCS_DIR/build.gradle.kts" + +log "==> [2/2] Generating JSON documentation via kdoc-to-json (dokka $DOKKA_VERSION)..." +# --refresh-dependencies forces Gradle to re-resolve the just-published +# SNAPSHOT jar from mavenLocal() rather than serving a same-GAV copy it +# cached from an earlier run of this same script. +( cd "$STDLIB_DOCS_DIR" && ./gradlew dokkaGenerateModuleJson \ + "-PdocsBuildDir=$JSON_OUTPUT_DIR" \ + "-Pdokka_version=$DOKKA_VERSION" \ + --refresh-dependencies ) >&2 + +ALL_LIBS_DIR="$JSON_OUTPUT_DIR/latest/all-libs" +if [ ! -d "$ALL_LIBS_DIR" ]; then + log "error: expected output at '$ALL_LIBS_DIR' but it wasn't created." + exit 1 +fi + +log "==> Done." +echo "$ALL_LIBS_DIR" diff --git a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts index db4c72983..48f68c3f2 100644 --- a/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts +++ b/Dokka-plugin-kdoc2json/scripts/kotlin/build.gradle.kts @@ -46,7 +46,14 @@ allprojects { // 3. Maven Central (Keep this for standard standard stable libraries like Gson/Coroutines) mavenCentral() - // ALL REMOTE JETBRAINS SNAPSHOT SERVERS HAVE BEEN REMOVED! + // 4. Dokka's own dev-snapshot server - required by plugins:dokka-samples-transformer-plugin + // and plugins:dokka-version-filter-plugin (both included by kotlin-stdlib-docs' + // settings.gradle.kts and pulled onto the build graph by its dokka-convention plugin), + // which pin to a Dokka dev build rather than a Maven Central release. Same property + + // default kotlin-stdlib-docs' own settings.gradle.kts uses, so this only ever points + // wherever that project already expects it to. + maven(url = providers.gradleProperty("dokka_repository") + .getOrElse("https://redirector.kotlinlang.org/maven/dokka-dev")) } // --- ADDED THIS EXCLUSION BLOCK --- diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..f3482d224 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,134 @@ +# Process Kotlin Website JSON + +Scripts for loading converted Kotlin website JSON, its navigation tree, and +its media straight into a `documentation.db`-schema SQLite database. + +The JSON conversion itself (`md_to_json.py`) is a separate ticket/PR +(ADFA-5039); `build_nav.py` and `populate_db.py` below import it directly, +so it needs to already be merged (or otherwise present in this directory) +for anything here to run. + +## Scripts + +| Script | Purpose | +|---|---| +| `md_to_json.py` (ADFA-5039, not in this PR) | Converts every `topics/**/*.md` page into one JSON file. Writes `theme.json` and copies `images/` into the output directory. See that ticket's README for the page JSON schema. | +| [`build_nav.py`](build_nav.py) | Builds `nav.json`/`nav.html` sidebar navigation from `kr.tree`, resolving each `` against `md_to_json.py`'s output. | +| [`find_missing_assets.py`](find_missing_assets.py) | QA pass: reports cross-page links, images, and `` targets in the source tree that don't resolve to anything. Reuses `md_to_json.py`'s own resolution logic, so it flags exactly what would end up broken on the rendered site. | +| [`populate_db.py`](populate_db.py) | The database path: converts the docs tree the same way `md_to_json.py` does, builds nav the same way `build_nav.py` does, and inserts pages + nav + images + CSS/JS directly into `documentation.db` (replacing everything under `k/html/` and `assets/`). Supports pruning whole `kr.tree` subtrees via `--blacklisted-element-titles`. | +| [`optimize_media.py`](optimize_media.py) | Standalone media optimizer: downscales/recompresses a directory of images (pngquant, Pillow, Scour/cairosvg for SVG) into a mirrored output directory. | +| [`insert_optimized_media.py`](insert_optimized_media.py) | Runs `optimize_media.py`'s pipeline over a directory of raw media, then replaces the corresponding `k/html/images/*` rows in an existing database, rewriting any page that referenced a renamed file and deleting anything left unreferenced. | + +## Requirements + +- Python 3.10+ and [`uv`](https://docs.astral.sh/uv/getting-started/installation/) — every command below is run as `uv run --with-requirements /requirements.txt + + + + +{# + Recursive block renderer. Macros only see the variables passed to them, so + every block that can nest other blocks (blockquote, note/tip/warning, list + items, table cells, tabs) passes its children back through renderBlock(). + Macros defined in a template are directly visible to themselves and to each + other within that same template, so no self-import is needed for recursion. +#} +{% macro renderBlock(b) %} +{% if b.type == "heading" %} +{{ b.html|raw }} + +{% elseif b.type == "paragraph" %} +

{{ b.html|raw }}

+ +{% elseif b.type == "code" %} +
{{ b.code }}
+ +{% elseif b.type == "blockquote" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "note" or b.type == "tip" or b.type == "warning" %} +
+{% if b.attrs.title %}

{{ b.attrs.title }}

{% endif %} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %}
+ +{% elseif b.type == "list" %} +{% if b.ordered %}
    {% else %}
      {% endif %} +{% for item in b.items %}
    • {% for child in item.blocks %}{{ renderBlock(child) }}{% endfor %}
      • +{% endfor %}{% if b.ordered %}
{% else %}{% endif %} + +{% elseif b.type == "table" %} + +{% if b.headers is not empty %} +{% for h in b.headers %}{% endfor %} +{% endif %} + +{% for row in b.rows %}{% for cell in row %}{% endfor %} +{% endfor %} +
{{ h|raw }}
{{ cell|raw }}
+ +{% elseif b.type == "image" %} +{{ b.alt|default('') }} + +{% elseif b.type == "hr" %} +
+ +{% elseif b.type == "tabs" and b.tabs is not empty %} +{# + Tab switching + the group-key syncing (e.g. picking "Groovy" in one + Kotlin/Groovy/Maven tabs block switches every other tabs block sharing the + same data-group on the page, matching Writerside's data-sync-tabs + behavior) is implemented in assets/tabs.js. md_to_json.py's + _finalize_container always gives every "tabs" block a non-empty "tabs" + list (synthesizing one from code-block languages, or dropping the wrapper + entirely, when the source had no children) - the "b.tabs is not + empty" guard here is just a defensive backstop against any other producer + of this JSON schema making the same mistake, not something this pipeline + itself still needs. +#} +
+
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %} + {% endfor %}
+ {% for tab in b.tabs %}{% set tabKey = tab.attrs["group-key"]|default(tab.title)|default(loop.index) %}
+ {% for child in tab.blocks %}{{ renderBlock(child) }} + {% endfor %}
+ {% endfor %} +
+ +{% elseif b.type == "html" %} +{{ b.html|raw }} + +{% elseif b.type == "tab" %} +{# A lone not wrapped in (e.g. seen in eap.json's HTML-table + compatibility layout); render its children rather than dropping them. #} +{% for child in b.blocks %}{{ renderBlock(child) }} +{% endfor %} + +{% else %} +{% if b.html %}{{ b.html|raw }}{% endif %} + +{% endif %} +{% endmacro %} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py new file mode 100644 index 000000000..ea0e51679 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_find_missing_assets.py @@ -0,0 +1,93 @@ +"""Regression test for find_missing_assets.py's exit-code behavior (PR #24 +review). find_missing_assets.py imports md_to_json.py, which doesn't exist +on this branch yet (it lands with ADFA-5039) - so this runs the script as a +subprocess against a minimal stand-in md_to_json module on PYTHONPATH +instead of importing the real one. +""" +import os +import subprocess +import sys +from pathlib import Path + +STUB_MD_TO_JSON = ''' +class Converter: + def __init__(self, md, variables, topic_index=None, image_index=None): + self.warnings = [] + + def convert_file(self, path, page_id, source_rel): + if path.read_text(encoding="utf-8").strip() == "FAIL": + raise ValueError(f"stub failure for {path}") + return {"id": page_id, "sourceFile": source_rel, "blocks": []} + + +def build_topic_index(topics_dir): + return {} + + +def build_image_index(images_dir): + return {}, [] + + +def load_variables(docs_root): + return {} + + +def make_markdown_it(): + return None +''' + + +def _write_stub_md_to_json(tmp_path): + stub_dir = tmp_path / "stub" + stub_dir.mkdir() + (stub_dir / "md_to_json.py").write_text(STUB_MD_TO_JSON, encoding="utf-8") + return stub_dir + + +def _write_minimal_docs_root(tmp_path, *, with_failure=False): + docs_root = tmp_path / "docs" + (docs_root / "topics").mkdir(parents=True) + (docs_root / "topics" / "good.md").write_text("# Good\n\nHello.\n", encoding="utf-8") + if with_failure: + (docs_root / "topics" / "bad.md").write_text("FAIL", encoding="utf-8") + return docs_root + + +def _run(stub_dir, *args): + script = Path(__file__).resolve().parent.parent / "find_missing_assets.py" + env = dict(os.environ) + existing = env.get("PYTHONPATH") + env["PYTHONPATH"] = f"{stub_dir}{os.pathsep}{existing}" if existing else str(stub_dir) + return subprocess.run([sys.executable, str(script), *map(str, args)], capture_output=True, text=True, env=env) + + +def test_exits_zero_and_reports_zero_failures_when_nothing_fails(tmp_path): + stub_dir = _write_stub_md_to_json(tmp_path) + docs_root = _write_minimal_docs_root(tmp_path) + report = tmp_path / "report.md" + result = _run(stub_dir, docs_root, report) + assert result.returncode == 0 + assert "0 file(s) failed to scan" in report.read_text(encoding="utf-8") + + +def test_exits_nonzero_when_a_file_fails_to_scan(tmp_path): + """A per-file scan failure used to be printed to stderr and otherwise + ignored - the report still claimed a clean summary and the process + still exited 0, so a totally broken corpus was indistinguishable from a + clean one (this is the pre-flight gate run before populate_db.py).""" + stub_dir = _write_stub_md_to_json(tmp_path) + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(stub_dir, docs_root, report) + assert result.returncode == 1 + text = report.read_text(encoding="utf-8") + assert "1 file(s) failed to scan" in text + assert "incomplete" in text.lower() + + +def test_allow_failures_exits_zero_despite_failure(tmp_path): + stub_dir = _write_stub_md_to_json(tmp_path) + docs_root = _write_minimal_docs_root(tmp_path, with_failure=True) + report = tmp_path / "report.md" + result = _run(stub_dir, docs_root, report, "--allow-failures") + assert result.returncode == 0 diff --git a/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh b/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh new file mode 100755 index 000000000..7c688a734 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/run_e2e_pipeline_test.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# End-to-end test of the Kotlin website JSON/DB pipeline (ADFA-4737): convert +# docs + prune blacklist into the database, re-optimize/reinsert media, +# generate fresh kotlin-stdlib/-reflect/-test JSON docs via a freshly-built +# kdoc-to-json plugin, then sync them into the database. Operates on a +# scratch copy of documentation.db so the real database is never touched. +# Re-run freely; each run recopies the source db from scratch. +# +# Before running: fill in every value below for your machine. +# The script refuses to start if any are left unfilled or don't exist on disk. +set -euo pipefail + +if ! command -v uv >/dev/null 2>&1; then + echo "error: uv is required - see https://docs.astral.sh/uv/getting-started/installation/" >&2 + exit 1 +fi + +# --- Repo-relative paths - auto-detected, no edits needed --------------- +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROCESS_DIR="$REPO_ROOT/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON" +SYNC_SCRIPT="$REPO_ROOT/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py" +UV_RUN=(uv run --with-requirements "$REPO_ROOT/requirements.txt") + +# config.json, templates/, and assets/ are staged directly in $PROCESS_DIR +# (this repo) rather than pulled from anyone's local machine - populate_db.py +# looks these up next to its own script location: +# config.json - theming config (broken-ext-link-color, menu-no-link-color) +# templates/*.peb - page.peb / nav.peb, upserted into the Templates table +# assets/* - docs.css / tabs.js / sidebar.js, inserted at assets/ +CONFIG_JSON="$PROCESS_DIR/config.json" + +# --- Machine-specific paths - fill these in -------------------------------- + +# DOCS_ROOT: the "docs" subdirectory of a Writerside checkout of the official +# Kotlin website. Get it from https://github.com/JetBrains/kotlin-web-site - +# clone that repo and point this at "/kotlin-web-site/docs" (the +# directory directly containing kr.tree, topics/, images/, v.list). +DOCS_ROOT="" + +# IMAGES_ZIP: Writerside's own image export for that same docs project (e.g. +# "webHelpImages.zip"). Produced by running IntelliJ IDEA's Writerside plugin +# build/export action against DOCS_ROOT's parent Writerside project; the zip +# is written next to kr.tree once that build finishes. +IMAGES_ZIP="" + +# STDLIB_DOCS_DIR: the "libraries/tools/kotlin-stdlib-docs" directory inside +# a full clone of https://github.com/JetBrains/kotlin (not the kotlin repo +# root itself - this exact subdirectory). Step 4 below builds a fresh copy +# of the kdoc-to-json plugin from this repo's Dokka-plugin-kdoc2json/ and +# runs it against this checkout to produce kotlin-stdlib/-reflect/-test JSON +# docs (common + jvm source sets only) - no separate manual doc-generation +# step needed. +STDLIB_DOCS_DIR="" + +# SOURCE_DB: the runtime "documentation.db" SQLite database this project's +# offline documentation app/server reads from (see docdb-studio/ and +# check-tools/ in this repo for tooling that operates on the same file). Must +# already have its schema populated (Languages, ContentTypes, Templates +# tables) - point this at your own working copy. +SOURCE_DB="" + +TEST_DB="$(dirname "$SOURCE_DB")/documentation.test.db" + +JPEG_QUALITY=85 +WEBP_QUALITY=90 + +# Full toc-title path (top-level -> ... -> target), joined with "\/" per +# populate_db.py's --blacklisted-element-titles convention. These are the +# concrete cases named in ADFA-4737; add more "path" entries here to prune +# additional sections. Re-derive these from your own DOCS_ROOT/kr.tree if the +# site's navigation structure has changed since this was written. +BLACKLIST=( + 'Development\/Web development' + 'Interoperability\/Swift/Objective-C and C interop' + 'Interoperability\/JavaScript interop' +) + +# --- Fail fast on unfilled placeholders or missing paths ------------------- +require_path() { + local name="$1" value="$2" + if [[ "$value" == "<"*">" ]]; then + echo "error: $name is still a placeholder ('$value') - edit this script and fill in your local path." >&2 + exit 1 + fi + if [[ ! -e "$value" ]]; then + echo "error: $name points to '$value', which does not exist." >&2 + exit 1 + fi +} +require_path DOCS_ROOT "$DOCS_ROOT" +require_path IMAGES_ZIP "$IMAGES_ZIP" +require_path STDLIB_DOCS_DIR "$STDLIB_DOCS_DIR" +require_path SOURCE_DB "$SOURCE_DB" + +WORKDIR="$(mktemp -d /tmp/adfa4737-e2e.XXXXXX)" +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +echo "== Copying $SOURCE_DB -> $TEST_DB ==" +rm -f "$TEST_DB" +cp "$SOURCE_DB" "$TEST_DB" + +echo +echo "== Step 1/5: find_missing_assets.py (source QA report) ==" +REPORT_PATH="$WORKDIR/missing-assets-report.md" +"${UV_RUN[@]}" "$PROCESS_DIR/find_missing_assets.py" "$DOCS_ROOT" "$REPORT_PATH" +echo "Report written to $REPORT_PATH" + +echo +echo "== Step 2/5: populate_db.py (convert docs, prune blacklist, insert into test db) ==" +( cd "$PROCESS_DIR" && "${UV_RUN[@]}" populate_db.py "$DOCS_ROOT" "$CONFIG_JSON" "$IMAGES_ZIP" "$TEST_DB" \ + --blacklisted-element-titles "${BLACKLIST[@]}" ) + +echo +echo "== Step 3/5: insert_optimized_media.py (re-optimize + reinsert k/html/images/*) ==" +# --webp requires an "image/webp" ContentTypes row, which this database +# doesn't ship with (see insert_optimized_media.py's own module docstring) - +# add it (idempotent) before running. +sqlite3 "$TEST_DB" "INSERT OR IGNORE INTO ContentTypes (value, compression) VALUES ('image/webp', 'brotli');" + +# insert_optimized_media.py addresses images by bare filename, matching +# populate_db.py's own flat k/html/images/ convention - so its input +# media_dir needs to be a directory of files with those same basenames. +# The images actually inserted above came from IMAGES_ZIP, so extract that +# same zip here rather than pointing at DOCS_ROOT/images (the raw, unoptimized +# Writerside source tree - a different, much larger set of files). +MEDIA_DIR="$WORKDIR/media" +mkdir -p "$MEDIA_DIR" +unzip -q "$IMAGES_ZIP" -d "$MEDIA_DIR" + +"${UV_RUN[@]}" "$PROCESS_DIR/insert_optimized_media.py" "$MEDIA_DIR" "$TEST_DB" \ + --jpeg-quality "$JPEG_QUALITY" --webp --webp-quality "$WEBP_QUALITY" --verbose + +echo +echo "== Step 4/5: build-stdlib-json-docs.sh (fresh plugin build -> kotlin-stdlib/-reflect/-test JSON) ==" +# STDLIB_DOCS_DIR is .../kotlin/libraries/tools/kotlin-stdlib-docs; the +# kotlin repo root (needed to locate gradle/libs.versions.toml and to +# resolve kotlin_root inside the injected build.gradle.kts) is exactly three +# levels up, matching that build.gradle.kts's own "../../../" convention. +KOTLIN_ROOT="$(cd "$STDLIB_DOCS_DIR/../../.." && pwd)" +STDLIB_ALL_LIBS="$("$REPO_ROOT/Dokka-plugin-kdoc2json/scripts/kotlin/build-stdlib-json-docs.sh" \ + "$KOTLIN_ROOT" "$WORKDIR/stdlib-json")" +echo "Generated JSON docs at $STDLIB_ALL_LIBS" + +echo +echo "== Step 5/5: sync_kdoc_json_to_db.py (overwrite kotlin-stdlib/-reflect/-test content) ==" +"${UV_RUN[@]}" "$SYNC_SCRIPT" "$STDLIB_ALL_LIBS" --db "$TEST_DB" + +echo +echo "== Summary ==" +"${UV_RUN[@]}" python3 - "$TEST_DB" <<'PYEOF' +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) + + +def count(where, params=()): + return conn.execute(f"SELECT count(*) FROM Content WHERE {where}", params).fetchone()[0] + + +print(f"Database: {db_path}") +print(f" k/html/* rows: {count('path LIKE ?', ('k/html/%',))}") +print(f" k/html/images/* rows: {count('path LIKE ?', ('k/html/images/%',))}") +print(f" k/html/images/*.webp rows: {count('path LIKE ?', ('k/html/images/%.webp%',))}") +print(f" k/kotlin-stdlib/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-stdlib/%', 'k/kotlin-stdlib'))}") +print(f" k/kotlin-reflect/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-reflect/%', 'k/kotlin-reflect'))}") +print(f" k/kotlin-test/* rows: {count('path LIKE ? OR path = ?', ('k/kotlin-test/%', 'k/kotlin-test'))}") + +conn.close() +PYEOF + +echo +echo "== Blacklist pruning verification ==" +# Recomputes, from the same kr.tree and BLACKLIST used above, exactly which +# topic stems populate_db.py's own prune_blacklisted_elements() decided to +# exclude - then confirms none of those pages made it into the database. +# Reusing that real pruning logic (rather than guessing at path patterns) +# means this check stays correct if BLACKLIST or kr.tree's structure change. +"${UV_RUN[@]}" python3 - "$PROCESS_DIR" "$DOCS_ROOT" "$TEST_DB" "${BLACKLIST[@]}" <<'PYEOF' +import sqlite3 +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +process_dir, docs_root, db_path, *blacklist_raw = sys.argv[1:] +sys.path.insert(0, process_dir) +import populate_db # noqa: E402 + +root = ET.parse(Path(docs_root) / "kr.tree").getroot() +blacklisted_paths = {populate_db.parse_blacklist_path(raw) for raw in blacklist_raw} +blacklisted_stems, unmatched_paths = populate_db.prune_blacklisted_elements(root, blacklisted_paths) + +conn = sqlite3.connect(db_path) +leftover = [] +for stem in sorted(blacklisted_stems): + path = f"k/html/{stem}.html" + if conn.execute("SELECT 1 FROM Content WHERE path = ?", (path,)).fetchone(): + leftover.append(path) +conn.close() + +print(f"Blacklisted toc-element path(s) checked: {len(blacklisted_paths)}") +for path in sorted(blacklisted_paths): + status = "unmatched (no such element in kr.tree)" if path in unmatched_paths else "matched" + print(f" {' > '.join(path)}: {status}") +print(f"Topic page(s) expected removed: {len(blacklisted_stems)}") + +if unmatched_paths: + print(f"FAIL: {len(unmatched_paths)} blacklist path(s) never matched a - " + "check BLACKLIST against this DOCS_ROOT's kr.tree.") + sys.exit(1) +if leftover: + print(f"FAIL: {len(leftover)} blacklisted page(s) still present in the database:") + for path in leftover: + print(f" {path}") + sys.exit(1) + +print(f"PASS: all {len(blacklisted_stems)} blacklisted topic page(s) confirmed absent from {db_path}.") +PYEOF + +echo +echo "Done. Backups (populate_db.py, insert_optimized_media.py, and sync_kdoc_json_to_db.py" +echo "each make their own) live alongside $TEST_DB as documentation.test.db.backup-* and" +echo "documentation.test.db.bak.*" +echo "The real database at $SOURCE_DB was never opened for writing." diff --git a/requirements.txt b/requirements.txt index 265d3c394..031f31439 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ brotli Pillow openpyxl>=3.1.0 tqdm-loggable>=0.1.0 +scour +cairosvg diff --git a/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py b/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py new file mode 100755 index 000000000..fb6149260 --- /dev/null +++ b/scripts/sync_kotlin_stdlib_docs/sync_kdoc_json_to_db.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Overwrites the kotlin-stdlib / kotlin-reflect / kotlin-test Content rows in +documentation.db with fresh output from the KDoc-to-JSON Dokka plugin. + +For every existing Content row whose path starts with "k/kotlin-stdlib", +"k/kotlin-reflect", or "k/kotlin-test": + - Compute the corresponding file in the plugin output tree: strip the "k/" + prefix, and if the path ends in ".html", swap that for ".json" (paths with + no extension, e.g. ".../package-list", are looked up unchanged). + - If that file exists, re-compress it (matching the row's existing + ContentTypes.compression) and overwrite the row's `content` blob only -- + `path`, `languageID`, `contentTypeID`, and `templateId` are left untouched. + - If it doesn't exist, delete the row. + +Any TooltipButtons row whose `uri` (ignoring a trailing "#fragment") matches one +of the deleted Content paths is now a dead link. Its entire parent Tooltips +record -- along with all of that tooltip's other TooltipButtons rows, dead or +not -- is deleted too, since TooltipButtons has no ON DELETE CASCADE and a +dangling tooltipId would otherwise be left behind. + +A timestamped backup of the database is made before anything is modified. +""" +import argparse +import os +import shutil +import sqlite3 +import sys +from datetime import datetime, timezone + +import brotli + +PREFIXES = ["k/kotlin-stdlib", "k/kotlin-reflect", "k/kotlin-test"] + + +def backup_database(db_path): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup_path = f"{db_path}.bak.{timestamp}" + shutil.copy2(db_path, backup_path) + return backup_path + + +def relative_target_path(content_path): + """'k/kotlin-stdlib/kotlin.text/index.html' -> 'kotlin-stdlib/kotlin.text/index.json' + 'k/kotlin-stdlib/package-list' -> 'kotlin-stdlib/package-list' (no extension to swap)""" + without_prefix = content_path[len("k/"):] + if without_prefix.endswith(".html"): + return without_prefix[: -len(".html")] + ".json" + return without_prefix + + +def compress_for(compression, raw_bytes, path): + if compression == "brotli": + return brotli.compress(raw_bytes) + if compression == "none": + return raw_bytes + raise ValueError(f"Unknown compression '{compression}' needed for {path}") + + +def cleanup_orphaned_tooltips(cur, deleted_paths, dry_run): + """Delete any Tooltips (and all their TooltipButtons) that reference a + now-deleted Content path via a TooltipButtons.uri. Returns (tooltips_removed, + buttons_removed).""" + if not deleted_paths: + return 0, 0 + + deleted_path_set = set(deleted_paths) + + where_clause = " OR ".join(["uri = ? OR uri LIKE ?"] * len(PREFIXES)) + params = [] + for prefix in PREFIXES: + params.extend([prefix, prefix + "/%"]) + + candidate_buttons = cur.execute( + f"SELECT tooltipId, uri FROM TooltipButtons WHERE {where_clause}", params + ).fetchall() + + orphaned_tooltip_ids = sorted( + {tooltip_id for tooltip_id, uri in candidate_buttons if uri.split("#", 1)[0] in deleted_path_set} + ) + if not orphaned_tooltip_ids: + return 0, 0 + + placeholders = ",".join("?" * len(orphaned_tooltip_ids)) + buttons_count = cur.execute( + f"SELECT count(*) FROM TooltipButtons WHERE tooltipId IN ({placeholders})", orphaned_tooltip_ids + ).fetchone()[0] + + if dry_run: + for tooltip_id in orphaned_tooltip_ids: + print(f" [DELETE TOOLTIP] id={tooltip_id}") + else: + cur.execute(f"DELETE FROM TooltipButtons WHERE tooltipId IN ({placeholders})", orphaned_tooltip_ids) + cur.execute(f"DELETE FROM Tooltips WHERE id IN ({placeholders})", orphaned_tooltip_ids) + + return len(orphaned_tooltip_ids), buttons_count + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "plugin_output_root", + help="Root dir directly containing kotlin-stdlib/, kotlin-reflect/, kotlin-test/ " + "(e.g. .../all-libs from a KDoc-to-JSON run)", + ) + parser.add_argument("--db", default="documentation.db", help="Path to documentation.db (default: documentation.db in the current directory)") + parser.add_argument("--dry-run", action="store_true", help="Report what would happen without modifying anything") + args = parser.parse_args() + + if not os.path.isdir(args.plugin_output_root): + print(f"Error: '{args.plugin_output_root}' is not a directory.", file=sys.stderr) + sys.exit(2) + if not os.path.isfile(args.db): + print(f"Error: database '{args.db}' not found.", file=sys.stderr) + sys.exit(2) + + if args.dry_run: + print("Dry run: no backup will be made and no changes will be written.") + else: + backup_path = backup_database(args.db) + print(f"Backed up database to: {backup_path}") + + conn = sqlite3.connect(args.db) + cur = conn.cursor() + + compression_by_type = dict(cur.execute("SELECT id, compression FROM ContentTypes")) + + where_clause = " OR ".join(["path = ? OR path LIKE ?"] * len(PREFIXES)) + params = [] + for prefix in PREFIXES: + params.extend([prefix, prefix + "/%"]) + + rows = cur.execute( + f"SELECT id, path, contentTypeID FROM Content WHERE {where_clause}", params + ).fetchall() + + print(f"Found {len(rows)} existing Content record(s) under {PREFIXES}.") + + updated = 0 + deleted = 0 + deleted_paths = [] + unknown_types = set() + + try: + conn.execute("BEGIN") + for content_id, path, content_type_id in rows: + rel_target = relative_target_path(path) + source_file = os.path.join(args.plugin_output_root, rel_target) + + if os.path.isfile(source_file): + with open(source_file, "rb") as f: + raw_bytes = f.read() + + compression = compression_by_type.get(content_type_id) + if compression is None: + unknown_types.add(content_type_id) + compression = "none" + + new_blob = compress_for(compression, raw_bytes, path) + + if args.dry_run: + print(f" [UPDATE] {path} <- {rel_target}") + else: + cur.execute("UPDATE Content SET content = ? WHERE id = ?", (new_blob, content_id)) + updated += 1 + else: + if args.dry_run: + print(f" [DELETE] {path} (no matching {rel_target})") + else: + cur.execute("DELETE FROM Content WHERE id = ?", (content_id,)) + deleted += 1 + deleted_paths.append(path) + + tooltips_removed, buttons_removed = cleanup_orphaned_tooltips(cur, deleted_paths, args.dry_run) + + if unknown_types: + print( + f"WARNING: contentTypeID(s) {sorted(unknown_types)} not found in ContentTypes; " + "treated as uncompressed.", + file=sys.stderr, + ) + + if args.dry_run: + conn.rollback() + print( + f"\nDry run complete: would update {updated}, delete {deleted} Content record(s); " + f"would delete {tooltips_removed} Tooltips record(s) ({buttons_removed} TooltipButtons " + "row(s)). No changes made." + ) + else: + conn.commit() + print( + f"\nDone: updated {updated}, deleted {deleted} Content record(s); " + f"deleted {tooltips_removed} Tooltips record(s) ({buttons_removed} TooltipButtons row(s))." + ) + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +if __name__ == "__main__": + main()