ADFA-5094: catalog freshness service (weekly publish + on-device refresh)

.github/workflows/publish-catalogs.yml

@@ -0,0 +1,86 @@
+name: Publish content catalogs
+
+# ADFA-5094 (ADR-5094): generate the Kolibri/Kiwix catalogs once a week off the build and
+# publish them + a versioned manifest to Cloudflare R2, so the app can refresh the catalog
+# without an APK release. Reuses the R2 pattern the release build uses for update.json
+# (android-release-build.yml): aws s3 cp with the CLOUDFLARE_* vars, fixed keys that overwrite
+# so a URL always resolves to the latest. Content stays on the Nginx mirror; only the catalog
+# metadata is published here, served from k2go-download.appdevforall.org/catalogs/.
+on:
+  schedule:
+    # Weekly, Monday 06:17 UTC. Off-peak, odd minute (GitHub throttles round-hour crons).
+    - cron: '17 6 * * 1'
+  workflow_dispatch: {}
+
+jobs:
+  publish-catalogs:
+    name: Generate catalogs & upload to Cloudflare R2
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./controller/app
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      # The generators query Studio/Kiwix once, here, on a runner with bandwidth — not from
+      # every release build (ADR-4954 D1). Never fail the job: a blocked fetch keeps the
+      # committed asset, and we simply skip its upload below.
+      - name: Generate catalogs
+        run: |
+          python3 tools/build_kolibri_catalog.py || echo "::warning::kolibri generator failed; its upload will be skipped"
+          python3 tools/build_kiwix_catalog.py   || echo "::warning::kiwix generator failed; its upload will be skipped"
+
+      - name: Build manifests and upload to Cloudflare R2
+        env:
+          AWS_ACCESS_KEY_ID: ${{ vars.CLOUDFLARE_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          R2_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+          BUCKET_NAME: "iiaboa-apk-repo"
+          BASE_URL: "https://k2go-download.appdevforall.org/catalogs"
+        run: |
+          set -euo pipefail
+          ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
+          GENERATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+          VERSION="$(date -u +%Y.%m.%d)"
+
+          # publish <name> <file> <content-type> <items-mode>
+          #   items-mode "jsonl" -> extract per-channel {id,version} for later per-item delta /
+          #   the "a newer version exists" signal; "none" -> file-level hash only (e.g. Kiwix CSV).
+          publish() {
+            name="$1"; file="$2"; ctype="$3"; items_mode="$4"
+            if [ ! -f "$file" ]; then
+              echo "::warning::$file not found; skipping $name (kept whatever is already published)"
+              return 0
+            fi
+            base="$(basename "$file")"
+            hash="sha256:$(sha256sum "$file" | cut -d' ' -f1)"
+            items='[]'
+            if [ "$items_mode" = "jsonl" ]; then
+              items="$(grep -v '"catalog"' "$file" | jq -c '{id, version}' | jq -sc '.')"
+            fi
+            jq -n \
+              --arg catalog "$name" --arg version "$VERSION" --arg generated "$GENERATED" \
+              --arg hash "$hash" --arg url "$BASE_URL/$base" --argjson items "$items" \
+              '{catalog:$catalog, version:$version, generated:$generated, hash:$hash, url:$url, items:$items}' \
+              > "$name.manifest.json"
+            echo "----- $name.manifest.json -----"; cat "$name.manifest.json"
+
+            # Upload the catalog first, then the manifest, so the manifest never points at a
+            # missing file. Fixed keys -> overwrite -> the URL always resolves to the latest.
+            aws s3 cp "$file" "s3://$BUCKET_NAME/catalogs/$base" \
+              --endpoint-url "$ENDPOINT" --content-type "$ctype"
+            aws s3 cp "$name.manifest.json" "s3://$BUCKET_NAME/catalogs/$name.manifest.json" \
+              --endpoint-url "$ENDPOINT" --content-type "application/json"
+          }
+
+          publish kolibri src/main/assets/kolibri_catalog.jsonl "application/x-ndjson" jsonl
+          publish kiwix   src/main/assets/kiwix_catalog.csv     "text/csv"             none

controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogManifestClient.java

@@ -0,0 +1,124 @@
+/*
+ * ============================================================================
+ * Name        : CatalogManifestClient.java
+ * Author      : AppDevForAll
+ * Copyright   : Copyright (c) 2026 AppDevForAll
+ * Description : ADFA-5094 (ADR-5094). Fetches a catalog manifest with a
+ *               conditional GET (ETag / If-None-Match -> 304) and downloads the
+ *               catalog body, mirroring the OTA update client's HTTP. Blocking;
+ *               call from an IO thread. Never throws.
+ * ============================================================================
+ */
+package org.iiab.controller.catalog.data;
+
+import android.util.Log;
+
+import org.iiab.controller.catalog.domain.CatalogManifest;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public final class CatalogManifestClient {
+
+    private static final String TAG = "K2Go-Catalog";
+    private static final int CONNECT_TIMEOUT_MS = 5000;
+    private static final int READ_TIMEOUT_MS = 8000;
+
+    public enum Status { OK, NOT_MODIFIED, FAILED }
+
+    /** Outcome of a manifest fetch. {@code manifest}/{@code etag} are set only on {@code OK}. */
+    public static final class Result {
+        public final Status status;
+        public final CatalogManifest manifest;
+        public final String etag;
+
+        private Result(Status status, CatalogManifest manifest, String etag) {
+            this.status = status;
+            this.manifest = manifest;
+            this.etag = etag;
+        }
+
+        static Result ok(CatalogManifest m, String etag) { return new Result(Status.OK, m, etag); }
+        static Result notModified() { return new Result(Status.NOT_MODIFIED, null, null); }
+        static Result failed() { return new Result(Status.FAILED, null, null); }
+    }
+
+    /** GET the manifest; sends {@code If-None-Match} when {@code knownEtag} is set. */
+    public Result fetchManifest(String url, String knownEtag) {
+        HttpURLConnection conn = null;
+        try {
+            conn = (HttpURLConnection) new URL(url).openConnection();
+            conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
+            conn.setReadTimeout(READ_TIMEOUT_MS);
+            conn.setRequestMethod("GET");
+            if (knownEtag != null && !knownEtag.isEmpty()) {
+                conn.setRequestProperty("If-None-Match", knownEtag);
+            }
+            int code = conn.getResponseCode();
+            if (code == HttpURLConnection.HTTP_NOT_MODIFIED) {
+                return Result.notModified();
+            }
+            if (code != HttpURLConnection.HTTP_OK) {
+                Log.w(TAG, "manifest fetch HTTP " + code + " for " + url);
+                return Result.failed();
+            }
+            String etag = conn.getHeaderField("ETag");
+            StringBuilder body = new StringBuilder();
+            try (BufferedReader r = new BufferedReader(new InputStreamReader(conn.getInputStream()))) {
+                String line;
+                while ((line = r.readLine()) != null) {
+                    body.append(line);
+                }
+            }
+            CatalogManifest m = CatalogManifestParser.parse(body.toString());
+            return m == null ? Result.failed() : Result.ok(m, etag);
+        } catch (Throwable t) {
+            Log.w(TAG, "manifest fetch failed: " + t.getMessage());
+            return Result.failed();
+        } finally {
+            if (conn != null) {
+                conn.disconnect();
+            }
+        }
+    }
+
+    /**
+     * Download the catalog body to {@code dest} via a temp file swapped in on success, so a
+     * partial download never replaces a good overlay. Returns true on success.
+     */
+    public boolean downloadTo(String url, File dest) {
+        HttpURLConnection conn = null;
+        File tmp = new File(dest.getAbsolutePath() + ".tmp");
+        try {
+            conn = (HttpURLConnection) new URL(url).openConnection();
+            conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
+            conn.setReadTimeout(READ_TIMEOUT_MS);
+            conn.setRequestMethod("GET");
+            if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
+                return false;
+            }
+            try (InputStream in = conn.getInputStream();
+                 FileOutputStream out = new FileOutputStream(tmp)) {
+                byte[] buf = new byte[8192];
+                int n;
+                while ((n = in.read(buf)) != -1) {
+                    out.write(buf, 0, n);
+                }
+            }
+            return tmp.renameTo(dest);
+        } catch (Throwable t) {
+            Log.w(TAG, "catalog download failed: " + t.getMessage());
+            tmp.delete();
+            return false;
+        } finally {
+            if (conn != null) {
+                conn.disconnect();
+            }
+        }
+    }
+}

controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogManifestParser.java

@@ -0,0 +1,58 @@
+/*
+ * ============================================================================
+ * Name        : CatalogManifestParser.java
+ * Author      : AppDevForAll
+ * Copyright   : Copyright (c) 2026 AppDevForAll
+ * Description : ADFA-5094 (ADR-5094). Parses the manifest JSON the weekly
+ *               workflow publishes into a CatalogManifest. Never throws — a
+ *               malformed manifest yields null, and the caller keeps the current
+ *               catalog.
+ * ============================================================================
+ */
+package org.iiab.controller.catalog.data;
+
+import org.iiab.controller.catalog.domain.CatalogManifest;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class CatalogManifestParser {
+
+    private CatalogManifestParser() {
+    }
+
+    /** @return the parsed manifest, or null if the input is missing or malformed. */
+    public static CatalogManifest parse(String json) {
+        if (json == null || json.isEmpty()) {
+            return null;
+        }
+        try {
+            JSONObject o = new JSONObject(json);
+            List<CatalogManifest.Item> items = new ArrayList<>();
+            JSONArray arr = o.optJSONArray("items");
+            if (arr != null) {
+                for (int i = 0; i < arr.length(); i++) {
+                    JSONObject it = arr.optJSONObject(i);
+                    if (it == null) {
+                        continue;
+                    }
+                    String id = it.optString("id", "");
+                    if (!id.isEmpty()) {
+                        items.add(new CatalogManifest.Item(id, it.optInt("version", 0)));
+                    }
+                }
+            }
+            return new CatalogManifest(
+                    o.optString("catalog", ""),
+                    o.optString("version", ""),
+                    o.optString("generated", ""),
+                    o.optString("hash", ""),
+                    o.optString("url", ""),
+                    items);
+        } catch (Throwable t) {
+            return null;
+        }
+    }
+}

controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogOverlay.java

@@ -0,0 +1,38 @@
+/*
+ * ============================================================================
+ * Name        : CatalogOverlay.java
+ * Author      : AppDevForAll
+ * Copyright   : Copyright (c) 2026 AppDevForAll
+ * Description : ADFA-5094 (ADR-5094). Where a pulled catalog lands on the
+ *               device. The refresh worker writes the overlay here and the
+ *               catalog source reads it (when present and newer) in place of the
+ *               APK-bundled asset. Same path convention on both sides.
+ * ============================================================================
+ */
+package org.iiab.controller.catalog.data;
+
+import android.content.Context;
+
+import java.io.File;
+
+public final class CatalogOverlay {
+
+    private static final String DIR = "catalogs";
+
+    private CatalogOverlay() {
+    }
+
+    /** {@code filesDir/catalogs}, created if missing. */
+    public static File dir(Context ctx) {
+        File d = new File(ctx.getApplicationContext().getFilesDir(), DIR);
+        if (!d.exists()) {
+            d.mkdirs();
+        }
+        return d;
+    }
+
+    /** The overlay file for a catalog whose bundled asset is named {@code basename}. */
+    public static File file(Context ctx, String basename) {
+        return new File(dir(ctx), basename);
+    }
+}

controller/app/src/main/java/org/iiab/controller/catalog/data/CatalogRefreshScheduler.java

@@ -0,0 +1,69 @@
+/*
+ * ============================================================================
+ * Name        : CatalogRefreshScheduler.java
+ * Author      : AppDevForAll
+ * Copyright   : Copyright (c) 2026 AppDevForAll
+ * Description : ADFA-5094 (ADR-5094). Enqueues the catalog refresh: a weekly,
+ *               network-constrained periodic job, plus an opportunistic one-shot
+ *               (e.g. when the picker opens). The worker's own TTL gate keeps the
+ *               one-shot cheap. Unique per catalog, KEEP so relaunches don't
+ *               reset the schedule.
+ * ============================================================================
+ */
+package org.iiab.controller.catalog.data;
+
+import android.content.Context;
+
+import androidx.work.Constraints;
+import androidx.work.Data;
+import androidx.work.ExistingPeriodicWorkPolicy;
+import androidx.work.ExistingWorkPolicy;
+import androidx.work.NetworkType;
+import androidx.work.OneTimeWorkRequest;
+import androidx.work.PeriodicWorkRequest;
+import androidx.work.WorkManager;
+
+import java.util.concurrent.TimeUnit;
+
+public final class CatalogRefreshScheduler {
+
+    private CatalogRefreshScheduler() {
+    }
+
+    private static Data input(String name, String manifestUrl, String basename) {
+        return new Data.Builder()
+                .putString(CatalogRefreshWorker.KEY_NAME, name)
+                .putString(CatalogRefreshWorker.KEY_MANIFEST_URL, manifestUrl)
+                .putString(CatalogRefreshWorker.KEY_BASENAME, basename)
+                .build();
+    }
+
+    private static Constraints connected() {
+        return new Constraints.Builder()
+                .setRequiredNetworkType(NetworkType.CONNECTED)
+                .build();
+    }
+
+    /** Weekly periodic refresh, enqueued once per catalog (KEEP). Safe to call on every launch. */
+    public static void scheduleWeekly(Context ctx, String name, String manifestUrl, String basename) {
+        PeriodicWorkRequest req = new PeriodicWorkRequest.Builder(
+                CatalogRefreshWorker.class, 7, TimeUnit.DAYS)
+                .setConstraints(connected())
+                .setInputData(input(name, manifestUrl, basename))
+                .build();
+        WorkManager.getInstance(ctx.getApplicationContext())
+                .enqueueUniquePeriodicWork("catalog-refresh-" + name,
+                        ExistingPeriodicWorkPolicy.KEEP, req);
+    }
+
+    /** Opportunistic one-shot; the worker's TTL gate no-ops it when still fresh. */
+    public static void refreshNow(Context ctx, String name, String manifestUrl, String basename) {
+        OneTimeWorkRequest req = new OneTimeWorkRequest.Builder(CatalogRefreshWorker.class)
+                .setConstraints(connected())
+                .setInputData(input(name, manifestUrl, basename))
+                .build();
+        WorkManager.getInstance(ctx.getApplicationContext())
+                .enqueueUniqueWork("catalog-refresh-now-" + name,
+                        ExistingWorkPolicy.KEEP, req);
+    }
+}