DEVOPS-8235 schedule-deployment: Generate pool custom resource with jinja

.github/workflows/schedule-deployment.yml

@@ -38,6 +38,16 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          version: "0.12.5"
+          enable-cache: false
+
+      - name: Set up uv Python
+        run: |
+          uv python install 3.13
+
       - name: Verify inputs
         id: verify-inputs
         env:
@@ -54,8 +64,8 @@ jobs:
               errors+=("The matcher '${matcher}' is not found.")
               continue
             fi
-            sha_short=$(git rev-parse HEAD | cut -c 1-7)
-            schedules+=("${matcher}:${sha_short}")
+            sha=$(git rev-parse HEAD)
+            schedules+=("${matcher}:${sha}")
           done
           if [ ${#errors[@]} -gt 0 ]; then
             printf "ERROR: %s\n" "${errors[@]}"
@@ -80,32 +90,31 @@ jobs:
           repositories: ${{ secrets.REPO_NAME_GITOPS }}
           permission-contents: write
 
-      - name: Checkout GitOps Repository
+      - name: Checkout GitOps repository
         uses: actions/checkout@v7
         with:
           repository: indexdata/${{ secrets.REPO_NAME_GITOPS }}
           path: ${{ secrets.REPO_NAME_GITOPS }}
           token: ${{ steps.app-token.outputs.token }}
 
-      - name: Append JSONL to file
+      - name: Generate data files
         env:
           ACTION: ${{ inputs.action }}
           SCHEDULE: ${{ steps.verify-inputs.outputs.schedule }}
           DIR_OUTPUT: ${{ secrets.REPO_NAME_GITOPS }}
           JOB_ID: ${{ github.run_number }}
-        run: python3 .github/workflows/scripts/schedule_deployments.py -l info
-
-      - name: Show recent schedule
-        run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/schedule-deployments.jsonl
+        run: uv run .github/workflows/scripts/schedule_deployments.py -l info
 
       - name: Commit git changes and push
         run: |
           cd ${{ secrets.REPO_NAME_GITOPS }}
-          ls
           git config --global user.name "github-actions-reservoir-scripts"
           git config --global user.email "github-actions-reservoir-scripts@indexdata.com"
           git add .
           git status
-          git commit -m "Apply schedule deployments"
+          git commit -m "Generate pool custom resource"
           echo "Do git push ..."
           git push
+
+      - name: Show recent schedule
+        run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/minitex-matchers/log/schedule-deployments.jsonl

.github/workflows/scripts/schedule_deployments.py

@@ -6,15 +6,27 @@
 NOTE: Please use 'black' to re-format code.
 """
 
+# /// script
+# requires-python = ">=3.13"
+# dependencies = [
+#   "jinja2",
+# ]
+# [tool.uv]
+# exclude-newer = "7 days"
+# ///
+
 import argparse
 from datetime import datetime, timezone
 import json
 import logging
 import os
 from pathlib import Path
+import pprint  # pylint: disable=unused-import
 import sys
 
-SCRIPT_VERSION = "1.4.0"
+from jinja2 import Environment, FileSystemLoader
+
+SCRIPT_VERSION = "1.5.3"
 
 LOGLEVELS = {
     "debug": logging.DEBUG,
@@ -69,26 +81,146 @@ def get_options():
     except KeyError:
         LOGGER.error("Missing env: JOB_ID")
         options_okay = False
-    schedule_pn = PROG_PATH.parent.parent.parent.joinpath(
-        dir_output, "schedule-deployments.jsonl"
-    )
+    templates_pn = PROG_PATH.joinpath("templates")
+    if not templates_pn.exists():
+        LOGGER.error("The jinja templates '%s' not found.", templates_pn)
+        options_okay = False
+    dir_namespace = "minitex-matchers"
+    dir_storage = PROG_PATH.parent.parent.parent.joinpath(dir_output, dir_namespace)
+    if not dir_storage.exists():
+        LOGGER.error("The namespace directory '%s' not found.", dir_namespace)
+        options_okay = False
     if not options_okay:
         sys.exit(2)
-    return int(job_id), action, schedule, schedule_pn
+    return int(job_id), action, schedule, templates_pn, dir_storage
+
+
+def load_matchers_summary():
+    """
+    Loads the summary of all matchers.
+    """
+    input_fn = "matchers-summary.json"
+    input_pn = PROG_PATH.parent.parent.parent.joinpath(input_fn)
+    # LOGGER.debug("%s", input_pn)
+    with open(input_pn, mode="r", encoding="utf-8") as json_fh:
+        try:
+            summary = json.load(json_fh)
+        except json.decoder.JSONDecodeError as err:
+            msg = f"Trouble loading '{input_fn}' JSON file: {err.lineno} {err.msg}"
+            LOGGER.critical(msg)
+            sys.exit(1)
+    return summary
+
+
+def get_matcher_script(matchers_summary, matcher):
+    """
+    Get the script pathname and type for this matcher.
+    """
+    script_fn = None
+    script_type = None
+    input_fn = "matchers-summary.json"
+    if not any(dictionary.get("name") == matcher for dictionary in matchers_summary):
+        msg = f"Matcher '{matcher}' not found in '{input_fn}' file."
+        LOGGER.critical(msg)
+        sys.exit(1)
+    else:
+        matcher_details = [d for d in matchers_summary if d["name"] == matcher]
+        try:
+            matcher_details[0]["script"]
+        except KeyError:
+            msg = (
+                "The 'script' property is not found for "
+                f"matcher '{matcher}' in '{input_fn}' file."
+            )
+            LOGGER.critical(msg)
+            sys.exit(1)
+        script_fn = matcher_details[0]["script"]
+        script_pn = PROG_PATH.parent.parent.parent.joinpath(script_fn)
+        if not script_pn.exists():
+            msg = (
+                f"The script '{script_fn}' declared for "
+                f"matcher '{matcher}' in '{input_fn}' file "
+                "does not exist."
+            )
+            LOGGER.critical(msg)
+            sys.exit(1)
+        try:
+            matcher_details[0]["type"]
+        except KeyError:
+            msg = (
+                "The 'type' property is not found for "
+                f"matcher '{matcher}' in '{input_fn}' file."
+            )
+            LOGGER.critical(msg)
+            sys.exit(1)
+        else:
+            script_type = matcher_details[0]["type"]
+    return script_fn, script_type
+
+
+def assemble_pool_details(schedule, matchers_summary):
+    """
+    Assembles the details of this pool.
+    """
+    # LOGGER.debug("schedule=%s", schedule)
+    deployments = schedule.split(",")
+    matchers = []
+    pool_matchers = []
+    pool_details = {"matchers": []}
+    for deployment in deployments:
+        matcher_packet = {}
+        matcher, sha = deployment.split(":")
+        matcher_packet["name"] = matcher
+        matcher_packet["sha"] = sha
+        matcher_fn, matcher_type = get_matcher_script(matchers_summary, matcher)
+        matcher_packet["script"] = matcher_fn
+        matcher_packet["type"] = matcher_type
+        id_matcher = f"{matcher}~{sha[0:7]}"
+        matcher_packet["id"] = id_matcher
+        matchers.append(id_matcher)
+        pool_matchers.append(f"{id_matcher}-matcher::matchkey")
+        pool_details["matchers"].append(matcher_packet)
+    id_pool = "_".join(matchers)
+    pool_details["id_pool"] = id_pool
+    pool_details["pool_matcher"] = ", ".join(pool_matchers)
+    return id_pool, pool_details
+
+
+def generate_cr(templates_pn, pool_details, dir_storage):
+    """
+    Generates and stores the custom resource YAML.
+    """
+    # pprint.pprint(pool_details)
+    dir_pools = dir_storage.joinpath("pools")
+    os.makedirs(dir_pools, exist_ok=True)
+    pool_pn = dir_pools.joinpath(f"{pool_details['id_pool']}.yaml")
+    env_jinja = Environment(loader=FileSystemLoader(templates_pn))
+    template_cr = env_jinja.get_template("cr.yaml.jinja")
+    content_cr = template_cr.render(
+        id_pool=pool_details["id_pool"],
+        pool_matcher=pool_details["pool_matcher"],
+        matchers=pool_details["matchers"],
+    )
+    with open(pool_pn, mode="w", encoding="utf-8") as output_fh:
+        output_fh.write(content_cr)
+        output_fh.write("\n")
 
 
-def append_schedule(job_id, action, schedule, schedule_pn):
+def append_schedule(job_id, action, id_pool, dir_storage):
     """
     Composes the JSONL and appends to file.
     """
+    dir_log = dir_storage.joinpath("log")
+    os.makedirs(dir_log, exist_ok=True)
+    schedule_pn = dir_log.joinpath("schedule-deployments.jsonl")
     json_packet = {}
     json_packet["id"] = job_id
     json_packet["scheduleDate"] = (
         datetime.now(timezone.utc).replace(microsecond=0).isoformat()
     )
     json_packet["action"] = action
     json_packet["initialized"] = False
-    json_packet["deployment"] = schedule
+    json_packet["poolId"] = id_pool
     with open(schedule_pn, mode="a", encoding="utf-8") as output_fh:
         output_fh.write(json.dumps(json_packet, sort_keys=False, indent=None))
         output_fh.write("\n")
@@ -98,9 +230,11 @@ def main():
     """
     Append the schedule JSONL to schedule-deployments.jsonl file.
     """
-    job_id, action, schedule, schedule_pn = get_options()
-    LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn)
-    append_schedule(job_id, action, schedule, schedule_pn)
+    job_id, action, schedule, templates_pn, dir_storage = get_options()
+    matchers_summary = load_matchers_summary()
+    id_pool, pool_details = assemble_pool_details(schedule, matchers_summary)
+    generate_cr(templates_pn, pool_details, dir_storage)
+    append_schedule(job_id, action, id_pool, dir_storage)
 
 
 if __name__ == "__main__":

.github/workflows/scripts/templates/cr.yaml.jinja

@@ -0,0 +1,16 @@
+apiVersion: indexdata.com/v1alpha1
+kind: Pool
+metadata:
+  name: {{id_pool}}
+  namespace: minitex-matchers
+spec:
+  pool:
+    id: {{id_pool}}
+    matcher: {{pool_matcher}}
+    update: ingest
+  codeModules:
+  {%- for matcher in matchers %}
+    - id: {{matcher.id}}-matcher
+      type: {{matcher.type}}
+      url: https://raw.githubusercontent.com/indexdata/reservoir-scripts/{{matcher.sha}}/{{matcher.script}}
+  {%- endfor -%}

.yamllint

@@ -0,0 +1,10 @@
+---
+extends: default
+
+rules:
+  comments-indentation: disable
+  trailing-spaces: enable
+  new-line-at-end-of-file: enable
+  line-length: disable
+  document-start:
+    present: false

js/README.md

@@ -117,6 +117,7 @@ Each matcher is briefly explained in the [js/matchers/README.md](matchers/README
 For each matcher there is an entry in the [js/package.json](package.json) file to declare its test to be run using Node.js (e.g. `test-goldrush2024`).
 
 To add a new matcher, follow the structure of an existing matcher.
+Also declare it at the file [matchers-summary.json](../matchers-summary.json).
 
 > [!IMPORTANT]
 > The matcher names are restricted to alpha-numeric or hyphen (dash) characters.

matchers-summary.json

@@ -0,0 +1,42 @@
+[
+  {
+    "name": "deepdish",
+    "type": "javascript",
+    "script": "js/matchers/deepdish/deepdish.mjs"
+  },
+  {
+    "name": "goldrush",
+    "type": "javascript",
+    "script": "js/matchers/goldrush/goldrush.mjs"
+  },
+  {
+    "name": "goldrush2024",
+    "type": "javascript",
+    "script": "js/matchers/goldrush2024/goldrush.mjs"
+  },
+  {
+    "name": "isxn",
+    "type": "javascript",
+    "script": "js/matchers/isxn/isxn.mjs"
+  },
+  {
+    "name": "malort",
+    "type": "javascript",
+    "script": "js/matchers/malort/malort.mjs"
+  },
+  {
+    "name": "shareInsts",
+    "type": "javascript",
+    "script": "js/matchers/shareInsts/shareInsts.mjs"
+  },
+  {
+    "name": "shareWorks",
+    "type": "javascript",
+    "script": "js/matchers/shareWorks/shareWorks.mjs"
+  },
+  {
+    "name": "sharevde",
+    "type": "javascript",
+    "script": "js/matchers/sharevde/sharevde.mjs"
+  }
+]