From 3bfd03d04e58545be691e59d66f4a147229baf26 Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sat, 22 Aug 2026 14:49:44 +1000 Subject: [PATCH 1/8] Initial uv and jinja DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 24 +++++++++++----- .../workflows/scripts/schedule_deployments.py | 28 +++++++++++++++++-- .../workflows/scripts/templates/cr.yaml.jinja | 2 ++ 3 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/scripts/templates/cr.yaml.jinja diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 68d68a9..413eea5 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -38,6 +38,16 @@ jobs: with: fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: "0.12.5" + enable-cache: false + + - name: Set up uv Python + run: | + uv python install 3.13 + - name: Verify inputs id: verify-inputs env: @@ -87,13 +97,13 @@ jobs: path: ${{ secrets.REPO_NAME_GITOPS }} token: ${{ steps.app-token.outputs.token }} - - name: Append JSONL to file + - name: Generate data files env: ACTION: ${{ inputs.action }} SCHEDULE: ${{ steps.verify-inputs.outputs.schedule }} DIR_OUTPUT: ${{ secrets.REPO_NAME_GITOPS }} JOB_ID: ${{ github.run_number }} - run: python3 .github/workflows/scripts/schedule_deployments.py -l info + run: uv run .github/workflows/scripts/schedule_deployments.py -l info - name: Show recent schedule run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/schedule-deployments.jsonl @@ -104,8 +114,8 @@ jobs: ls git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" - git add . - git status - git commit -m "Apply schedule deployments" - echo "Do git push ..." - git push +# git add . +# git status +# git commit -m "Apply schedule deployments" +# echo "Do git push ..." +# git push diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 7c5cbdb..453bdab 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -6,15 +6,27 @@ NOTE: Please use 'black' to re-format code. """ +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "jinja2", +# ] +# [tool.uv] +# exclude-newer = "7 days" +# /// + import argparse from datetime import datetime, timezone import json import logging import os from pathlib import Path +import pprint # pylint: disable=unused-import import sys -SCRIPT_VERSION = "1.4.0" +from jinja2 import Environment, FileSystemLoader + +SCRIPT_VERSION = "1.5.0" LOGLEVELS = { "debug": logging.DEBUG, @@ -69,12 +81,16 @@ def get_options(): except KeyError: LOGGER.error("Missing env: JOB_ID") options_okay = False + templates_pn = PROG_PATH.joinpath("templates") + if not templates_pn.exists(): + LOGGER.error("The jinja templates '%s' not found.", templates_pn) + options_okay = False schedule_pn = PROG_PATH.parent.parent.parent.joinpath( dir_output, "schedule-deployments.jsonl" ) if not options_okay: sys.exit(2) - return int(job_id), action, schedule, schedule_pn + return int(job_id), action, schedule, templates_pn, schedule_pn def append_schedule(job_id, action, schedule, schedule_pn): @@ -98,9 +114,15 @@ def main(): """ Append the schedule JSONL to schedule-deployments.jsonl file. """ - job_id, action, schedule, schedule_pn = get_options() + job_id, action, schedule, templates_pn, schedule_pn = get_options() LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn) append_schedule(job_id, action, schedule, schedule_pn) + env_jinja = Environment(loader=FileSystemLoader(templates_pn)) + template_cr = env_jinja.get_template("cr.yaml.jinja") + content_cr = template_cr.render( + mytext="FooBar", + ) + pprint.pprint(content_cr) if __name__ == "__main__": diff --git a/.github/workflows/scripts/templates/cr.yaml.jinja b/.github/workflows/scripts/templates/cr.yaml.jinja new file mode 100644 index 0000000..2290673 --- /dev/null +++ b/.github/workflows/scripts/templates/cr.yaml.jinja @@ -0,0 +1,2 @@ +baz: + metadata: {{mytext}} From 4f8aea8fc5afb793aec47af7c31ee5d70a2b1969 Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sat, 22 Aug 2026 15:44:31 +1000 Subject: [PATCH 2/8] Pass full sha to script, compose poolId DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 6 +++--- .../workflows/scripts/schedule_deployments.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 413eea5..0d3807f 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -64,8 +64,8 @@ jobs: errors+=("The matcher '${matcher}' is not found.") continue fi - sha_short=$(git rev-parse HEAD | cut -c 1-7) - schedules+=("${matcher}:${sha_short}") + sha=$(git rev-parse HEAD) + schedules+=("${matcher}:${sha}") done if [ ${#errors[@]} -gt 0 ]; then printf "ERROR: %s\n" "${errors[@]}" @@ -90,7 +90,7 @@ jobs: repositories: ${{ secrets.REPO_NAME_GITOPS }} permission-contents: write - - name: Checkout GitOps Repository + - name: Checkout GitOps repository uses: actions/checkout@v7 with: repository: indexdata/${{ secrets.REPO_NAME_GITOPS }} diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 453bdab..268c0d3 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -93,7 +93,7 @@ def get_options(): return int(job_id), action, schedule, templates_pn, schedule_pn -def append_schedule(job_id, action, schedule, schedule_pn): +def append_schedule(job_id, action, id_pool, schedule_pn): """ Composes the JSONL and appends to file. """ @@ -104,7 +104,7 @@ def append_schedule(job_id, action, schedule, schedule_pn): ) json_packet["action"] = action json_packet["initialized"] = False - json_packet["deployment"] = schedule + json_packet["poolId"] = id_pool with open(schedule_pn, mode="a", encoding="utf-8") as output_fh: output_fh.write(json.dumps(json_packet, sort_keys=False, indent=None)) output_fh.write("\n") @@ -115,8 +115,16 @@ def main(): Append the schedule JSONL to schedule-deployments.jsonl file. """ job_id, action, schedule, templates_pn, schedule_pn = get_options() - LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn) - append_schedule(job_id, action, schedule, schedule_pn) + # LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn) + deployments = schedule.split(",") + matchers = [] + id_pool = "" + for deployment in deployments: + matcher, sha = deployment.split(":") + id_matcher = f"{matcher}~{sha[0:7]}" + matchers.append(id_matcher) + id_pool = "_".join(matchers) + append_schedule(job_id, action, id_pool, schedule_pn) env_jinja = Environment(loader=FileSystemLoader(templates_pn)) template_cr = env_jinja.get_template("cr.yaml.jinja") content_cr = template_cr.render( From 201338bec54bccd0377603b79e2af718a9cfd0ab Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 09:06:10 +1000 Subject: [PATCH 3/8] Ensure namespace dir exists, store log/schedule DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 4 ++-- .../workflows/scripts/schedule_deployments.py | 22 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 0d3807f..8e6e110 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -106,12 +106,12 @@ jobs: run: uv run .github/workflows/scripts/schedule_deployments.py -l info - name: Show recent schedule - run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/schedule-deployments.jsonl + run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/log/schedule-deployments.jsonl - name: Commit git changes and push run: | cd ${{ secrets.REPO_NAME_GITOPS }} - ls + ls -R git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" # git add . diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 268c0d3..033bee0 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -85,18 +85,24 @@ def get_options(): if not templates_pn.exists(): LOGGER.error("The jinja templates '%s' not found.", templates_pn) options_okay = False - schedule_pn = PROG_PATH.parent.parent.parent.joinpath( - dir_output, "schedule-deployments.jsonl" - ) + dir_namespace = "minitex-matchers" + dir_storage = PROG_PATH.parent.parent.parent.joinpath(dir_output, dir_namespace) + if not dir_storage.exists(): + LOGGER.error("The namespace directory '%s' not found.", dir_namespace) + options_okay = False if not options_okay: sys.exit(2) - return int(job_id), action, schedule, templates_pn, schedule_pn + return int(job_id), action, schedule, templates_pn, dir_storage -def append_schedule(job_id, action, id_pool, schedule_pn): +def append_schedule(job_id, action, id_pool, dir_storage): """ Composes the JSONL and appends to file. """ + dir_log = dir_storage.joinpath("log") + os.makedirs(dir_log, exist_ok=True) + schedule_pn = dir_log.joinpath("schedule-deployments.jsonl") + LOGGER.debug("schedule_pn=%s", schedule_pn) json_packet = {} json_packet["id"] = job_id json_packet["scheduleDate"] = ( @@ -114,8 +120,8 @@ def main(): """ Append the schedule JSONL to schedule-deployments.jsonl file. """ - job_id, action, schedule, templates_pn, schedule_pn = get_options() - # LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn) + job_id, action, schedule, templates_pn, dir_storage = get_options() + LOGGER.debug("schedule=%s", schedule) deployments = schedule.split(",") matchers = [] id_pool = "" @@ -124,7 +130,7 @@ def main(): id_matcher = f"{matcher}~{sha[0:7]}" matchers.append(id_matcher) id_pool = "_".join(matchers) - append_schedule(job_id, action, id_pool, schedule_pn) + append_schedule(job_id, action, id_pool, dir_storage) env_jinja = Environment(loader=FileSystemLoader(templates_pn)) template_cr = env_jinja.get_template("cr.yaml.jinja") content_cr = template_cr.render( From 695b99b822005e9567b12a4c999c546ba2cb5e53 Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 09:17:36 +1000 Subject: [PATCH 4/8] Move step show recent schedule DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 8e6e110..7913786 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -105,17 +105,17 @@ jobs: JOB_ID: ${{ github.run_number }} run: uv run .github/workflows/scripts/schedule_deployments.py -l info - - name: Show recent schedule - run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/log/schedule-deployments.jsonl - - name: Commit git changes and push run: | cd ${{ secrets.REPO_NAME_GITOPS }} ls -R git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" -# git add . -# git status -# git commit -m "Apply schedule deployments" -# echo "Do git push ..." -# git push + git add . + git status + git commit -m "Apply schedule deployments" + echo "Do git push ..." + git push + + - name: Show recent schedule + run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/log/schedule-deployments.jsonl From 6d45f3470c7ee39150d7276e2f89c427f822242a Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 09:31:35 +1000 Subject: [PATCH 5/8] Fix pathname recent schedule DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 7913786..0ce56c2 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -108,14 +108,14 @@ jobs: - name: Commit git changes and push run: | cd ${{ secrets.REPO_NAME_GITOPS }} - ls -R git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" - git add . git status - git commit -m "Apply schedule deployments" - echo "Do git push ..." - git push +# git add . +# git status +# git commit -m "Apply schedule deployments" +# echo "Do git push ..." +# git push - name: Show recent schedule - run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/log/schedule-deployments.jsonl + run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/minitex-matchers/log/schedule-deployments.jsonl From 190cd75c57f198b8870e6d5b0c8b48683c97c30a Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 11:33:36 +1000 Subject: [PATCH 6/8] Add file matchers-summary.json DEVOPS-8235 Initially used just to declare path to the implementation because some are not guessable. --- .../workflows/scripts/schedule_deployments.py | 57 ++++++++++++++++++- js/README.md | 1 + matchers-summary.json | 42 ++++++++++++++ 3 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 matchers-summary.json diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 033bee0..08c574e 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -26,7 +26,7 @@ from jinja2 import Environment, FileSystemLoader -SCRIPT_VERSION = "1.5.0" +SCRIPT_VERSION = "1.5.1" LOGLEVELS = { "debug": logging.DEBUG, @@ -95,6 +95,57 @@ def get_options(): return int(job_id), action, schedule, templates_pn, dir_storage +def load_matchers_summary(): + """ + Loads the summary of all matchers. + """ + input_fn = "matchers-summary.json" + input_pn = PROG_PATH.parent.parent.parent.joinpath(input_fn) + LOGGER.debug("%s", input_pn) + with open(input_pn, mode="r", encoding="utf-8") as json_fh: + try: + summary = json.load(json_fh) + except json.decoder.JSONDecodeError as err: + msg = f"Trouble loading '{input_fn}' JSON file: {err.lineno} {err.msg}" + LOGGER.critical(msg) + sys.exit(1) + return summary + + +def get_matcher_script_path(matchers_summary, matcher): + """ + Get the script pathname for this matcher. + """ + script_pn = None + input_fn = "matchers-summary.json" + if not any(dictionary.get("name") == matcher for dictionary in matchers_summary): + msg = f"Matcher '{matcher}' not found in '{input_fn}' file." + LOGGER.critical(msg) + sys.exit(1) + else: + matcher_details = [d for d in matchers_summary if d["name"] == matcher] + try: + matcher_details[0]["script"] + except KeyError: + msg = ( + "The 'script' property is not found for " + f"matcher '{matcher}' in '{input_fn}' file." + ) + LOGGER.critical(msg) + sys.exit(1) + script_fn = matcher_details[0]["script"] + script_pn = PROG_PATH.parent.parent.parent.joinpath(script_fn) + if not script_pn.exists(): + msg = ( + f"The script '{script_fn}' declared for " + f"matcher '{matcher}' in '{input_fn}' file " + "does not exist." + ) + LOGGER.critical(msg) + sys.exit(1) + return script_pn + + def append_schedule(job_id, action, id_pool, dir_storage): """ Composes the JSONL and appends to file. @@ -102,7 +153,6 @@ def append_schedule(job_id, action, id_pool, dir_storage): dir_log = dir_storage.joinpath("log") os.makedirs(dir_log, exist_ok=True) schedule_pn = dir_log.joinpath("schedule-deployments.jsonl") - LOGGER.debug("schedule_pn=%s", schedule_pn) json_packet = {} json_packet["id"] = job_id json_packet["scheduleDate"] = ( @@ -122,11 +172,12 @@ def main(): """ job_id, action, schedule, templates_pn, dir_storage = get_options() LOGGER.debug("schedule=%s", schedule) + matchers_summary = load_matchers_summary() deployments = schedule.split(",") matchers = [] - id_pool = "" for deployment in deployments: matcher, sha = deployment.split(":") + matcher_pn = get_matcher_script_path(matchers_summary, matcher) id_matcher = f"{matcher}~{sha[0:7]}" matchers.append(id_matcher) id_pool = "_".join(matchers) diff --git a/js/README.md b/js/README.md index 3efa0d9..8475a6e 100644 --- a/js/README.md +++ b/js/README.md @@ -117,6 +117,7 @@ Each matcher is briefly explained in the [js/matchers/README.md](matchers/README For each matcher there is an entry in the [js/package.json](package.json) file to declare its test to be run using Node.js (e.g. `test-goldrush2024`). To add a new matcher, follow the structure of an existing matcher. +Also declare it at the file [matchers-summary.json](../matchers-summary.json). > [!IMPORTANT] > The matcher names are restricted to alpha-numeric or hyphen (dash) characters. diff --git a/matchers-summary.json b/matchers-summary.json new file mode 100644 index 0000000..19c093f --- /dev/null +++ b/matchers-summary.json @@ -0,0 +1,42 @@ +[ + { + "name": "deepdish", + "type": "javascript", + "script": "js/matchers/deepdish/deepdish.mjs" + }, + { + "name": "goldrush", + "type": "javascript", + "script": "js/matchers/goldrush/goldrush.mjs" + }, + { + "name": "goldrush2024", + "type": "javascript", + "script": "js/matchers/goldrush2024/goldrush.mjs" + }, + { + "name": "isxn", + "type": "javascript", + "script": "js/matchers/isxn/isxn.mjs" + }, + { + "name": "malort", + "type": "javascript", + "script": "js/matchers/malort/malort.mjs" + }, + { + "name": "shareInsts", + "type": "javascript", + "script": "js/matchers/shareInsts/shareInsts.mjs" + }, + { + "name": "shareWorks", + "type": "javascript", + "script": "js/matchers/shareWorks/shareWorks.mjs" + }, + { + "name": "sharevde", + "type": "javascript", + "script": "js/matchers/sharevde/sharevde.mjs" + } +] From fba57baa6133ad608f832fe1b029ad238db1d61e Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 12:18:16 +1000 Subject: [PATCH 7/8] Do assemble_pool_details DEVOPS-8235 --- .../workflows/scripts/schedule_deployments.py | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 08c574e..dd48b6f 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -26,7 +26,7 @@ from jinja2 import Environment, FileSystemLoader -SCRIPT_VERSION = "1.5.1" +SCRIPT_VERSION = "1.5.2" LOGLEVELS = { "debug": logging.DEBUG, @@ -116,7 +116,7 @@ def get_matcher_script_path(matchers_summary, matcher): """ Get the script pathname for this matcher. """ - script_pn = None + script_fn = None input_fn = "matchers-summary.json" if not any(dictionary.get("name") == matcher for dictionary in matchers_summary): msg = f"Matcher '{matcher}' not found in '{input_fn}' file." @@ -143,7 +143,31 @@ def get_matcher_script_path(matchers_summary, matcher): ) LOGGER.critical(msg) sys.exit(1) - return script_pn + return script_fn + + +def assemble_pool_details(schedule, matchers_summary): + """ + Assembles the details of this pool. + """ + LOGGER.debug("schedule=%s", schedule) + deployments = schedule.split(",") + matchers = [] + pool_details = {"matchers": []} + for deployment in deployments: + matcher_packet = {} + matcher, sha = deployment.split(":") + matcher_packet["name"] = matcher + matcher_packet["sha"] = sha + matcher_fn = get_matcher_script_path(matchers_summary, matcher) + matcher_packet["script"] = matcher_fn + id_matcher = f"{matcher}~{sha[0:7]}" + matcher_packet["id"] = id_matcher + matchers.append(id_matcher) + pool_details["matchers"].append(matcher_packet) + id_pool = "_".join(matchers) + pool_details["id_pool"] = id_pool + return id_pool, pool_details def append_schedule(job_id, action, id_pool, dir_storage): @@ -171,16 +195,8 @@ def main(): Append the schedule JSONL to schedule-deployments.jsonl file. """ job_id, action, schedule, templates_pn, dir_storage = get_options() - LOGGER.debug("schedule=%s", schedule) matchers_summary = load_matchers_summary() - deployments = schedule.split(",") - matchers = [] - for deployment in deployments: - matcher, sha = deployment.split(":") - matcher_pn = get_matcher_script_path(matchers_summary, matcher) - id_matcher = f"{matcher}~{sha[0:7]}" - matchers.append(id_matcher) - id_pool = "_".join(matchers) + id_pool, pool_details = assemble_pool_details(schedule, matchers_summary) append_schedule(job_id, action, id_pool, dir_storage) env_jinja = Environment(loader=FileSystemLoader(templates_pn)) template_cr = env_jinja.get_template("cr.yaml.jinja") From 9371d9b508c1410e95f68451a2b22f388ca32226 Mon Sep 17 00:00:00 2001 From: David Crossley Date: Sun, 23 Aug 2026 16:48:42 +1000 Subject: [PATCH 8/8] Generate pool custom resource with jinja DEVOPS-8235 --- .github/workflows/schedule-deployment.yml | 9 ++- .../workflows/scripts/schedule_deployments.py | 57 ++++++++++++++----- .../workflows/scripts/templates/cr.yaml.jinja | 18 +++++- .yamllint | 10 ++++ 4 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 .yamllint diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 0ce56c2..b4b8a01 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -110,12 +110,11 @@ jobs: cd ${{ secrets.REPO_NAME_GITOPS }} git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" + git add . git status -# git add . -# git status -# git commit -m "Apply schedule deployments" -# echo "Do git push ..." -# git push + git commit -m "Generate pool custom resource" + echo "Do git push ..." + git push - name: Show recent schedule run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/minitex-matchers/log/schedule-deployments.jsonl diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index dd48b6f..b8544eb 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -26,7 +26,7 @@ from jinja2 import Environment, FileSystemLoader -SCRIPT_VERSION = "1.5.2" +SCRIPT_VERSION = "1.5.3" LOGLEVELS = { "debug": logging.DEBUG, @@ -101,7 +101,7 @@ def load_matchers_summary(): """ input_fn = "matchers-summary.json" input_pn = PROG_PATH.parent.parent.parent.joinpath(input_fn) - LOGGER.debug("%s", input_pn) + # LOGGER.debug("%s", input_pn) with open(input_pn, mode="r", encoding="utf-8") as json_fh: try: summary = json.load(json_fh) @@ -112,11 +112,12 @@ def load_matchers_summary(): return summary -def get_matcher_script_path(matchers_summary, matcher): +def get_matcher_script(matchers_summary, matcher): """ - Get the script pathname for this matcher. + Get the script pathname and type for this matcher. """ script_fn = None + script_type = None input_fn = "matchers-summary.json" if not any(dictionary.get("name") == matcher for dictionary in matchers_summary): msg = f"Matcher '{matcher}' not found in '{input_fn}' file." @@ -143,33 +144,68 @@ def get_matcher_script_path(matchers_summary, matcher): ) LOGGER.critical(msg) sys.exit(1) - return script_fn + try: + matcher_details[0]["type"] + except KeyError: + msg = ( + "The 'type' property is not found for " + f"matcher '{matcher}' in '{input_fn}' file." + ) + LOGGER.critical(msg) + sys.exit(1) + else: + script_type = matcher_details[0]["type"] + return script_fn, script_type def assemble_pool_details(schedule, matchers_summary): """ Assembles the details of this pool. """ - LOGGER.debug("schedule=%s", schedule) + # LOGGER.debug("schedule=%s", schedule) deployments = schedule.split(",") matchers = [] + pool_matchers = [] pool_details = {"matchers": []} for deployment in deployments: matcher_packet = {} matcher, sha = deployment.split(":") matcher_packet["name"] = matcher matcher_packet["sha"] = sha - matcher_fn = get_matcher_script_path(matchers_summary, matcher) + matcher_fn, matcher_type = get_matcher_script(matchers_summary, matcher) matcher_packet["script"] = matcher_fn + matcher_packet["type"] = matcher_type id_matcher = f"{matcher}~{sha[0:7]}" matcher_packet["id"] = id_matcher matchers.append(id_matcher) + pool_matchers.append(f"{id_matcher}-matcher::matchkey") pool_details["matchers"].append(matcher_packet) id_pool = "_".join(matchers) pool_details["id_pool"] = id_pool + pool_details["pool_matcher"] = ", ".join(pool_matchers) return id_pool, pool_details +def generate_cr(templates_pn, pool_details, dir_storage): + """ + Generates and stores the custom resource YAML. + """ + # pprint.pprint(pool_details) + dir_pools = dir_storage.joinpath("pools") + os.makedirs(dir_pools, exist_ok=True) + pool_pn = dir_pools.joinpath(f"{pool_details['id_pool']}.yaml") + env_jinja = Environment(loader=FileSystemLoader(templates_pn)) + template_cr = env_jinja.get_template("cr.yaml.jinja") + content_cr = template_cr.render( + id_pool=pool_details["id_pool"], + pool_matcher=pool_details["pool_matcher"], + matchers=pool_details["matchers"], + ) + with open(pool_pn, mode="w", encoding="utf-8") as output_fh: + output_fh.write(content_cr) + output_fh.write("\n") + + def append_schedule(job_id, action, id_pool, dir_storage): """ Composes the JSONL and appends to file. @@ -197,13 +233,8 @@ def main(): job_id, action, schedule, templates_pn, dir_storage = get_options() matchers_summary = load_matchers_summary() id_pool, pool_details = assemble_pool_details(schedule, matchers_summary) + generate_cr(templates_pn, pool_details, dir_storage) append_schedule(job_id, action, id_pool, dir_storage) - env_jinja = Environment(loader=FileSystemLoader(templates_pn)) - template_cr = env_jinja.get_template("cr.yaml.jinja") - content_cr = template_cr.render( - mytext="FooBar", - ) - pprint.pprint(content_cr) if __name__ == "__main__": diff --git a/.github/workflows/scripts/templates/cr.yaml.jinja b/.github/workflows/scripts/templates/cr.yaml.jinja index 2290673..a2799fd 100644 --- a/.github/workflows/scripts/templates/cr.yaml.jinja +++ b/.github/workflows/scripts/templates/cr.yaml.jinja @@ -1,2 +1,16 @@ -baz: - metadata: {{mytext}} +apiVersion: indexdata.com/v1alpha1 +kind: Pool +metadata: + name: {{id_pool}} + namespace: minitex-matchers +spec: + pool: + id: {{id_pool}} + matcher: {{pool_matcher}} + update: ingest + codeModules: + {%- for matcher in matchers %} + - id: {{matcher.id}}-matcher + type: {{matcher.type}} + url: https://raw.githubusercontent.com/indexdata/reservoir-scripts/{{matcher.sha}}/{{matcher.script}} + {%- endfor -%} diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..393f406 --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +extends: default + +rules: + comments-indentation: disable + trailing-spaces: enable + new-line-at-end-of-file: enable + line-length: disable + document-start: + present: false