diff --git a/.github/workflows/schedule-deployment.yml b/.github/workflows/schedule-deployment.yml index 68d68a9..b4b8a01 100644 --- a/.github/workflows/schedule-deployment.yml +++ b/.github/workflows/schedule-deployment.yml @@ -38,6 +38,16 @@ jobs: with: fetch-depth: 0 + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + version: "0.12.5" + enable-cache: false + + - name: Set up uv Python + run: | + uv python install 3.13 + - name: Verify inputs id: verify-inputs env: @@ -54,8 +64,8 @@ jobs: errors+=("The matcher '${matcher}' is not found.") continue fi - sha_short=$(git rev-parse HEAD | cut -c 1-7) - schedules+=("${matcher}:${sha_short}") + sha=$(git rev-parse HEAD) + schedules+=("${matcher}:${sha}") done if [ ${#errors[@]} -gt 0 ]; then printf "ERROR: %s\n" "${errors[@]}" @@ -80,32 +90,31 @@ jobs: repositories: ${{ secrets.REPO_NAME_GITOPS }} permission-contents: write - - name: Checkout GitOps Repository + - name: Checkout GitOps repository uses: actions/checkout@v7 with: repository: indexdata/${{ secrets.REPO_NAME_GITOPS }} path: ${{ secrets.REPO_NAME_GITOPS }} token: ${{ steps.app-token.outputs.token }} - - name: Append JSONL to file + - name: Generate data files env: ACTION: ${{ inputs.action }} SCHEDULE: ${{ steps.verify-inputs.outputs.schedule }} DIR_OUTPUT: ${{ secrets.REPO_NAME_GITOPS }} JOB_ID: ${{ github.run_number }} - run: python3 .github/workflows/scripts/schedule_deployments.py -l info - - - name: Show recent schedule - run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/schedule-deployments.jsonl + run: uv run .github/workflows/scripts/schedule_deployments.py -l info - name: Commit git changes and push run: | cd ${{ secrets.REPO_NAME_GITOPS }} - ls git config --global user.name "github-actions-reservoir-scripts" git config --global user.email "github-actions-reservoir-scripts@indexdata.com" git add . git status - git commit -m "Apply schedule deployments" + git commit -m "Generate pool custom resource" echo "Do git push ..." git push + + - name: Show recent schedule + run: tail -5 ${{ secrets.REPO_NAME_GITOPS }}/minitex-matchers/log/schedule-deployments.jsonl diff --git a/.github/workflows/scripts/schedule_deployments.py b/.github/workflows/scripts/schedule_deployments.py index 7c5cbdb..b8544eb 100755 --- a/.github/workflows/scripts/schedule_deployments.py +++ b/.github/workflows/scripts/schedule_deployments.py @@ -6,15 +6,27 @@ NOTE: Please use 'black' to re-format code. """ +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "jinja2", +# ] +# [tool.uv] +# exclude-newer = "7 days" +# /// + import argparse from datetime import datetime, timezone import json import logging import os from pathlib import Path +import pprint # pylint: disable=unused-import import sys -SCRIPT_VERSION = "1.4.0" +from jinja2 import Environment, FileSystemLoader + +SCRIPT_VERSION = "1.5.3" LOGLEVELS = { "debug": logging.DEBUG, @@ -69,18 +81,138 @@ def get_options(): except KeyError: LOGGER.error("Missing env: JOB_ID") options_okay = False - schedule_pn = PROG_PATH.parent.parent.parent.joinpath( - dir_output, "schedule-deployments.jsonl" - ) + templates_pn = PROG_PATH.joinpath("templates") + if not templates_pn.exists(): + LOGGER.error("The jinja templates '%s' not found.", templates_pn) + options_okay = False + dir_namespace = "minitex-matchers" + dir_storage = PROG_PATH.parent.parent.parent.joinpath(dir_output, dir_namespace) + if not dir_storage.exists(): + LOGGER.error("The namespace directory '%s' not found.", dir_namespace) + options_okay = False if not options_okay: sys.exit(2) - return int(job_id), action, schedule, schedule_pn + return int(job_id), action, schedule, templates_pn, dir_storage + + +def load_matchers_summary(): + """ + Loads the summary of all matchers. + """ + input_fn = "matchers-summary.json" + input_pn = PROG_PATH.parent.parent.parent.joinpath(input_fn) + # LOGGER.debug("%s", input_pn) + with open(input_pn, mode="r", encoding="utf-8") as json_fh: + try: + summary = json.load(json_fh) + except json.decoder.JSONDecodeError as err: + msg = f"Trouble loading '{input_fn}' JSON file: {err.lineno} {err.msg}" + LOGGER.critical(msg) + sys.exit(1) + return summary + + +def get_matcher_script(matchers_summary, matcher): + """ + Get the script pathname and type for this matcher. + """ + script_fn = None + script_type = None + input_fn = "matchers-summary.json" + if not any(dictionary.get("name") == matcher for dictionary in matchers_summary): + msg = f"Matcher '{matcher}' not found in '{input_fn}' file." + LOGGER.critical(msg) + sys.exit(1) + else: + matcher_details = [d for d in matchers_summary if d["name"] == matcher] + try: + matcher_details[0]["script"] + except KeyError: + msg = ( + "The 'script' property is not found for " + f"matcher '{matcher}' in '{input_fn}' file." + ) + LOGGER.critical(msg) + sys.exit(1) + script_fn = matcher_details[0]["script"] + script_pn = PROG_PATH.parent.parent.parent.joinpath(script_fn) + if not script_pn.exists(): + msg = ( + f"The script '{script_fn}' declared for " + f"matcher '{matcher}' in '{input_fn}' file " + "does not exist." + ) + LOGGER.critical(msg) + sys.exit(1) + try: + matcher_details[0]["type"] + except KeyError: + msg = ( + "The 'type' property is not found for " + f"matcher '{matcher}' in '{input_fn}' file." + ) + LOGGER.critical(msg) + sys.exit(1) + else: + script_type = matcher_details[0]["type"] + return script_fn, script_type + + +def assemble_pool_details(schedule, matchers_summary): + """ + Assembles the details of this pool. + """ + # LOGGER.debug("schedule=%s", schedule) + deployments = schedule.split(",") + matchers = [] + pool_matchers = [] + pool_details = {"matchers": []} + for deployment in deployments: + matcher_packet = {} + matcher, sha = deployment.split(":") + matcher_packet["name"] = matcher + matcher_packet["sha"] = sha + matcher_fn, matcher_type = get_matcher_script(matchers_summary, matcher) + matcher_packet["script"] = matcher_fn + matcher_packet["type"] = matcher_type + id_matcher = f"{matcher}~{sha[0:7]}" + matcher_packet["id"] = id_matcher + matchers.append(id_matcher) + pool_matchers.append(f"{id_matcher}-matcher::matchkey") + pool_details["matchers"].append(matcher_packet) + id_pool = "_".join(matchers) + pool_details["id_pool"] = id_pool + pool_details["pool_matcher"] = ", ".join(pool_matchers) + return id_pool, pool_details + + +def generate_cr(templates_pn, pool_details, dir_storage): + """ + Generates and stores the custom resource YAML. + """ + # pprint.pprint(pool_details) + dir_pools = dir_storage.joinpath("pools") + os.makedirs(dir_pools, exist_ok=True) + pool_pn = dir_pools.joinpath(f"{pool_details['id_pool']}.yaml") + env_jinja = Environment(loader=FileSystemLoader(templates_pn)) + template_cr = env_jinja.get_template("cr.yaml.jinja") + content_cr = template_cr.render( + id_pool=pool_details["id_pool"], + pool_matcher=pool_details["pool_matcher"], + matchers=pool_details["matchers"], + ) + with open(pool_pn, mode="w", encoding="utf-8") as output_fh: + output_fh.write(content_cr) + output_fh.write("\n") -def append_schedule(job_id, action, schedule, schedule_pn): +def append_schedule(job_id, action, id_pool, dir_storage): """ Composes the JSONL and appends to file. """ + dir_log = dir_storage.joinpath("log") + os.makedirs(dir_log, exist_ok=True) + schedule_pn = dir_log.joinpath("schedule-deployments.jsonl") json_packet = {} json_packet["id"] = job_id json_packet["scheduleDate"] = ( @@ -88,7 +220,7 @@ def append_schedule(job_id, action, schedule, schedule_pn): ) json_packet["action"] = action json_packet["initialized"] = False - json_packet["deployment"] = schedule + json_packet["poolId"] = id_pool with open(schedule_pn, mode="a", encoding="utf-8") as output_fh: output_fh.write(json.dumps(json_packet, sort_keys=False, indent=None)) output_fh.write("\n") @@ -98,9 +230,11 @@ def main(): """ Append the schedule JSONL to schedule-deployments.jsonl file. """ - job_id, action, schedule, schedule_pn = get_options() - LOGGER.debug("schedule=%s schedule_pn=%s", schedule, schedule_pn) - append_schedule(job_id, action, schedule, schedule_pn) + job_id, action, schedule, templates_pn, dir_storage = get_options() + matchers_summary = load_matchers_summary() + id_pool, pool_details = assemble_pool_details(schedule, matchers_summary) + generate_cr(templates_pn, pool_details, dir_storage) + append_schedule(job_id, action, id_pool, dir_storage) if __name__ == "__main__": diff --git a/.github/workflows/scripts/templates/cr.yaml.jinja b/.github/workflows/scripts/templates/cr.yaml.jinja new file mode 100644 index 0000000..a2799fd --- /dev/null +++ b/.github/workflows/scripts/templates/cr.yaml.jinja @@ -0,0 +1,16 @@ +apiVersion: indexdata.com/v1alpha1 +kind: Pool +metadata: + name: {{id_pool}} + namespace: minitex-matchers +spec: + pool: + id: {{id_pool}} + matcher: {{pool_matcher}} + update: ingest + codeModules: + {%- for matcher in matchers %} + - id: {{matcher.id}}-matcher + type: {{matcher.type}} + url: https://raw.githubusercontent.com/indexdata/reservoir-scripts/{{matcher.sha}}/{{matcher.script}} + {%- endfor -%} diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..393f406 --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +extends: default + +rules: + comments-indentation: disable + trailing-spaces: enable + new-line-at-end-of-file: enable + line-length: disable + document-start: + present: false diff --git a/js/README.md b/js/README.md index 3efa0d9..8475a6e 100644 --- a/js/README.md +++ b/js/README.md @@ -117,6 +117,7 @@ Each matcher is briefly explained in the [js/matchers/README.md](matchers/README For each matcher there is an entry in the [js/package.json](package.json) file to declare its test to be run using Node.js (e.g. `test-goldrush2024`). To add a new matcher, follow the structure of an existing matcher. +Also declare it at the file [matchers-summary.json](../matchers-summary.json). > [!IMPORTANT] > The matcher names are restricted to alpha-numeric or hyphen (dash) characters. diff --git a/matchers-summary.json b/matchers-summary.json new file mode 100644 index 0000000..19c093f --- /dev/null +++ b/matchers-summary.json @@ -0,0 +1,42 @@ +[ + { + "name": "deepdish", + "type": "javascript", + "script": "js/matchers/deepdish/deepdish.mjs" + }, + { + "name": "goldrush", + "type": "javascript", + "script": "js/matchers/goldrush/goldrush.mjs" + }, + { + "name": "goldrush2024", + "type": "javascript", + "script": "js/matchers/goldrush2024/goldrush.mjs" + }, + { + "name": "isxn", + "type": "javascript", + "script": "js/matchers/isxn/isxn.mjs" + }, + { + "name": "malort", + "type": "javascript", + "script": "js/matchers/malort/malort.mjs" + }, + { + "name": "shareInsts", + "type": "javascript", + "script": "js/matchers/shareInsts/shareInsts.mjs" + }, + { + "name": "shareWorks", + "type": "javascript", + "script": "js/matchers/shareWorks/shareWorks.mjs" + }, + { + "name": "sharevde", + "type": "javascript", + "script": "js/matchers/sharevde/sharevde.mjs" + } +]