diff --git a/README.md b/README.md index aab3785..fe11261 100644 --- a/README.md +++ b/README.md @@ -313,10 +313,10 @@ Pipelines download the script from a **pinned release tag**, not a moving branch so a push to `main` can never change what your build runs: ```bash -curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v1.0.4/ptc-cli.sh -o ptc-cli.sh +curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v1.0.5/ptc-cli.sh -o ptc-cli.sh ``` -Use an exact release tag such as `v1.0.4` to pin, or the floating `v1` tag to +Use an exact release tag such as `v1.0.5` to pin, or the floating `v1` tag to pick up backward-compatible updates automatically. `ptc init` scaffolds the pinned URL for you, at the version of the CLI that printed it. @@ -386,25 +386,54 @@ your own GitLab instance, so a component published anywhere else is unreachable. ptc-translate: stage: deploy image: alpine:3.22 + # Loop-safe twice over: the job only runs on a push to the default branch (the + # translation push targets ptc/translations, so it cannot retrigger this job), + # and rules: below refuses a commit marked [skip translations]. rules: - - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + # The second condition is what keeps this loop-safe, and it has to live + # here: GitLab evaluates rules: against the commit message, whereas + # `[skip ci]` in the message would suppress the pipeline of the merge + # request itself - leaving the translations untested and, with "Pipelines + # must succeed" enabled, unmergeable. + - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_COMMIT_MESSAGE !~ /\[skip translations\]/' before_script: + # jq is never invoked by the CLI. unzip is - it unpacks the downloaded + # translations; alpine already provides it as a busybox applet, so it is + # named here only to keep the job working if the image is ever changed. + # git is needed by the push step below, not by the CLI. - apk add --no-cache bash curl git unzip script: - - curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v1.0.4/ptc-cli.sh -o ptc-cli.sh - - chmod +x ptc-cli.sh - - ./ptc-cli.sh --config-file .ptc-config.yml - # `git add -A` comes BEFORE the check, and the check reads the index. - # On the first run the translations are new files, and a plain - # `git diff` only looks at tracked ones - it would report "nothing changed", - # skip the push, and leave a green job that produced no merge request. + # Downloaded OUTSIDE the checkout: anything this job writes into the working + # tree is a file the commit below could sweep into the merge request, and + # the CLI is 100+ KB of it. + - curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v1.0.5/ptc-cli.sh -o /tmp/ptc-cli.sh + - chmod +x /tmp/ptc-cli.sh + - rm -f /tmp/ptc-written + - /tmp/ptc-cli.sh --config-file .ptc-config.yml --written-manifest /tmp/ptc-written + # Pushing needs a token that may write to the repository. CI_JOB_TOKEN can, + # but ONLY if a maintainer turns on Settings > CI/CD > Job token permissions + # > "Allow Git push requests to the repository" (GitLab 18.4+, off by + # default). Otherwise set PTC_GIT_PUSH_TOKEN to a project access token with + # the write_repository scope, as a masked CI/CD variable. + # Staged from the manifest, so the merge request carries the translations and + # nothing else - not this job's downloads, not whatever an earlier step in + # your pipeline left in the working directory. + # + # `|| true` is not cosmetic: if a translation lands on a path your + # .gitignore covers, git exits 1 while still staging everything else, and + # GitLab would abort the job on that exit code alone. + # + # Staging comes BEFORE the check, and the check reads the index: on the + # first run the translations are new files, and a plain `git diff` only + # looks at tracked ones - it would report "nothing changed", skip the push, + # and leave a green job that produced no merge request. - | git config user.email "ci@ptc" git config user.name "PTC Translate" git checkout -B ptc/translations - git add -A + git add --pathspec-from-file=/tmp/ptc-written --pathspec-file-nul || true if ! git diff --cached --quiet; then - git commit -m "chore(i18n): update translations via PTC [skip ci]" + git commit -m "chore(i18n): update translations via PTC [skip translations]" git push -o merge_request.create \ -o merge_request.target="$CI_DEFAULT_BRANCH" \ -o merge_request.title="Update translations from PTC" \ diff --git a/ptc-cli.sh b/ptc-cli.sh index 814ae29..5ee919d 100755 --- a/ptc-cli.sh +++ b/ptc-cli.sh @@ -8,7 +8,7 @@ set -euo pipefail # Strict mode: exit on errors, undefined variables and pipe e # Constants readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_NAME="$(basename "$0")" -readonly VERSION="1.0.4" +readonly VERSION="1.0.5" readonly PTC_USER_AGENT="ptc-cli/${VERSION}" # Colors for output @@ -36,6 +36,10 @@ PTC_DRY_RUN=false PTC_MONITOR_INTERVAL=5 # seconds between status checks PTC_MONITOR_MAX_ATTEMPTS=100 # maximum number of status checks PTC_ACTION="" # specific action to perform: upload, status, download +# Where to record every file this run writes, for a CI job that must commit the +# translations and nothing else. Empty means "do not record". +PTC_WRITTEN_MANIFEST="" + @@ -363,6 +367,10 @@ OPTIONS: -d, --project-dir DIR Project directory (default: current) --api-url URL PTC API base URL (default: https://app.ptc.wpml.org/api/v1/) --api-token TOKEN API token override (prefer the PTC_API_TOKEN env var) + --written-manifest FILE Append every file this run writes to FILE, + NUL-separated and relative to the repository + root, for + git add --pathspec-from-file=FILE --pathspec-file-nul --monitor-interval SECONDS Seconds between status checks (default: 5) --monitor-max-attempts COUNT Maximum status check attempts (default: 100) --action ACTION Perform isolated action: upload, status, download @@ -426,13 +434,82 @@ show_version() { echo "$SCRIPT_NAME v$VERSION" } +# Records one written file in the manifest, if one was asked for. +# +# A CI job that commits translations has to know which files those are, and it +# cannot work them out: `git add -A` sweeps in whatever else the job left in the +# working directory, and the config's `output:` is not where the files land - +# the archive is unpacked next to the SOURCE file, by basename. +# +# NUL-separated and repository-root-relative, so a job can hand the file +# straight to `git add --pathspec-from-file=FILE --pathspec-file-nul` without +# parsing anything. Appended, never truncated: one manifest can span `ptc init` +# and the translate run that follows it. +record_written_path() { + local absolute="$1" root="$2" + + [[ -n "$PTC_WRITTEN_MANIFEST" ]] || return 0 + + local relative="${absolute#"$root"/}" + + # Still absolute means the path was not under the root we were given - + # which happens when one of them went through a symlink, as /var does on + # macOS. Recording it would be worse than skipping it: `git add` treats a + # pathspec that matches nothing as fatal and stages NOTHING at all, losing + # every other translation in the same call. + case "$relative" in + /*) + log_debug "Not recording '$absolute': outside the repository root '$root'" + return 0 + ;; + esac + + # `git add -- ` takes a PATHSPEC: `--` stops option parsing, it does + # not stop globbing. A translation written to messages[1].json - an ordinary + # Next.js or Nuxt layout - would otherwise stage the caller's + # messages1.json instead, and git would exit 0 having done it. + case "$relative" in + *'*'*|*'?'*|*'['*|:*) relative=":(literal)$relative" ;; + esac + + printf '%s\0' "$relative" >> "$PTC_WRITTEN_MANIFEST" +} + # Function to get current git branch +# +# CI runners check out a DETACHED HEAD - GitLab, Bitbucket Pipelines and the +# Jenkins git plugin all do. There `git branch --show-current` SUCCEEDS and +# prints an empty string, so the `||` fallbacks below are never reached, and the +# caller ends up with no file tag: the run then stops in validate_args before a +# single API call. That is why every CI recipe had to pass --file-tag-name by +# hand, and why the ones that forgot translated nothing at all. +# +# The runner knows the branch even when git does not, and says so in the +# environment. Those variables are consulted only after git has failed to +# answer, so a normal checkout is unaffected. get_current_branch() { + local branch="" + if command -v git >/dev/null 2>&1 && git rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git branch --show-current 2>/dev/null || git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "main" - else - echo "main" + branch=$(git branch --show-current 2>/dev/null) + if [[ -z "$branch" ]]; then + branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null) + # On a detached HEAD this is the literal string "HEAD", which is a + # worse tag than nothing - it would group every CI run together. + [[ "$branch" == "HEAD" ]] && branch="" + fi + fi + + if [[ -z "$branch" ]]; then + # GitLab, GitHub Actions, Bitbucket Pipelines, Jenkins, CircleCI. + branch="${CI_COMMIT_REF_NAME:-${GITHUB_REF_NAME:-${BITBUCKET_BRANCH:-${BRANCH_NAME:-${CIRCLE_BRANCH:-}}}}}" fi + + if [[ -z "$branch" ]]; then + branch="main" + fi + + echo "$branch" } # Function to get base directory (git root or current working directory) @@ -2496,7 +2573,17 @@ download_translations() { fi log_debug "Moving translation files to target directory..." local moved_count=0 - if find "$temp_extract_dir" -type f -name "*.json" -o -name "*.po" -o -name "*.pot" -o -name "*.mo" -o -name "*.yml" -o -name "*.yaml" 2>/dev/null | while read -r file; do + # Parenthesised, because `-o` binds looser than the implicit + # `-a`: without the group, `-type f` applied only to the first + # -name, so a DIRECTORY named e.g. "x.po" matched and was moved + # wholesale. + # + # .php and .properties are here because they are documented - + # `type: php` as an additional_translation_files companion, and + # .properties as a source pattern. Both were dropped silently: + # the archive carried them, the filter did not, and the user was + # left waiting for a compiled companion that never arrived. + if find "$temp_extract_dir" -type f \( -name "*.json" -o -name "*.po" -o -name "*.pot" -o -name "*.mo" -o -name "*.yml" -o -name "*.yaml" -o -name "*.php" -o -name "*.properties" -o -name "*.xml" -o -name "*.strings" -o -name "*.resx" \) 2>/dev/null | while read -r file; do local filename=$(basename "$file") local target_file="$target_dir/$filename" log_debug "Moving: $filename → $target_file" @@ -2512,6 +2599,11 @@ download_translations() { if mv "$file" "$target_file" 2>/dev/null; then # Verify the move was successful if [[ -f "$target_file" ]]; then + # Recorded here, where the file is proven on disk, + # so the manifest never names something that is not + # there - a pathspec matching nothing is fatal to + # `git add` and takes the whole staging call with it. + record_written_path "$target_file" "$base_dir" local final_size=$(stat -f%z "$target_file" 2>/dev/null || stat -c%s "$target_file" 2>/dev/null || echo "unknown") log_debug "Successfully moved $filename ($final_size bytes)" if [[ "$PTC_VERBOSE" == "true" ]]; then @@ -2957,10 +3049,14 @@ ptc-translate: image: alpine:3.22 # Loop-safe twice over: the job only runs on a push to the default branch (the # translation push targets ptc/translations, so it cannot retrigger this job), - # and the commit carries [skip ci] - the only skip token GitLab honours. The - # GitHub-side convention this recipe used before means nothing to GitLab. + # and rules: below refuses a commit marked [skip translations]. rules: - - if: '\$CI_PIPELINE_SOURCE == "push" && \$CI_COMMIT_BRANCH == \$CI_DEFAULT_BRANCH' + # The second condition is what keeps this loop-safe, and it has to live + # here: GitLab evaluates rules: against the commit message, whereas + # \`[skip ci]\` in the message would suppress the pipeline of the merge + # request itself - leaving the translations untested and, with "Pipelines + # must succeed" enabled, unmergeable. + - if: '\$CI_PIPELINE_SOURCE == "push" && \$CI_COMMIT_BRANCH == \$CI_DEFAULT_BRANCH && \$CI_COMMIT_MESSAGE !~ /\[skip translations\]/' before_script: # jq is never invoked by the CLI. unzip is - it unpacks the downloaded # translations; alpine already provides it as a busybox applet, so it is @@ -2968,25 +3064,37 @@ ptc-translate: # git is needed by the push step below, not by the CLI. - apk add --no-cache bash curl git unzip script: - - curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v${VERSION}/ptc-cli.sh -o ptc-cli.sh - - chmod +x ptc-cli.sh - - ./ptc-cli.sh --config-file .ptc-config.yml + # Downloaded OUTSIDE the checkout: anything this job writes into the working + # tree is a file the commit below could sweep into the merge request, and + # the CLI is 100+ KB of it. + - curl -fsSL https://raw.githubusercontent.com/OnTheGoSystems/ptc-cli/v${VERSION}/ptc-cli.sh -o /tmp/ptc-cli.sh + - chmod +x /tmp/ptc-cli.sh + - rm -f /tmp/ptc-written + - /tmp/ptc-cli.sh --config-file .ptc-config.yml --written-manifest /tmp/ptc-written # Pushing needs a token that may write to the repository. CI_JOB_TOKEN can, # but ONLY if a maintainer turns on Settings > CI/CD > Job token permissions # > "Allow Git push requests to the repository" (GitLab 18.4+, off by # default). Otherwise set PTC_GIT_PUSH_TOKEN to a project access token with # the write_repository scope, as a masked CI/CD variable. - # \`git add -A\` comes BEFORE the check, and the check reads the index. - # On the first run the translations are new files, and a plain - # \`git diff\` only looks at tracked ones - it would report "nothing changed", - # skip the push, and leave a green job that produced no merge request. + # Staged from the manifest, so the merge request carries the translations and + # nothing else - not this job's downloads, not whatever an earlier step in + # your pipeline left in the working directory. + # + # \`|| true\` is not cosmetic: if a translation lands on a path your + # .gitignore covers, git exits 1 while still staging everything else, and + # GitLab would abort the job on that exit code alone. + # + # Staging comes BEFORE the check, and the check reads the index: on the + # first run the translations are new files, and a plain \`git diff\` only + # looks at tracked ones - it would report "nothing changed", skip the push, + # and leave a green job that produced no merge request. - | git config user.email "ci@ptc" git config user.name "PTC Translate" git checkout -B ptc/translations - git add -A + git add --pathspec-from-file=/tmp/ptc-written --pathspec-file-nul || true if ! git diff --cached --quiet; then - git commit -m "chore(i18n): update translations via PTC [skip ci]" + git commit -m "chore(i18n): update translations via PTC [skip translations]" git push -o merge_request.create \\ -o merge_request.target="\$CI_DEFAULT_BRANCH" \\ -o merge_request.title="Update translations from PTC" \\ @@ -3235,6 +3343,14 @@ main() { PTC_API_TOKEN="${1#*=}" shift ;; + --written-manifest) + PTC_WRITTEN_MANIFEST="$2" + shift 2 + ;; + --written-manifest=*) + PTC_WRITTEN_MANIFEST="${1#*=}" + shift + ;; --monitor-interval) PTC_MONITOR_INTERVAL="$2" shift 2 diff --git a/tests/mock_ptc_api.py b/tests/mock_ptc_api.py new file mode 100644 index 0000000..1b4e2e3 --- /dev/null +++ b/tests/mock_ptc_api.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Minimal stand-in for the PTC API, enough to drive ptc-cli.sh 1.0.4 end to end. + +Endpoints (all under /api/v1/): + GET languages -> preflight #1 (+ balance headers) + GET balance -> preflight #2 (plan/active/status) + POST source_files -> upload (201) + PUT source_files/process -> start processing (200) + GET source_files/translation_status -> poll (200 completed) + GET source_files/download_translations -> zip of translations + POST detect_config -> `ptc init` layout detection + +Knobs (env): + PTC_MOCK_PORT default 8787 + PTC_MOCK_LOCALES comma-separated target locales, default "de,fr" + PTC_MOCK_PENDING how many status polls answer "in_progress" before + "completed" (per file). Default 0. + PTC_MOCK_LOG path to append a one-line-per-request journal. +""" +import io +import json +import os +import sys +import zipfile +from collections import defaultdict +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse, parse_qs + +PORT = int(os.environ.get("PTC_MOCK_PORT", "8787")) +# Loopback by default: binding 0.0.0.0 on a hosted macOS runner is refused by +# the firewall, and the suite then skipped itself while the job stayed green. +# Everything that talks to this mock is on the same host - under act the job +# container shares the VM's network namespace, so 127.0.0.1 reaches it there +# too. Override with PTC_MOCK_BIND if something ever needs the wider bind. +BIND = os.environ.get("PTC_MOCK_BIND", "127.0.0.1") +LOCALES = [x for x in os.environ.get("PTC_MOCK_LOCALES", "de,fr").split(",") if x] +PENDING = int(os.environ.get("PTC_MOCK_PENDING", "0")) +LOGFILE = os.environ.get("PTC_MOCK_LOG", "") + +# One process, one port per scenario, so a workflow picks a failure mode purely +# by the api-url it passes — no restart, no shared mutable state. +# +0 happy everything works +# +1 bad_token every authenticated call answers 401 -> preflight must abort +# +2 failed translation_status answers the terminal "failed" status +# +3 no_detect detect_config answers kind:"any" with no files +# +4 soft_fail upload answers 201 but with "success": false +SCENARIOS = ["happy", "bad_token", "failed", "no_detect", "soft_fail"] + +poll_counts = defaultdict(int) + + +def journal(line): + sys.stderr.write(line + "\n") + sys.stderr.flush() + if LOGFILE: + with open(LOGFILE, "a") as fh: + fh.write(line + "\n") + + +def make_zip(file_path): + """A translations archive: one file per target locale, named after the + source file with the locale substituted, as the real API returns.""" + base = os.path.basename(file_path) # en.json + stem, ext = os.path.splitext(base) # en, .json + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + for loc in LOCALES: + name = f"{loc}{ext}" if stem in ("en", "source") else f"{stem}-{loc}{ext}" + payload = { + "greeting": f"[{loc}] Hello", + "farewell": f"[{loc}] Goodbye", + "_mock": True, + } + z.writestr(name, json.dumps(payload, ensure_ascii=False, indent=2) + "\n") + return buf.getvalue() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "mock-ptc/1" + scenario = "happy" + + def log_message(self, fmt, *args): # silence the default noisy logger + pass + + # ---------- helpers ---------- + def _auth(self): + return self.headers.get("Authorization", "") + + def _send(self, code, body=b"", ctype="application/json", extra=None): + if isinstance(body, str): + body = body.encode() + self.send_response(code) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + # The balance headers ride on every /api/v1 response (CLI reads them + # off the `languages` call during preflight). + self.send_header("X-PTC-TRIAL-BALANCE", "12000") + self.send_header("X-PTC-PREPAID-BALANCE", "50000") + for k, v in (extra or {}).items(): + self.send_header(k, v) + self.end_headers() + if body: + self.wfile.write(body) + + def _json(self, code, obj, extra=None): + self._send(code, json.dumps(obj), "application/json", extra) + + def _path(self): + u = urlparse(self.path) + p = u.path + for prefix in ("/api/v1/", "/api/v1"): + if p.startswith(prefix): + p = p[len(prefix):] + break + return p.strip("/"), parse_qs(u.query) + + def _body(self): + n = int(self.headers.get("Content-Length", "0") or 0) + return self.rfile.read(n) if n else b"" + + def _need_token(self): + if self.scenario == "bad_token": + self._json(401, {"success": False, "message": "token rejected", "errors": [401]}) + return True + if not self._auth().startswith("Bearer "): + self._json(401, {"success": False, "message": "missing token", "errors": [401]}) + return True + return False + + # ---------- verbs ---------- + def do_GET(self): + route, q = self._path() + journal(f"GET /{route} q={ {k: v[0] for k, v in q.items()} } auth={'yes' if self._auth() else 'no'}") + + if route == "languages": + if self._need_token(): + return + return self._json(200, { + "source_language": {"iso": "en", "name": "English"}, + "languages": [{"iso": l, "name": l.upper()} for l in LOCALES], + }) + + if route == "balance": + if self._need_token(): + return + return self._json(200, {"plan": "pro", "active": True, "status": "unlimited", + "trial_balance": 12000, "prepaid_balance": 50000}) + + if route == "source_files/translation_status": + if self._need_token(): + return + fp = q.get("file_path", [""])[0] + if self.scenario == "failed": + return self._json(200, {"translation_status": { + "status": "failed", "completeness": 0}}) + poll_counts[(self.scenario, fp)] += 1 + if poll_counts[(self.scenario, fp)] <= PENDING: + return self._json(200, {"translation_status": { + "status": "in_progress", "completeness": 40}}) + + return self._json(200, {"translation_status": { + "status": "completed", "completeness": 100}}) + + if route == "source_files/download_translations": + if self._need_token(): + return + fp = q.get("file_path", [""])[0] + blob = make_zip(fp) + journal(f" -> zip {len(blob)} bytes for {fp} ({','.join(LOCALES)})") + return self._send(200, blob, "application/zip") + + return self._json(404, {"success": False, "message": f"no route {route}", "errors": [404]}) + + def do_POST(self): + route, q = self._path() + body = self._body() + journal(f"POST /{route} {len(body)}B auth={'yes' if self._auth() else 'no'}") + + if route == "source_files": + if self._need_token(): + return + if self.scenario == "soft_fail": + # A 201 that still carries "success": false is a + # rejected upload dressed as a created one. + return self._json(201, {"success": False, "message": "content rejected", + "errors": [4201]}) + return self._json(201, {"success": True, "id": 1, "message": "created"}) + + if route == "detect_config": + if self.scenario == "no_detect": + return self._json(200, {"kind": "any", "source_locale": "en", "files": [], + "available_locales": LOCALES}) + # anonymous by design + try: + paths = json.loads(body or b"{}").get("file_paths", []) + except Exception: + paths = [] + src = next((p for p in paths if p.endswith("/en.json") or p.endswith("en.json")), None) + if not src: + return self._json(200, {"kind": "any", "source_locale": "en", "files": [], + "available_locales": LOCALES}) + out = src.replace("en.json", "{{lang}}.json") + return self._json(200, { + "kind": "json-locale-files", + "source_locale": "en", + "available_locales": LOCALES, + "files": [{"file": src, "output": out}], + }) + + return self._json(404, {"success": False, "message": f"no route {route}", "errors": [404]}) + + def do_PUT(self): + route, q = self._path() + body = self._body() + journal(f"PUT /{route} {len(body)}B auth={'yes' if self._auth() else 'no'}") + + if route == "source_files/process": + if self._need_token(): + return + return self._json(200, {"success": True, "message": "processing started"}) + + return self._json(404, {"success": False, "message": f"no route {route}", "errors": [404]}) + + +if __name__ == "__main__": + import threading + + servers = [] + for i, name in enumerate(SCENARIOS): + cls = type(f"Handler_{name}", (Handler,), {"scenario": name}) + srv = ThreadingHTTPServer((BIND, PORT + i), cls) + servers.append(srv) + journal(f"mock PTC API {BIND}:{PORT + i} scenario={name}") + journal(f"locales={LOCALES} pending={PENDING}") + for srv in servers[1:]: + threading.Thread(target=srv.serve_forever, daemon=True).start() + servers[0].serve_forever() diff --git a/tests/test-ci-recipe.sh b/tests/test-ci-recipe.sh new file mode 100755 index 0000000..51b550b --- /dev/null +++ b/tests/test-ci-recipe.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# Runs the GitLab recipe this CLI prints, instead of grepping it. +# +# The recipe is copied verbatim into a user's .gitlab-ci.yml, so what matters is +# whether it works, not whether it contains particular words. Three defects +# lived in it while text assertions passed: +# +# 1. It did not translate anything. A GitLab runner checks out a DETACHED +# HEAD; `git branch --show-current` there succeeds and prints an empty +# string, so the fallbacks in get_current_branch are never reached and +# validate_args rejects the empty file tag - before a single API call. +# 2. It curled ptc-cli.sh into the project root and then ran `git add -A`, +# so the merge request carried the CLI itself, plus whatever the caller's +# job had already dirtied. +# 3. Because that download is always a new file, `git diff --cached --quiet` +# never short-circuited: every run force-updated the merge request even +# when no translation had changed. +# +# The real CLI runs here, against a mock PTC API on localhost. Only two things +# are shimmed: `curl`, so the recipe installs the working copy rather than +# downloading a release, and `git push`, because there is nowhere to push. +set -uo pipefail + +readonly TEST_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly CLI_UNDER_TEST="$TEST_DIR/../ptc-cli.sh" +readonly MOCK="$TEST_DIR/mock_ptc_api.py" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' +test_count=0; passed_count=0; failed_count=0 +pass() { echo -e "${GREEN}[PASS]${NC} $*"; passed_count=$((passed_count + 1)); test_count=$((test_count + 1)); } +fail() { echo -e "${RED}[FAIL]${NC} $*"; failed_count=$((failed_count + 1)); test_count=$((test_count + 1)); } + +assert_eq() { + local desc="$1" got="$2" want="$3" + if [[ "$got" == "$want" ]]; then pass "$desc"; else fail "$desc (got '$got', want '$want')"; fi +} + +# --- the mock --------------------------------------------------------------- +# A plain python process, no docker: the CLI only needs an --api-url. +MOCK_PID="" +MOCK_PORT="" + +MOCK_LOG="" + +start_mock() { + MOCK_LOG="$(mktemp "${TMPDIR:-/tmp}/ptc-mock-log-XXXXXX")" + local port + for port in 18787 18797 18807 18817; do + PTC_MOCK_PORT="$port" PTC_MOCK_LOCALES=de,fr PTC_MOCK_PENDING=0 \ + python3 "$MOCK" >>"$MOCK_LOG" 2>&1 & + local pid=$! + local i + # Up to ~12s: a cold python start on a hosted macOS runner is slow, and + # a short wait here turns into a silently skipped suite - which is how + # this suite stopped covering macOS without anyone noticing. + for i in $(seq 1 30); do + if python3 -c "import socket; socket.create_connection(('127.0.0.1', $port), 0.4).close()" 2>/dev/null; then + MOCK_PID="$pid"; MOCK_PORT="$port"; return 0 + fi + kill -0 "$pid" 2>/dev/null || break + sleep 0.4 + done + disown "$pid" 2>/dev/null || true + kill "$pid" 2>/dev/null + done + echo "the mock did not come up; its output was:" >&2 + sed 's/^/ /' "$MOCK_LOG" >&2 + return 1 +} + +stop_mock() { + # `disown` first: without it the shell announces "Terminated" on stderr when + # the background mock is killed, which reads like a test failure. + if [ -n "$MOCK_PID" ]; then + disown "$MOCK_PID" 2>/dev/null || true + kill "$MOCK_PID" 2>/dev/null + fi + [ -n "$MOCK_LOG" ] && rm -f "$MOCK_LOG" + return 0 +} +trap stop_mock EXIT + +# --- fixture ---------------------------------------------------------------- +# A repository the way a GitLab runner leaves it: detached HEAD, and a working +# directory an earlier job step has already dirtied. +make_fixture() { + local dir + # -P: on macOS TMPDIR lives under /var, a symlink to /private/var, and the + # CLI compares the git root against paths found by `find`. A mismatched + # prefix there makes it fall back to absolute paths, which is a real defect + # but not the one this suite is about. + dir="$(cd "$(mktemp -d "${TMPDIR:-/tmp}/ptc-recipe-XXXXXX")" && pwd -P)" + mkdir -p "$dir/locales" + printf '{"hello":"world"}\n' > "$dir/locales/en.json" + # api_url in the config, because the recipe does not print --api-url and + # PTC_API_URL is assigned unconditionally in the CLI, so the environment + # cannot redirect it. Without this the recipe reaches the real PTC and the + # suite fails with a 401 that looks like a bug in the recipe. + cat > "$dir/.ptc-config.yml" < "$bin/curl" < "$bin/git" < "\$PTC_TEST_PUSH_LOG" + exit 0 +fi +exec $(command -v git) "\$@" +SHIM + chmod +x "$bin/curl" "$bin/git" +} + +# Extracts the shell of the printed GitLab job, in order. +recipe_script() { + bash -c "source '$CLI_UNDER_TEST' >/dev/null 2>&1; render_ci_gitlab" | awk ' + /^ script:/ { s = 1; next } + s && /^ [a-z_]+:/ { s = 0 } + s { + line = $0 + if (line ~ /^ - \|$/) next + sub(/^ - /, "", line) + sub(/^ /, "", line) + print line + }' +} + +# Runs the recipe in the fixture and prints nothing; the caller inspects the repo. +run_recipe() { + local dir="$1" bin="$dir/../bin.$$" + make_shims "$bin" + export PTC_TEST_PUSH_LOG="$dir/../push.args.$$" + rm -f "$PTC_TEST_PUSH_LOG" + # Written outside the repository: a script inside it is another file the + # recipe could sweep into the merge request, which would mask the defect. + local script="$dir/../recipe.$$.sh" + # `set -e` because GitLab aborts a job at the first failing script line - + # without it a CLI that translated nothing still reaches the git block and + # the recipe looks like it succeeded. + { echo 'set -e'; recipe_script; } > "$script" + ( cd "$dir" && PATH="$bin:$PATH" \ + PTC_API_TOKEN=mock-token-abcdef \ + CI_DEFAULT_BRANCH=main \ + CI_COMMIT_REF_NAME=main \ + CI_SERVER_HOST=gitlab.example \ + CI_PROJECT_PATH=group/project \ + CI_JOB_TOKEN=job-token \ + bash "$script" >"$dir/../recipe.log.$$" 2>&1 ) + local rc=$? + # A failing recipe is the interesting case; print why rather than leaving + # the reader with an exit code. + if [ "$rc" -ne 0 ] || [ -n "${PTC_TEST_TRACE:-}" ]; then + echo "--- recipe output (exit $rc) ---" >&2 + tail -15 "$dir/../recipe.log.$$" >&2 + echo "--- end ---" >&2 + fi + rm -f "$dir/../recipe.log.$$" "$script" + echo $rc +} + +committed_files() { + git -C "$1" show --stat --format="" HEAD 2>/dev/null | \ + sed -n 's/^ \([^|]*\)|.*/\1/p' | sed 's/ *$//' | LC_ALL=C sort | tr '\n' ' ' | sed 's/ *$//' +} + +# --- 1. the recipe actually translates on a detached HEAD ------------------- +test_recipe_translates() { + echo -e "${YELLOW}[TEST]${NC} the printed recipe translates on a detached HEAD" + local dir rc + dir="$(make_fixture)" + rc="$(run_recipe "$dir")" + assert_eq "the recipe exits 0 on a runner's detached HEAD" "$rc" "0" + assert_eq "the translations reached disk" \ + "$(ls "$dir/locales" | LC_ALL=C sort | tr '\n' ' ' | sed 's/ *$//')" \ + "de.json en.json fr.json" + rm -rf "$dir" +} + +# --- 2. the merge request carries translations and nothing else ------------- +test_recipe_commits_only_translations() { + echo -e "${YELLOW}[TEST]${NC} the merge request carries only what the run wrote" + local dir + dir="$(make_fixture)" + # What an earlier step in the caller's job leaves behind. + mkdir -p "$dir/dist" + printf 'BUILD\n' > "$dir/dist/bundle.js" + printf '{"touched":true}\n' > "$dir/package-lock.json" + run_recipe "$dir" >/dev/null + + assert_eq "only the translations are committed" \ + "$(committed_files "$dir")" "locales/de.json locales/fr.json" + rm -rf "$dir" +} + +# --- 3. the CLI the recipe downloads does not end up in the repository ------ +test_cli_not_left_behind() { + echo -e "${YELLOW}[TEST]${NC} the downloaded CLI stays out of the repository" + local dir + dir="$(make_fixture)" + run_recipe "$dir" >/dev/null + + if printf '%s' "$(committed_files "$dir")" | grep -q 'ptc-cli.sh'; then + fail "the downloaded CLI was committed into the merge request" + else + pass "the downloaded CLI was not committed" + fi + if [ -e "$dir/ptc-cli.sh" ]; then + fail "the downloaded CLI was left in the working tree" + else + pass "the downloaded CLI was not left in the working tree" + fi + rm -rf "$dir" +} + +# --- 4. a run that writes nothing pushes nothing ---------------------------- +test_no_translations_no_push() { + echo -e "${YELLOW}[TEST]${NC} a run that writes nothing opens no merge request" + local dir + dir="$(make_fixture)" + # Already translated: the mock returns the same content, so nothing changes. + run_recipe "$dir" >/dev/null + git -C "$dir" add -A >/dev/null 2>&1 + git -C "$dir" commit -qm "translations already in" >/dev/null 2>&1 + rm -f "$PTC_TEST_PUSH_LOG" + run_recipe "$dir" >/dev/null + + if [ -f "$PTC_TEST_PUSH_LOG" ]; then + fail "a second identical run still pushed (merge request churn)" + else + pass "a second identical run pushed nothing" + fi + rm -rf "$dir" +} + +# --- 5. the push is a merge request against the default branch -------------- +test_push_shape() { + echo -e "${YELLOW}[TEST]${NC} the push asks GitLab for a merge request" + local dir args + dir="$(make_fixture)" + run_recipe "$dir" >/dev/null + args="$(cat "$PTC_TEST_PUSH_LOG" 2>/dev/null || echo '')" + + if printf '%s' "$args" | grep -q 'merge_request.create'; then + pass "the push creates a merge request" + else + fail "the push does not create a merge request (args: $args)" + fi + if printf '%s' "$args" | grep -q 'merge_request.target=main'; then + pass "the merge request targets the default branch" + else + fail "the merge request does not target the default branch (args: $args)" + fi + rm -rf "$dir" +} + +main() { + echo "=== printed GitLab recipe, executed ===" + for tool in python3 curl unzip git; do + if ! command -v "$tool" >/dev/null 2>&1; then + if [ -n "${PTC_REQUIRE_E2E:-}" ]; then + fail "$tool is required for this suite" + echo "Total: $test_count Passed: $passed_count Failed: $failed_count" + return 1 + fi + echo "skipped: $tool is not installed" + return 0 + fi + done + + if ! start_mock; then + if [ -n "${PTC_REQUIRE_E2E:-}" ]; then + fail "the mock PTC API did not start" + return 1 + fi + echo "skipped: could not start the mock PTC API" + return 0 + fi + echo "mock PTC API on 127.0.0.1:$MOCK_PORT" + + test_recipe_translates + test_recipe_commits_only_translations + test_cli_not_left_behind + test_no_translations_no_push + test_push_shape + + echo + echo "Total: $test_count Passed: $passed_count Failed: $failed_count" + [ "$failed_count" -eq 0 ] +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/tests/test-git-context.sh b/tests/test-git-context.sh index 0859448..a7e4dcd 100755 --- a/tests/test-git-context.sh +++ b/tests/test-git-context.sh @@ -63,13 +63,31 @@ main() { fi # 2. Detached HEAD — what every CI checkout looks like by default. + # + # This used to stop the run: `git branch --show-current` succeeds and prints + # an empty string there, so the fallbacks were never reached and the empty + # tag was rejected before a single API call. Every CI recipe that forgot to + # pass --file-tag-name therefore translated nothing at all, which is exactly + # what the printed GitLab recipe did. ( cd "$repo" && git checkout -q --detach HEAD ) - output="$(run_cli "$repo")" + output="$(CI_COMMIT_REF_NAME=feature/from-ci run_cli "$repo")" if echo "$output" | grep -q 'could not auto-detect git branch'; then - pass "detached HEAD: auto-detection fails with an explicit message" + fail "detached HEAD: the run still stops instead of using the runner's branch" + echo "$output" | tail -3 | sed 's/^/ /' else - fail "detached HEAD: expected the auto-detect failure message" + pass "detached HEAD: the runner's branch variable carries the file tag" + fi + + # ...and with no runner variables either, a default beats stopping. + # A subshell with the variables unset - `env -u` cannot run a shell + # function, and would have made this assert pass on empty output. + output="$( unset CI_COMMIT_REF_NAME GITHUB_REF_NAME BITBUCKET_BRANCH BRANCH_NAME CIRCLE_BRANCH + run_cli "$repo" )" + if echo "$output" | grep -q 'could not auto-detect git branch'; then + fail "detached HEAD with no CI variables: the run stops" echo "$output" | tail -3 | sed 's/^/ /' + else + pass "detached HEAD with no CI variables: falls back rather than stopping" fi # 3. ...and an explicit tag is the way through it. This is why the CI @@ -84,14 +102,10 @@ main() { # 4. Outside a repository there IS a default, and the run proceeds. # - # Note the asymmetry with case 2, which is what it looks like: no repo - # falls back to "main", while a repo on a detached HEAD falls back to - # nothing. get_current_branch chains - # git branch --show-current || git rev-parse --abbrev-ref HEAD || echo main - # and that chain assumes the first command fails on a detached HEAD. It does - # not — it prints an empty string and exits 0, so neither fallback is - # reached. Asserted here as it stands; changing it changes CLI behaviour and - # belongs in its own change, not in a test. + # The asymmetry this note used to describe — no repo falls back to "main", + # a detached HEAD fell back to nothing — is gone: get_current_branch now + # treats an empty `git branch --show-current` as "no answer", consults the + # runner's own branch variables, and only then defaults. local bare bare="$(mktemp -d)" mkdir -p "$bare/locales" diff --git a/tests/test-init.sh b/tests/test-init.sh index cad698f..5de8c6c 100755 --- a/tests/test-init.sh +++ b/tests/test-init.sh @@ -397,10 +397,21 @@ test_ci_snippets_use_action() { assert_not_contains "gitlab does not float the CLI on main" "$gl" "ptc-cli/main/ptc-cli.sh" assert_contains "gitlab brings its own runner image" "$gl" "image: alpine:" assert_contains "gitlab installs bash for the CLI" "$gl" "apk add --no-cache bash" - # [skip ci] is the only skip token GitLab actually honours - [skip - # translations] is a GitHub-side convention and means nothing here. - assert_contains "gitlab guards the loop with a real skip token" "$gl" '[skip ci]' - assert_not_contains "gitlab does not rely on a token GitLab ignores" "$gl" '[skip translations]' + # [skip ci] IS the token GitLab honours - by creating no pipeline at all for + # that push. Used here it silenced the pipeline of the translation merge + # request itself, so the translations could not be tested before merge and, + # with "Pipelines must succeed" enabled, could not be merged at all. Worse, + # a squash or merge commit carried [skip ci] into the default branch and + # silenced the whole project's pipeline. + # + # The guard belongs in rules:, which GitLab evaluates against the commit + # message without suppressing anything. + # The commit message specifically - the recipe's comments mention [skip ci] + # to explain why it is not used, and a bare substring check would trip on + # its own rationale. + assert_not_contains "gitlab does not silence the merge request's own pipeline" \ + "$(printf '%s\n' "$gl" | grep 'git commit -m')" '[skip ci]' + assert_contains "gitlab guards the loop in rules:, where GitLab evaluates it" "$gl" 'CI_COMMIT_MESSAGE !~' assert_contains "gitlab only runs on the default branch" "$gl" 'CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' assert_contains "gitlab reuses one stable MR branch" "$gl" "HEAD:ptc/translations" assert_contains "gitlab opens the MR via push options" "$gl" "merge_request.create" @@ -408,20 +419,46 @@ test_ci_snippets_use_action() { # by default), so the recipe must accept a write_repository token instead. assert_contains "gitlab allows a push token override" "$gl" 'PTC_GIT_PUSH_TOKEN:-$CI_JOB_TOKEN' - # On the first run the translations are NEW files. `git diff` - # reads the worktree against the index and only sees tracked paths, so it - # reports "nothing changed", the push is skipped, and the job goes green - # having produced no merge request. Staging first and asking `--cached` - # is what makes run #1 push. Order is the defect, so assert the order, - # not merely the presence of both commands. + # On the first run the translations are NEW files. `git diff` reads the + # worktree against the index and only sees tracked paths, so it reports + # "nothing changed", the push is skipped, and the job goes green having + # produced no merge request. Staging first and asking `--cached` is what + # makes run #1 push. assert_contains "gitlab checks the index, where new files land" "$gl" "git diff --cached --quiet" assert_not_contains "gitlab does not check the worktree, which misses new files" "$gl" "if ! git diff --quiet" - local add_at check_at - add_at=$(printf '%s\n' "$gl" | grep -n "git add -A" | head -1 | cut -d: -f1) - check_at=$(printf '%s\n' "$gl" | grep -n "git diff --cached --quiet" | head -1 | cut -d: -f1) - assert_eq "gitlab stages before it checks for changes" \ - "$([ -n "$add_at" ] && [ -n "$check_at" ] && [ "$add_at" -lt "$check_at" ] && echo "add-then-check" || echo "check-then-add")" \ - "add-then-check" + + # What is staged comes from the manifest the CLI writes, not from the whole + # working directory: the job downloads the CLI, and an earlier step in the + # caller's pipeline may have written anything else. + assert_not_contains "gitlab does not commit the whole working directory" "$gl" "git add -A" + assert_contains "gitlab stages what the run recorded" "$gl" "--pathspec-from-file=/tmp/ptc-written" + assert_contains "gitlab reads that file as NUL-separated" "$gl" "--pathspec-file-nul" + assert_contains "gitlab asks the CLI for that manifest" "$gl" "--written-manifest /tmp/ptc-written" + # An ignored translation path makes git exit 1 while still staging the rest, + # and GitLab would abort the job on that alone. + assert_contains "gitlab survives a partially ignored stage" "$gl" "--pathspec-file-nul || true" + # The CLI is downloaded outside the checkout, or it ends up in the MR. + assert_not_contains "gitlab does not download the CLI into the project" "$gl" "-o ptc-cli.sh" + assert_contains "gitlab downloads the CLI outside the checkout" "$gl" "-o /tmp/ptc-cli.sh" + + # Behaviour, not wording: tests/test-ci-recipe.sh executes this recipe. + + # The README carries a copy of this recipe. Copies of it have drifted before + # - at one point the CLI printed one shape, a fresh install printed another, + # and both READMEs printed a third that nothing produced. Compare them. + local readme="$TEST_DIR/../README.md" + if [ -f "$readme" ]; then + local in_readme + in_readme=$(awk '/^```yaml$/ { block = ""; inblock = 1; next } + inblock && /^```$/ { if (block ~ /ptc-translate:/) { printf "%s", block; exit } inblock = 0; next } + inblock { block = block $0 "\n" }' "$readme") + if [ "$(printf '%s' "$in_readme" | sed 's/[[:space:]]*$//')" = "$(printf '%s\n' "$gl" | sed 's/[[:space:]]*$//')" ]; then + pass "the README carries the recipe the CLI actually prints" + else + fail "the README recipe has drifted from what ptc init prints" + diff <(printf '%s' "$in_readme") <(printf '%s\n' "$gl") | head -8 | sed 's/^/ /' + fi + fi # The standalone path is still offered, so the CLI does not depend on the action. assert_contains "standalone CLI usage is still shown" "$block" "./ptc-cli.sh --config-file .ptc-config.yml"