From db259ce2869ed68f5359a33578b917b6688effb8 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:23:59 -0300 Subject: [PATCH 1/9] Write exit-code when the base revision is missing check.sh exited 2 before writing its outputs when the base revision was not in the checkout, so the documented exit-code output was empty exactly when the run was incomplete. Every exit now goes through one function that writes exit-code and report first. --- check.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/check.sh b/check.sh index 32da052..ef56f69 100755 --- a/check.sh +++ b/check.sh @@ -2,11 +2,22 @@ # Run `jevgate check` and pass its exit code on: 0 passed, 1 failed, 2 incomplete. set -uo pipefail +report=.jevgate/latest.json + +# Write the step's outputs, then exit with CODE. +finish() { + { + echo "exit-code=$1" + echo "report=$PWD/$report" + } >> "$GITHUB_OUTPUT" + exit "$1" +} + command=(jevgate check) if [ -n "$BASE" ]; then if ! git cat-file -e "$BASE^{commit}" 2> /dev/null; then echo "::error::The base revision $BASE is not in the checkout. Check out with fetch-depth: 0 so --base can find the fork point." - exit 2 + finish 2 fi command+=(--base "$BASE") fi @@ -26,9 +37,7 @@ if [ -n "${SARIF_FILE:-}" ]; then fi fi -echo "exit-code=$code" >> "$GITHUB_OUTPUT" -echo "report=$PWD/.jevgate/latest.json" >> "$GITHUB_OUTPUT" if [ "$code" = 2 ] && [ -z "${TYPESAFE_API_KEY:-}" ]; then echo "::error::No TypeSafe API key. Pass api-key: \${{ secrets.TYPESAFE_API_KEY }}. Pull requests from forks don't receive secrets; skip the job for them." fi -exit "$code" +finish "$code" From d8d542807f643de652db4d618d67a50a01219464 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:24:50 -0300 Subject: [PATCH 2/9] Give the report output as a Windows path on Windows On Windows runners the bash shell is Git Bash, whose $PWD is /d/a/...; the report output built from it names a path Node resolves to D:\d\a\..., so upload-artifact and other JavaScript actions could not open it. cygpath -m turns it into D:/a/..., which both bash and Node open; elsewhere the path is unchanged. A new job checks it on all four runners: a repository created in the job, whose one change no cache answers, runs with --cache-only, which writes a report and ends incomplete with no key and nothing sent. It checks exit code 2 and that Node can open the report output. --- .github/workflows/test.yml | 30 ++++++++++++++++++++++++++++++ check.sh | 7 ++++++- test/fixture-repository.sh | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 test/fixture-repository.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2474fe4..54110b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,33 @@ jobs: [ "$CODE" = 0 ] || { echo "::error::exit code $CODE"; exit 1; } jevgate --version grep -q '"version": "2.1.0"' jevgate.sarif || { echo "::error::no SARIF log"; exit 1; } + incomplete: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # A change whose answers no cache holds: --cache-only writes a report + # and ends incomplete, with no key and nothing sent. + - name: Create a repository to check + shell: bash + run: bash test/fixture-repository.sh fixture + - id: jevgate + uses: ./ + continue-on-error: true + with: + working-directory: fixture + base: HEAD~1 + args: --cache-only + cache: "false" + - name: Check the outputs + shell: bash + env: + CODE: ${{ steps.jevgate.outputs.exit-code }} + REPORT: ${{ steps.jevgate.outputs.report }} + run: | + [ "$CODE" = 2 ] || { echo "::error::exit code $CODE"; exit 1; } + # The path must open outside bash too, as upload-artifact opens it. + node -e 'require("fs").accessSync(process.argv[1])' "$REPORT" || { echo "::error::no report at $REPORT"; exit 1; } diff --git a/check.sh b/check.sh index ef56f69..791ca8b 100755 --- a/check.sh +++ b/check.sh @@ -4,11 +4,16 @@ set -uo pipefail report=.jevgate/latest.json +# A path Node can open: on Windows, Git Bash's $PWD (/d/a/...) is not one. +native() { + if command -v cygpath > /dev/null; then cygpath -m "$1"; else echo "$1"; fi +} + # Write the step's outputs, then exit with CODE. finish() { { echo "exit-code=$1" - echo "report=$PWD/$report" + echo "report=$(native "$PWD")/$report" } >> "$GITHUB_OUTPUT" exit "$1" } diff --git a/test/fixture-repository.sh b/test/fixture-repository.sh new file mode 100644 index 0000000..65bca2c --- /dev/null +++ b/test/fixture-repository.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Create a repository at DIR whose last commit adds one function. Its answers +# are never in a cache, so `jevgate check --base HEAD~1 --cache-only` writes a +# report and ends incomplete (exit 2), with no key and nothing sent. +set -euo pipefail + +dir=$1 +git -c init.defaultBranch=main init -q "$dir" +cd "$dir" +commit() { + git -c user.name=test -c user.email=test@example.com commit -q "$@" +} +commit --allow-empty -m "Start" +mkdir -p src +cat > src/records.js << 'EOF' +function parseRecord(line, options) { + const fields = line.split(options.separator || ","); + const record = {}; + for (let i = 0; i < fields.length; i++) { + const raw = fields[i].trim(); + if (raw === "") { + continue; + } + if (options.numbers && !isNaN(Number(raw))) { + record[options.columns[i]] = Number(raw); + } else if (raw === "true" || raw === "false") { + record[options.columns[i]] = raw === "true"; + } else { + record[options.columns[i]] = raw; + } + } + return record; +} +module.exports = { parseRecord }; +EOF +git add src +commit -m "Parse records" From bc507ccff034ed23416d8a035e7b639dab129d52 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:26:50 -0300 Subject: [PATCH 3/9] Take OpenRouter and Vercel AI Gateway keys with api-key-kind JevGate 0.26.0 accepts keys from the two gateways that serve Jev. The new input api-key-kind (typesafe, openrouter or vercel; typesafe by default) says which service issued api-key, and the check gets the key as TYPESAFE_API_KEY, OPENROUTER_API_KEY or AI_GATEWAY_API_KEY, and no other kind's key: a job that keeps an OpenRouter key for its own tests must not have JevGate spend it because it also reads that variable. With a gateway kind and a JevGate older than 0.26.0, the check stops with an error naming the version, instead of JevGate reporting that no key is configured. The key is set only when api-key is given, so a key a workflow puts in the job's env for that kind is no longer replaced by an empty input. test/key-kinds.sh runs check.sh against a stand-in jevgate that records the key variables it receives, on Linux, macOS and Windows. --- .github/workflows/test.yml | 11 +++++++ README.md | 17 ++++++++++- action.yml | 9 ++++-- check.sh | 32 ++++++++++++++++++-- test/key-kinds.sh | 60 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 test/key-kinds.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54110b1..3c51290 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,17 @@ on: permissions: contents: read jobs: + unit: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Key kinds + shell: bash + run: bash test/key-kinds.sh dry-run: strategy: fail-fast: false diff --git a/README.md b/README.md index 7906c49..62d61d6 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,8 @@ The action installs a release binary (checked against its SHA-256), keeps JevGat | Input | Default | Meaning | |---|---|---| -| `api-key` | | TypeSafe API key. [Get one](https://console.typesafe.ai/settings/keys) and save it as a repository secret | +| `api-key` | | TypeSafe API key ([get one](https://console.typesafe.ai/settings/keys)), or an OpenRouter or Vercel AI Gateway key with `api-key-kind`. Save it as a repository secret | +| `api-key-kind` | `typesafe` | Which service issued `api-key`: `typesafe`, `openrouter` or `vercel` (the gateways need JevGate 0.26.0 or later) | | `version` | `latest` | JevGate version; pin one for repeatable results | | `base` | the pull request's base commit | Review only files changed since this revision; empty on other events, which review the whole repository | | `args` | | More `jevgate check` arguments, such as `--rule default --rule security --include-tests` | @@ -68,6 +69,20 @@ jobs: category: jevgate ``` +## Keys from OpenRouter or Vercel AI Gateway + +OpenRouter and Vercel AI Gateway also serve Jev. With JevGate 0.26.0 or later, pass their key and say which it is; the action gives it to JevGate as `OPENROUTER_API_KEY` or `AI_GATEWAY_API_KEY`, and no other kind's key, so a key the job holds for something else is never used: + +```yaml + - uses: Tech-Byte-Frontier/jevgate-action@v1 + with: + api-key: ${{ secrets.OPENROUTER_API_KEY }} + api-key-kind: openrouter # or vercel + version: 0.26.0 +``` + +With an older version the check stops with an error instead of running without a key. + ## Pull requests from forks GitHub doesn't give secrets to pull requests from forks, so the run there ends incomplete with "No API key configured". Skip the job for them: diff --git a/action.yml b/action.yml index ab91515..638b96f 100644 --- a/action.yml +++ b/action.yml @@ -6,9 +6,13 @@ branding: color: blue inputs: api-key: - description: TypeSafe API key, usually secrets.TYPESAFE_API_KEY. Pull requests from forks don't receive secrets. + description: API key, usually secrets.TYPESAFE_API_KEY; `api-key-kind` says which service issued it. Pull requests from forks don't receive secrets. required: false default: "" + api-key-kind: + description: Which service issued `api-key`, `typesafe`, `openrouter` (OpenRouter) or `vercel` (Vercel AI Gateway). The gateways need JevGate 0.26.0 or later. + required: false + default: typesafe version: description: JevGate version to install, such as 0.17.0, or `latest`. Pin one for repeatable results. required: false @@ -64,7 +68,8 @@ runs: shell: bash working-directory: ${{ inputs.working-directory }} env: - TYPESAFE_API_KEY: ${{ inputs.api-key }} + API_KEY: ${{ inputs.api-key }} + API_KEY_KIND: ${{ inputs.api-key-kind }} BASE: ${{ inputs.base || github.event.pull_request.base.sha }} FORMAT: ${{ inputs.format }} ARGS: ${{ inputs.args }} diff --git a/check.sh b/check.sh index 791ca8b..989a7e8 100755 --- a/check.sh +++ b/check.sh @@ -3,6 +3,7 @@ set -uo pipefail report=.jevgate/latest.json +version=$(jevgate --version 2> /dev/null) # A path Node can open: on Windows, Git Bash's $PWD (/d/a/...) is not one. native() { @@ -18,6 +19,33 @@ finish() { exit "$1" } +# The key goes in the variable JevGate reads for its kind. +case "$API_KEY_KIND" in + typesafe) key_variable=TYPESAFE_API_KEY ;; + openrouter) key_variable=OPENROUTER_API_KEY ;; + vercel) key_variable=AI_GATEWAY_API_KEY ;; + *) + echo "::error::api-key-kind is typesafe, openrouter or vercel, not $API_KEY_KIND." + finish 2 + ;; +esac +if [ "$API_KEY_KIND" != typesafe ]; then + # Older versions read only TYPESAFE_API_KEY and would say no key is configured. + IFS=. read -r major minor _ <<< "${version##* }" + if [ "$major" = 0 ] && [ "${minor:-0}" -lt 26 ] 2> /dev/null; then + echo "::error::api-key-kind: $API_KEY_KIND needs JevGate 0.26.0 or later; this is ${version##* }." + finish 2 + fi +fi +# Only a key that was given, so an empty input leaves one set in the job's env. +if [ -n "$API_KEY" ]; then + export "$key_variable=$API_KEY" +fi +# And only that kind's: a key the job holds for another service is never spent. +for variable in TYPESAFE_API_KEY OPENROUTER_API_KEY AI_GATEWAY_API_KEY; do + if [ "$variable" != "$key_variable" ]; then unset "$variable"; fi +done + command=(jevgate check) if [ -n "$BASE" ]; then if ! git cat-file -e "$BASE^{commit}" 2> /dev/null; then @@ -42,7 +70,7 @@ if [ -n "${SARIF_FILE:-}" ]; then fi fi -if [ "$code" = 2 ] && [ -z "${TYPESAFE_API_KEY:-}" ]; then - echo "::error::No TypeSafe API key. Pass api-key: \${{ secrets.TYPESAFE_API_KEY }}. Pull requests from forks don't receive secrets; skip the job for them." +if [ "$code" = 2 ] && [ -z "${!key_variable:-}" ]; then + echo "::error::No API key. Pass api-key: \${{ secrets.$key_variable }}. Pull requests from forks don't receive secrets; skip the job for them." fi finish "$code" diff --git a/test/key-kinds.sh b/test/key-kinds.sh new file mode 100644 index 0000000..1d6cf54 --- /dev/null +++ b/test/key-kinds.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# check.sh gives JevGate api-key in the variable it reads for api-key-kind, +# and no other kind's key; it stops before checking when the installed +# JevGate predates gateway keys. A stand-in jevgate records the key +# variables it was given. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir "$work/bin" +cat > "$work/bin/jevgate" << 'EOF' +#!/usr/bin/env bash +if [ "$1" = --version ]; then + echo "jevgate $STUB_VERSION" + exit 0 +fi +for name in TYPESAFE_API_KEY OPENROUTER_API_KEY AI_GATEWAY_API_KEY; do + if [ -n "${!name:-}" ]; then echo "$name=${!name}"; fi +done > "$STUB_SEEN" +EOF +chmod +x "$work/bin/jevgate" + +# The exit code of check.sh given api-key KEY of KIND with JevGate VERSION and +# the job's key variables VAR=VALUE..., then the key variables the check saw. +check() { + local kind=$1 version=$2 key=$3 code=0 seen + shift 3 + : > "$work/seen" + (cd "$work" && env -u TYPESAFE_API_KEY -u OPENROUTER_API_KEY -u AI_GATEWAY_API_KEY "$@" \ + PATH="$work/bin:$PATH" STUB_VERSION="$version" STUB_SEEN="$work/seen" \ + GITHUB_OUTPUT="$work/output" RUNNER_TEMP="$work" BASE= FORMAT=agent ARGS= \ + SARIF_FILE= API_KEY="$key" API_KEY_KIND="$kind" bash "$here/../check.sh" > /dev/null 2>&1) || code=$? + seen=$(tr '\n' ' ' < "$work/seen") + echo "exit $code${seen:+ ${seen% }}" +} + +failed=0 +# expect EXPECTED KIND VERSION KEY [VAR=VALUE...] +expect() { + local expected=$1 actual + shift + actual=$(check "$@") + if [ "$actual" != "$expected" ]; then + echo "::error::check $*: expected '$expected', got '$actual'" + failed=1 + fi +} + +expect "exit 0 TYPESAFE_API_KEY=k" typesafe 0.25.0 k +expect "exit 0 OPENROUTER_API_KEY=k" openrouter 0.26.0 k +expect "exit 0 AI_GATEWAY_API_KEY=k" vercel 0.26.0 k +expect "exit 0 AI_GATEWAY_API_KEY=k" vercel 1.0.0 k +expect "exit 2" openrouter 0.25.0 k +expect "exit 2" anthropic 0.26.0 k +# Keys the job holds for other services never reach JevGate. +expect "exit 0 TYPESAFE_API_KEY=k" typesafe 0.26.0 k OPENROUTER_API_KEY=job AI_GATEWAY_API_KEY=job +# With no api-key, the job's own key of that kind is used. +expect "exit 0 OPENROUTER_API_KEY=job" openrouter 0.26.0 "" OPENROUTER_API_KEY=job TYPESAFE_API_KEY=job +exit "$failed" From 5841788008ef5dba9838d5e189bee5eb9471903c Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:31:48 -0300 Subject: [PATCH 4/9] List every finding in one pull request comment GitHub shows at most 10 error and 10 warning annotations per step, so a pull request with more findings showed only some of them on the diff. On pull request events the action now posts one comment and updates it on each run: the gate's result and reasons, the run's files, API requests, input tokens and cost, then every finding nobody accepted, by level (reviews open, considers collapsed past 10, notes collapsed) and by file, each linked to its line at the checked commit. A run that could not finish (exit 2: no key, a provider error such as HTTP 402, the request budget) opens with a caution alert and each distinct reason with the files it stopped. - render.cjs writes it from the JSON report alone, so it works with any JevGate version, and comment.cjs posts it, run by actions/github-script pinned by SHA on any runner that runs JavaScript actions. check.sh saves this run's report before the SARIF replay replaces it (the replay sends no request, so its report shows no cost), and only when the check wrote a new one (checksum before and after), never a report left from an earlier check. - A hidden first line keys the comment by job and working directory, and names the run that wrote it: a run for an older push leaves a newer run's comment alone, and copies left by two first runs racing are deleted. Only bot comments starting with the marker are edited. - The body stays under GitHub's 65,536-character limit, measured in UTF-8 bytes: notes are cut first, then the lowest-ranked considers, with a line saying how many of each are left out. On the largest corpus reports (1,500 to 1,750 findings of whole-repository runs) it keeps every review and fills 64,645 to 65,091 bytes in under 30 ms. - Text from the report quotes the change's code, so outside code spans it is escaped: no HTML, links, mentions or comment markers, and an unclosed backtick cannot pair with the next field's code span. - Without pull-requests: write, as on pull requests from forks, it says so in the log and the job summary and the step passes; any other failure is a warning too. comment: false turns it off. Tested with node --test (renders JevGate's own --base report and 0.25.0 reports of a run without a key and after a 402, a fake issues API for create, update, duplicates, older runs and refusals) and in CI: on four runners a read-only token's refused comment must not fail a passing check, and on pull requests from this repository two checks in one job must leave one comment holding the second's incomplete banner. --- .github/workflows/test.yml | 58 +- README.md | 14 +- action.yml | 21 + check.sh | 15 + comment.cjs | 122 +++ render.cjs | 385 +++++++++ test/comment.test.cjs | 194 +++++ test/fixture-repository.sh | 4 + test/render.test.cjs | 245 ++++++ test/reports/base-run.json | 1510 ++++++++++++++++++++++++++++++++++++ test/reports/http-402.json | 133 ++++ test/reports/no-key.json | 133 ++++ 12 files changed, 2828 insertions(+), 6 deletions(-) create mode 100644 comment.cjs create mode 100644 render.cjs create mode 100644 test/comment.test.cjs create mode 100644 test/render.test.cjs create mode 100644 test/reports/base-run.json create mode 100644 test/reports/http-402.json create mode 100644 test/reports/no-key.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c51290..3714407 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,9 @@ jobs: - name: Key kinds shell: bash run: bash test/key-kinds.sh + - name: Comment + shell: bash + run: node --test test/render.test.cjs test/comment.test.cjs dry-run: strategy: fail-fast: false @@ -45,7 +48,7 @@ jobs: [ "$CODE" = 0 ] || { echo "::error::exit code $CODE"; exit 1; } jevgate --version grep -q '"version": "2.1.0"' jevgate.sarif || { echo "::error::no SARIF log"; exit 1; } - incomplete: + fixture: strategy: fail-fast: false matrix: @@ -58,7 +61,7 @@ jobs: - name: Create a repository to check shell: bash run: bash test/fixture-repository.sh fixture - - id: jevgate + - id: incomplete uses: ./ continue-on-error: true with: @@ -66,12 +69,57 @@ jobs: base: HEAD~1 args: --cache-only cache: "false" + # Nothing changed since HEAD: the gate passes. On a pull request the + # read-only token cannot comment, which must not fail the step. + - id: passed + uses: ./ + with: + working-directory: fixture + base: HEAD + cache: "false" - name: Check the outputs shell: bash env: - CODE: ${{ steps.jevgate.outputs.exit-code }} - REPORT: ${{ steps.jevgate.outputs.report }} + INCOMPLETE: ${{ steps.incomplete.outputs.exit-code }} + PASSED: ${{ steps.passed.outputs.exit-code }} + REPORT: ${{ steps.incomplete.outputs.report }} run: | - [ "$CODE" = 2 ] || { echo "::error::exit code $CODE"; exit 1; } + [ "$INCOMPLETE" = 2 ] || { echo "::error::exit code $INCOMPLETE, not 2"; exit 1; } + [ "$PASSED" = 0 ] || { echo "::error::exit code $PASSED, not 0"; exit 1; } # The path must open outside bash too, as upload-artifact opens it. node -e 'require("fs").accessSync(process.argv[1])' "$REPORT" || { echo "::error::no report at $REPORT"; exit 1; } + comment: + # It writes to the pull request, so only where the token may. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Create a repository to check + shell: bash + run: bash test/fixture-repository.sh fixture + # Two checks in one job share one comment: the second replaces the first. + - uses: ./ + with: + working-directory: fixture + base: HEAD + cache: "false" + - uses: ./ + continue-on-error: true + with: + working-directory: fixture + base: HEAD~1 + args: --cache-only + cache: "false" + - name: Check the comment + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: | + bodies=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST/comments" \ + --jq '.[] | select(.user.type == "Bot" and (.body | startswith("/.exec(body); + return run ? BigInt(run[1]) : 0n; +} + +/** Text outside code spans, escaped so it stays text: no links, HTML or mentions. */ +function escapeText(text) { + return text + .replace(/[\\[\]]/g, '\\$&') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/@(?=\w)/g, '@\u200b'); +} + +/** The length of the run of backticks at `index`. */ +function backticks(text, index) { + let end = index; + while (text[end] === '`') { + end += 1; + } + return end - index; +} + +/** Where a run of exactly `width` backticks starts at or after `from`, or -1. */ +function closing(text, from, width) { + for (let at = text.indexOf('`', from); at >= 0; ) { + const run = backticks(text, at); + if (run === width) { + return at; + } + at = text.indexOf('`', at + run); + } + return -1; +} + +/** Report text on one line of Markdown. Messages quote code from the change, so + * outside code spans nothing can add HTML, links, a comment marker or mentions; + * code spans are kept as JevGate wrote them. They are found as CommonMark finds + * them, a run of backticks closed by a run of the same length, so text can + * never pass as code. */ +function inline(text) { + const flat = String(text).replace(/\s*[\r\n]+\s*/g, ' '); + let out = ''; + let at = 0; + for (let open = flat.indexOf('`'); open >= 0; open = flat.indexOf('`', at)) { + const width = backticks(flat, open); + const close = closing(flat, open + width, width); + if (close < 0) { + // An unclosed run is escaped: left as it is, it could close on a + // backtick of the next text in the same line and expose that + // text's code span as Markdown and HTML. + out += escapeText(flat.slice(at, open)) + '\\`'.repeat(width); + at = open + width; + } else { + out += escapeText(flat.slice(at, open)) + flat.slice(open, close + width); + at = close + width; + } + } + return out + escapeText(flat.slice(at)); +} + +/** `text` as one code span, fenced by more backticks than any run it holds. */ +function code(text) { + const flat = String(text).replace(/[\r\n]+/g, ' '); + const longest = Math.max(0, ...(flat.match(/`+/g) || []).map((run) => run.length)); + const fence = '`'.repeat(longest + 1); + const pad = flat.startsWith('`') || flat.endsWith('`') ? ' ' : ''; + return `${fence}${pad}${flat}${pad}${fence}`; +} + +/** A path in a URL: each segment encoded, parentheses too, so it cannot end a + * Markdown link early. */ +function urlPath(path) { + const encode = (segment) => + encodeURIComponent(segment).replace(/[()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`); + return path.split('/').map(encode).join('/'); +} + +/** A report path as a repository path, with forward slashes. */ +function repositoryPath(path, run) { + return run.prefix + (run.windows ? path.replace(/\\/g, '/') : path); +} + +/** A link to `path` at the checked commit, with a line anchor or none. */ +function blobUrl(path, anchor, run) { + return `${run.serverUrl}/${run.repository}/blob/${run.commit}/${urlPath(repositoryPath(path, run))}${anchor}`; +} + +/** Exit codes 0 and 1 mean the gate was applied; any other code stopped the run. */ +function finished(exitCode) { + return exitCode === 0 || exitCode === 1; +} + +/** Findings nobody accepted, at a known level, with their file's path. */ +function listed(report) { + const known = new Set(LEVELS.map((level) => level.strength)); + return (report.files || []).flatMap((file) => + (file.findings || []) + .filter((finding) => known.has(finding.strength) && !finding.baselined && !finding.suppressed) + .map((finding) => ({ path: file.path, finding })), + ); +} + +function levelIndex(strength) { + return LEVELS.findIndex((level) => level.strength === strength); +} + +/** Findings in the order they are kept when the comment must be cut: by level, + * then by JevGate's rank. */ +function byPriority(findings) { + return [...findings].sort( + (a, b) => + levelIndex(a.finding.strength) - levelIndex(b.finding.strength) || + (b.finding.rank || 0) - (a.finding.rank || 0), + ); +} + +function title(exitCode) { + if (!finished(exitCode)) { + return '### JevGate: run incomplete'; + } + return exitCode === 1 ? '### JevGate: gate failed' : '### JevGate: gate passed'; +} + +/** Distinct reasons the run could not finish, with the number of files each + * stopped: run errors first (no files), then the errors of files not judged. */ +function reasons(report) { + const files = new Map(); + for (const error of report.errors || []) { + files.set(error, 0); + } + for (const file of report.files || []) { + if (file.status === 'error') { + const error = file.error || 'Not judged'; + files.set(error, (files.get(error) || 0) + 1); + } + } + return [...files]; +} + +/** The loud part of an incomplete run: what it means, then why. */ +function banner(report, run) { + const lines = [ + '> [!CAUTION]', + `> **JevGate could not finish this run (exit code ${run.exitCode}).** The gate was not applied, and findings may be missing.`, + ]; + const why = report ? reasons(report) : []; + for (const [reason, files] of why.slice(0, MAX_REASONS)) { + lines.push(`> - ${inline(reason)}${files ? ` (${count(files, 'file')})` : ''}`); + } + const others = why.slice(MAX_REASONS).reduce((sum, [, files]) => sum + files, 0); + if (others > 0) { + lines.push(`> - ${count(others, 'more file')} stopped for other reasons.`); + } + if (!report) { + lines.push(`>\n> JevGate stopped before writing a report; the [job log](${run.runUrl}) says why.`); + } else if (why.length === 0) { + lines.push(`>\n> The [job log](${run.runUrl}) says why.`); + } + return lines.join('\n'); +} + +/** This run's cost: paid input tokens in dollars for the priced model, "cost + * unknown" when requests were answered without a priced token count (a + * gateway may answer without usage), or `null` when nothing was paid for. */ +function cost(report) { + const tokens = report.paid_input_tokens || 0; + if (tokens === 0) { + const answered = Object.values(report.stages || {}).reduce( + (sum, stage) => sum + (stage.successful_requests || 0), + 0, + ); + return answered > 0 ? 'cost unknown' : null; + } + // The model that answered, else the one asked for. + const models = new Set((report.files || []).map((file) => file.model).filter(Boolean)); + const model = models.size > 0 ? [...models].join(', ') : report.requested_model; + if (model !== PRICED_MODEL) { + return 'cost unknown'; + } + return `~$${((tokens * USD_PER_MILLION_INPUT_TOKENS) / 1e6).toFixed(4)}`; +} + +/** The gate's reasons as JevGate gave them, the run's size and cost, and the + * files left undecided. */ +function outcome(report) { + const lines = []; + const gate = report.gate; + if (gate && !gate.passed && (gate.reasons || []).length > 0) { + lines.push(`Gate failed: ${gate.reasons.map(inline).join('; ')}.`); + } + const files = report.files || []; + const usage = [ + count(files.length, 'file'), + count(report.api_requests || 0, 'API request'), + count(report.paid_input_tokens || 0, 'input token'), + cost(report), + ]; + lines.push(usage.filter(Boolean).join(' · ')); + const uncertain = files.filter((file) => + Object.values(file.dimensions || {}).some((dimension) => dimension.status === 'uncertain'), + ).length; + const context = files.filter((file) => file.status === 'needs-context').length; + const open = [ + uncertain > 0 ? `${count(uncertain, 'file')} with uncertain units` : '', + context > 0 ? `${count(context, 'file')} needing context` : '', + ].filter(Boolean); + if (open.length > 0) { + lines.push(`${open.join(' · ')}.`); + } + return lines.join('\n\n'); +} + +/** One finding: its line, linked when the commit is known, the rule, the + * message and the next step. */ +function item({ path, finding }, run) { + const location = (finding.locations || []).find( + (l) => l.path === path && l.start_line === finding.line && l.end_line > finding.line, + ); + const anchor = location ? `#L${finding.line}-L${location.end_line}` : `#L${finding.line}`; + const line = run.commit ? `[Line ${finding.line}](${blobUrl(path, anchor, run)})` : `Line ${finding.line}`; + return `- ${line} ${code(finding.rule)}: ${inline(finding.message)}
→ ${inline(finding.action)}`; +} + +/** A file's heading and its findings by line. */ +function fileBlock(path, entries, run) { + const name = code(repositoryPath(path, run)); + const heading = run.commit ? `[${name}](${blobUrl(path, '', run)})` : name; + const items = [...entries].sort((a, b) => a.finding.line - b.finding.line).map((entry) => item(entry, run)); + return [`**${heading}**`, ...items].join('\n'); +} + +/** One level's findings grouped by file, the files by path as the pull + * request lists them. Reviews are open; notes, and considers past + * `OPEN_CONSIDERS`, are collapsed. */ +function section(level, shown, total, run) { + const byPath = new Map(); + for (const entry of shown) { + const group = byPath.get(entry.path); + if (group) { + group.push(entry); + } else { + byPath.set(entry.path, [entry]); + } + } + const blocks = [...byPath.keys()].sort().map((path) => fileBlock(path, byPath.get(path), run)); + const counted = shown.length === total ? number(total) : `${number(shown.length)} of ${number(total)}`; + if (level.strength === 'review' || (level.strength === 'consider' && total <= OPEN_CONSIDERS)) { + return [`#### ${level.title} (${counted})`, ...blocks].join('\n\n'); + } + const optional = level.strength === 'note' ? ', optional' : ''; + return [`
${level.title} (${counted}${optional})`, ...blocks, '
'].join('\n\n'); +} + +/** A section for each level with findings kept, then a line for what was cut. */ +function sections(kept, all, run) { + const parts = []; + const cut = []; + for (const level of LEVELS) { + const atLevel = (entry) => entry.finding.strength === level.strength; + const shown = kept.filter(atLevel); + const total = all.filter(atLevel).length; + if (shown.length > 0) { + parts.push(section(level, shown, total, run)); + } + if (total > shown.length) { + cut.push(count(total - shown.length, level.noun)); + } + } + if (cut.length > 0) { + parts.push( + `**${count(all.length - kept.length, 'more finding')}** did not fit in this comment: ${cut.join(', ')}. The JSON report (the action's \`report\` output) lists them all.`, + ); + } + return parts; +} + +function footer(run) { + const parts = [run.version ? inline(run.version) : 'JevGate']; + if (run.commit) { + parts.push(`commit ${run.commit.slice(0, 7)}`); + } + parts.push(`[workflow run](${run.runUrl})`, 'updated on each run'); + return `${parts.join(' · ')}`; +} + +/** The comment for `report` (`null` when the run wrote none), listing the + * findings in `kept` of `all`. */ +function compose(report, run, kept, all) { + const parts = [`${marker(run.key)}run=${run.runId} -->\n${title(run.exitCode)}`]; + if (!finished(run.exitCode)) { + parts.push(banner(report, run)); + } + if (report && report.status === 'no-changed-source') { + parts.push('No supported file changed since the base revision.'); + } else if (report) { + parts.push(outcome(report)); + if (finished(run.exitCode) && !all.some((entry) => entry.finding.strength !== 'note')) { + parts.push('No new review or consider findings.'); + } + parts.push(...sections(kept, all, run)); + const accepted = (report.files || []) + .flatMap((file) => file.findings || []) + .filter((finding) => finding.baselined || finding.suppressed).length; + if (accepted > 0) { + const verb = accepted === 1 ? 'is' : 'are'; + parts.push(`${count(accepted, 'finding')} accepted by the baseline or an inline allow ${verb} not listed.`); + } + } + parts.push(footer(run)); + return `${parts.join('\n\n')}\n`; +} + +/** The comment for a report, or for a run that wrote none (`report` null). + * It lists as many findings as fit under GitHub's limit, cutting the lowest + * ranked first: notes before considers before reviews. */ +function render(report, run) { + const all = report ? byPriority(listed(report)) : []; + const body = (n) => compose(report, run, all.slice(0, n), all); + const fits = (n) => Buffer.byteLength(body(n), 'utf8') <= MAX_BODY_BYTES; + if (fits(all.length)) { + return body(all.length); + } + // Bisect over the counts that leave a "N more" line: each finding kept + // adds a line longer than the digits it saves there, so the size grows + // with the count. + let low = 0; + let high = all.length - 1; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (fits(middle)) { + low = middle; + } else { + high = middle - 1; + } + } + return body(low); +} + +module.exports = { render, marker, runOf, finished, inline, code, MAX_BODY_BYTES }; diff --git a/test/comment.test.cjs b/test/comment.test.cjs new file mode 100644 index 0000000..9ba47b4 --- /dev/null +++ b/test/comment.test.cjs @@ -0,0 +1,194 @@ +// Tests for posting the comment: `node --test test/comment.test.cjs`, with an +// in-memory issues API standing in for GitHub. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { run, commentKey } = require('../comment.cjs'); +const { marker } = require('../render.cjs'); + +const COMMIT = '0123456789abcdef0123456789abcdef01234567'; + +test('each job and working directory has its own comment', () => { + assert.equal(commentKey('review', '.'), 'review'); + assert.equal(commentKey('review', './'), 'review'); + assert.equal(commentKey('review', 'services/api'), 'review services/api'); + assert.equal(commentKey('review', './services/api/'), 'review services/api'); +}); + +const BOT = { login: 'github-actions[bot]', type: 'Bot' }; + +function refusal(status, message) { + return Object.assign(new Error(message), { status }); +} + +function fakeGitHub(comments = [], failures = {}) { + const calls = []; + let next = 1000; + const url = (id) => `https://github.com/o/r/pull/7#issuecomment-${id}`; + const fail = (name) => { + if (failures[name]) { + throw failures[name]; + } + }; + const issues = { + async listComments(params) { + calls.push(['list', params.issue_number, params.per_page]); + fail('list'); + return { data: comments.map((c) => ({ ...c, html_url: url(c.id) })) }; + }, + async createComment({ issue_number, body }) { + calls.push(['create', issue_number]); + fail('create'); + const comment = { id: next++, body, user: BOT }; + comments.push(comment); + return { data: { ...comment, html_url: url(comment.id) } }; + }, + async updateComment({ comment_id, body }) { + calls.push(['update', comment_id]); + fail('update'); + comments.find((c) => c.id === comment_id).body = body; + return { data: { id: comment_id, html_url: url(comment_id) } }; + }, + async deleteComment({ comment_id }) { + calls.push(['delete', comment_id]); + fail('delete'); + comments.splice(comments.findIndex((c) => c.id === comment_id), 1); + return {}; + }, + }; + const github = { rest: { issues }, paginate: async (method, params) => (await method(params)).data }; + return { github, comments, calls }; +} + +function fakeCore() { + const log = { info: [], warning: [], summary: [] }; + const summary = { + addRaw(text) { + log.summary.push(text); + return summary; + }, + async write() { + return summary; + }, + }; + return { core: { info: (m) => log.info.push(m), warning: (m) => log.warning.push(m), summary }, log }; +} + +function context(overrides = {}) { + return { + payload: { pull_request: { number: 7, head: { repo: { full_name: 'o/r' } } } }, + repo: { owner: 'o', repo: 'r' }, + job: 'review', + runId: 123, + serverUrl: 'https://github.com', + ...overrides, + }; +} + +function env(name, exitCode, extra = {}) { + return { + JEVGATE_REPORT: name ? path.join(__dirname, 'reports', `${name}.json`) : '', + JEVGATE_EXIT_CODE: String(exitCode), + JEVGATE_COMMIT: COMMIT, + JEVGATE_PREFIX: '', + JEVGATE_VERSION: 'jevgate 0.25.0', + JEVGATE_WORKING_DIRECTORY: '.', + ...extra, + }; +} + +async function step({ comments, failures, ctx = context(), environment = env('base-run', 1) } = {}) { + const fake = fakeGitHub(comments, failures); + const { core, log } = fakeCore(); + await run({ github: fake.github, context: ctx, core, env: environment, platform: 'linux' }); + return { ...fake, log }; +} + +test('the first run creates the comment and links it from the job summary', async () => { + const { comments, calls, log } = await step(); + assert.deepEqual(calls, [['list', 7, 100], ['create', 7]]); + assert.equal(comments.length, 1); + assert.ok(comments[0].body.startsWith('\n### JevGate: gate failed')); + assert.deepEqual(log.warning, []); + assert.ok(log.summary[0].includes('[a pull request comment](https://github.com/o/r/pull/7#issuecomment-1000)')); +}); + +test('a later run updates its own comment in place and leaves others alone', async () => { + const others = [ + { id: 1, body: `${marker('review')}run=100 -->\nquoted by a person`, user: { login: 'someone', type: 'User' } }, + { id: 2, body: `${marker('security')}run=100 -->\nanother job`, user: BOT }, + { id: 3, body: `${marker('review')}run=100 -->\nold findings`, user: BOT }, + ]; + const { comments, calls } = await step({ comments: others }); + assert.deepEqual(calls.slice(1), [['update', 3]]); + assert.ok(comments.find((c) => c.id === 3).body.includes('### JevGate: gate failed')); + assert.ok(comments.find((c) => c.id === 1).body.includes('quoted by a person')); + assert.ok(comments.find((c) => c.id === 2).body.includes('another job')); +}); + +test('an unchanged comment is not written again', async () => { + const first = await step(); + const again = await step({ comments: first.comments }); + assert.deepEqual(again.calls, [['list', 7, 100]]); +}); + +test('copies left by racing first runs are deleted, keeping the oldest', async () => { + const racing = [4, 5, 6].map((id) => ({ id, body: `${marker('review')}run=123 -->\nrace`, user: BOT })); + const { comments, calls } = await step({ comments: racing }); + assert.deepEqual(calls.slice(1), [['update', 4], ['delete', 5], ['delete', 6]]); + assert.deepEqual(comments.map((c) => c.id), [4]); +}); + +test('a run older than the comment leaves it alone', async () => { + const newer = [{ id: 8, body: `${marker('review')}run=124 -->\nnewer`, user: BOT }]; + const { comments, calls, log } = await step({ comments: newer }); + assert.deepEqual(calls, [['list', 7, 100]]); + assert.ok(comments[0].body.endsWith('newer')); + assert.ok(log.info.some((m) => m.includes('newer run'))); +}); + +test('a read-only token is a warning in the log and the summary, not a failure', async () => { + const failures = { create: refusal(403, 'Resource not accessible by integration') }; + const { log } = await step({ failures }); + assert.equal(log.warning.length, 1); + assert.ok(log.warning[0].startsWith('The token cannot comment on this pull request: give the job `permissions: pull-requests: write`, or set `comment: false`.')); + assert.ok(log.warning[0].includes('(GitHub: Resource not accessible by integration)')); + assert.ok(log.summary[0].includes('pull-requests: write')); +}); + +test('a fork is told why it gets no comment', async () => { + const fork = context({ payload: { pull_request: { number: 7, head: { repo: { full_name: 'someone/r' } } } } }); + const { log } = await step({ ctx: fork, failures: { list: refusal(404, 'Not Found') } }); + assert.ok(log.warning[0].startsWith('GitHub gives workflows on pull requests from forks a read-only token')); +}); + +test('any other failure is a warning, never an exception', async () => { + const failed = await step({ failures: { update: refusal(500, 'Server Error') }, comments: [{ id: 3, body: `${marker('review')}run=1 -->`, user: BOT }] }); + assert.deepEqual(failed.log.warning, ['JevGate could not update its pull request comment: Server Error']); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'jevgate-comment-')); + const broken = path.join(directory, 'latest.json'); + fs.writeFileSync(broken, '{"files": ['); + const unreadable = await step({ environment: env('', 1, { JEVGATE_REPORT: broken }) }); + assert.equal(unreadable.calls.length, 0); + assert.ok(unreadable.log.warning[0].startsWith('JevGate could not update its pull request comment:')); +}); + +test('events without a pull request, and runs without a report, post nothing', async () => { + const push = await step({ ctx: context({ payload: {} }) }); + assert.equal(push.calls.length, 0); + const dryRun = await step({ environment: env('', 0) }); + assert.equal(dryRun.calls.length, 0); + assert.ok(dryRun.log.info[0].includes('no comment')); + const stopped = await step({ environment: env('', 2) }); + assert.deepEqual(stopped.calls, [['list', 7, 100], ['create', 7]]); + assert.ok(stopped.comments[0].body.includes('JevGate stopped before writing a report')); +}); + +test('a job checking a subdirectory keys its comment by it', async () => { + const { comments } = await step({ environment: env('base-run', 1, { JEVGATE_WORKING_DIRECTORY: 'services/api' }) }); + assert.ok(comments[0].body.startsWith('')); +}); diff --git a/test/fixture-repository.sh b/test/fixture-repository.sh index 65bca2c..5e42062 100644 --- a/test/fixture-repository.sh +++ b/test/fixture-repository.sh @@ -5,6 +5,10 @@ set -euo pipefail dir=$1 +if [ -e "$dir" ]; then + echo "$dir already exists" >&2 + exit 1 +fi git -c init.defaultBranch=main init -q "$dir" cd "$dir" commit() { diff --git a/test/render.test.cjs b/test/render.test.cjs new file mode 100644 index 0000000..fdcdab7 --- /dev/null +++ b/test/render.test.cjs @@ -0,0 +1,245 @@ +// Tests for the comment's text: `node --test test/render.test.cjs`. The +// reports under test/reports are JevGate's own: a --base run on its +// repository, and 0.25.0 runs without a key and after an HTTP 402. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { render, inline, code, marker, MAX_BODY_BYTES } = require('../render.cjs'); + +const COMMIT = '0123456789abcdef0123456789abcdef01234567'; +const RUN = { + key: 'review', + runId: '123', + runUrl: 'https://github.com/o/r/actions/runs/123', + serverUrl: 'https://github.com', + repository: 'o/r', + exitCode: 1, + commit: COMMIT, + prefix: '', + version: 'jevgate 0.25.0', + windows: false, +}; + +function report(name) { + return JSON.parse(fs.readFileSync(path.join(__dirname, 'reports', `${name}.json`), 'utf8')); +} + +function finding(strength, line, extra = {}) { + return { + rule: 'maintainability/shared-logic', + strength, + line, + message: `Finding at line ${line}`, + action: 'Share one implementation', + rank: 1, + locations: [], + baselined: false, + ...extra, + }; +} + +/** A complete report with these files: `{ path: [findings] }`. */ +function reportWith(files, extra = {}) { + return { + status: 'review', + complete: true, + errors: [], + gate: { passed: false, reasons: ['1 new review finding'] }, + api_requests: 3, + paid_input_tokens: 1000, + requested_model: 'jev-1.13.0', + stages: {}, + files: Object.entries(files).map(([file, findings]) => ({ path: file, status: 'review', findings })), + ...extra, + }; +} + +test('a real --base report lists every finding by level, then by file and line', () => { + const body = render(report('base-run'), RUN); + assert.ok(body.startsWith('\n### JevGate: gate failed\n')); + assert.match(body, /\nGate failed: 1 new review finding\(s\)\.\n/); + assert.match(body, /\n6 files · 55 API requests · 160,708 input tokens · ~\$0\.0067\n/); + assert.match(body, /\n3 files with uncertain units\.\n/); + const review = body.indexOf('#### Review (1)'); + const consider = body.indexOf('#### Consider (1)'); + const notes = body.indexOf('
Notes (11, optional)'); + assert.ok(review > 0 && consider > review && notes > consider, body); + assert.ok(body.includes( + `- [Line 213](https://github.com/o/r/blob/${COMMIT}/src/output.rs#L213-L217) \`maintainability/shared-logic\`: Lines 213 and 247 of \`emit_findings\``, + )); + // Notes: files by path, findings by line. + const noteText = body.slice(notes); + const files = [...noteText.matchAll(/\*\*\[`([^`]+)`\]/g)].map((m) => m[1]); + assert.deepEqual(files, ['src/options/commands.rs', 'src/options/mod.rs', 'src/output.rs']); + const lines = [...noteText.slice(noteText.indexOf('src/output.rs')).matchAll(/\[Line (\d+)\]/g)].map((m) => Number(m[1])); + assert.deepEqual(lines, [...lines].sort((a, b) => a - b)); + assert.ok(body.endsWith(`jevgate 0.25.0 · commit 0123456 · [workflow run](${RUN.runUrl}) · updated on each run\n`)); +}); + +test('a run without a key says loudly that the gate was not applied, and why', () => { + const body = render(report('no-key'), { ...RUN, exitCode: 2 }); + assert.ok(body.includes('### JevGate: run incomplete\n\n> [!CAUTION]\n> **JevGate could not finish this run (exit code 2).** The gate was not applied, and findings may be missing.\n')); + assert.ok(body.includes('> - No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH (2 files)\n')); + assert.ok(body.includes('\n2 files · 0 API requests · 0 input tokens\n')); + assert.ok(!body.includes('No new review or consider findings'), 'nothing was judged, so nothing is clean'); +}); + +test('an HTTP 402 lists each distinct reason with the files it stopped', () => { + const body = render(report('http-402'), { ...RUN, exitCode: 2 }); + assert.ok(body.includes('> - TypeSafe HTTP 402; request was not retried (1 file)\n')); + assert.ok(body.includes('> - TypeSafe request not sent after HTTP 402; restore account access and rerun the review (1 file)\n')); + assert.ok(body.includes('2 files · 1 API request · 0 input tokens')); +}); + +test('run errors come first, and reasons past ten are counted', () => { + const files = Array.from({ length: 14 }, (_, i) => ({ path: `f${i}.js`, status: 'error', error: `Reason ${i}`, findings: [] })); + const body = render(reportWith({}, { complete: false, gate: null, errors: ['Session API request budget exhausted'], files }), { ...RUN, exitCode: 2 }); + const reasons = body.split('\n').filter((line) => line.startsWith('> - ')); + assert.equal(reasons[0], '> - Session API request budget exhausted'); + assert.equal(reasons.length, 11); + assert.equal(reasons[10], '> - 5 more files stopped for other reasons.'); +}); + +test('a run that wrote no report still says it stopped', () => { + const body = render(null, { ...RUN, exitCode: 2 }); + assert.ok(body.includes(`> JevGate stopped before writing a report; the [job log](${RUN.runUrl}) says why.`)); + assert.ok(!body.includes('API request')); +}); + +test('a passing gate with nothing new says so', () => { + const body = render(reportWith({ 'a.js': [finding('note', 3)] }, { status: 'note', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); + assert.ok(body.includes('### JevGate: gate passed\n')); + assert.ok(!body.includes('Gate failed')); + assert.ok(body.includes('No new review or consider findings.')); + assert.ok(body.includes('
Notes (1, optional)')); +}); + +test('no changed source is said plainly', () => { + const body = render(reportWith({}, { status: 'no-changed-source', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); + assert.ok(body.includes('No supported file changed since the base revision.')); +}); + +test('accepted findings are counted, not listed', () => { + const files = { + 'a.js': [finding('review', 1, { baselined: true }), finding('consider', 2, { suppressed: 'generated' }), finding('consider', 9)], + }; + const body = render(reportWith(files), RUN); + assert.ok(!body.includes('Finding at line 1') && !body.includes('Finding at line 2')); + assert.ok(body.includes('Finding at line 9')); + assert.ok(body.includes('2 findings accepted by the baseline or an inline allow are not listed.')); + assert.ok(body.includes('No new review or consider findings.') === false); +}); + +test('more than ten considers are collapsed; reviews never are', () => { + const many = Array.from({ length: 11 }, (_, i) => finding('consider', i + 1)); + const reviews = Array.from({ length: 30 }, (_, i) => finding('review', i + 100)); + const body = render(reportWith({ 'a.js': [...many, ...reviews] }), RUN); + assert.ok(body.includes('#### Review (30)')); + assert.ok(body.includes('
Consider (11)')); + const ten = render(reportWith({ 'a.js': many.slice(1) }), RUN); + assert.ok(ten.includes('#### Consider (10)')); +}); + +test('cost is priced for jev-1.13.0 only, and unknown when answers carried no usage', () => { + const priced = render(reportWith({}, { paid_input_tokens: 1_000_000 }), RUN); + assert.ok(priced.includes('1,000,000 input tokens · ~$0.0420')); + const answered = { files: [{ path: 'a.js', status: 'clear', model: 'jev-1.13.0', findings: [] }] }; + assert.ok(render(reportWith({}, { requested_model: 'jev-latest', ...answered }), RUN).includes('~$0.0000')); + const other = { files: [{ path: 'a.js', status: 'clear', model: 'jev-2.0.0', findings: [] }] }; + assert.ok(render(reportWith({}, other), RUN).includes('1,000 input tokens · cost unknown')); + const noUsage = { paid_input_tokens: 0, stages: { functions: { successful_requests: 4 } } }; + assert.ok(render(reportWith({}, noUsage), RUN).includes('0 input tokens · cost unknown')); + const cached = render(reportWith({}, { paid_input_tokens: 0, api_requests: 0 }), RUN); + assert.ok(cached.includes('0 input tokens\n') && !cached.includes('$') && !cached.includes('cost unknown')); +}); + +test('a big run is cut under GitHub\'s limit, lowest ranked and least severe first', () => { + const long = 'A message long enough to matter, quoting `some_function_name` and 漢字 text. '.repeat(3); + const files = {}; + for (let f = 0; f < 60; f += 1) { + files[`src/module_${f}/file.rs`] = [ + finding('review', 10, { message: long, rank: 100 + f }), + ...Array.from({ length: 10 }, (_, i) => finding('consider', 20 + i, { message: long, rank: f + i })), + ...Array.from({ length: 40 }, (_, i) => finding('note', 100 + i, { message: long, rank: 1000 })), + ]; + } + const body = render(reportWith(files), RUN); + assert.ok(Buffer.byteLength(body, 'utf8') <= MAX_BODY_BYTES); + assert.ok(Buffer.byteLength(body, 'utf8') > MAX_BODY_BYTES - 1000, 'it uses the room it has'); + assert.ok(body.includes('#### Review (60)'), 'every review is kept'); + const considers = /Consider \((\d+) of 600\)/.exec(body); + assert.ok(considers, body.slice(0, 2000)); + const shown = Number(considers[1]); + assert.ok(!body.includes('Notes ('), 'notes go before any consider'); + assert.ok(body.includes(`**${(600 - shown + 2400).toLocaleString('en-US')} more findings** did not fit in this comment: ${600 - shown} consider findings, 2,400 notes.`)); + // No consider cut outranks one kept (a consider's rank is its module plus its index). + const kept = new Set( + [...body.matchAll(/\[Line (2\d)\]\(https:\/\/github\.com\/o\/r\/blob\/[0-9a-f]+\/src\/module_(\d+)/g)].map( + (m) => `${m[2]}:${Number(m[1]) - 20}`, + ), + ); + assert.equal(kept.size, shown); + const ranks = { kept: [], cut: [] }; + for (let f = 0; f < 60; f += 1) { + for (let i = 0; i < 10; i += 1) { + ranks[kept.has(`${f}:${i}`) ? 'kept' : 'cut'].push(f + i); + } + } + assert.ok(Math.min(...ranks.kept) >= Math.max(...ranks.cut), `${Math.min(...ranks.kept)} < ${Math.max(...ranks.cut)}`); +}); + +test('a run whose findings all fit is not cut', () => { + const files = { 'a.js': Array.from({ length: 50 }, (_, i) => finding('note', i + 1)) }; + const body = render(reportWith(files), RUN); + assert.ok(body.includes('Notes (50, optional)') && !body.includes('did not fit')); +}); + +test('Windows paths and a working directory below the root become repository links', () => { + const files = { 'src\\lib (old)\\a.js': [finding('review', 4, { locations: [{ path: 'src\\lib (old)\\a.js', start_line: 4, end_line: 9 }] })] }; + const body = render(reportWith(files), { ...RUN, windows: true, prefix: 'packages/app/' }); + assert.ok(body.includes(`**[\`packages/app/src/lib (old)/a.js\`](https://github.com/o/r/blob/${COMMIT}/packages/app/src/lib%20%28old%29/a.js)**`), body); + assert.ok(body.includes(`[Line 4](https://github.com/o/r/blob/${COMMIT}/packages/app/src/lib%20%28old%29/a.js#L4-L9)`)); + const linux = render(reportWith({ 'a\\b.js': [finding('review', 1)] }), RUN); + assert.ok(linux.includes('a%5Cb.js'), 'a backslash is part of a name on Linux'); +}); + +test('without a commit, locations are not links', () => { + const body = render(reportWith({ 'a.js': [finding('review', 4)] }), { ...RUN, commit: '' }); + assert.ok(body.includes('**`a.js`**\n- Line 4 `maintainability/shared-logic`:')); + assert.ok(!body.includes('/blob/') && !body.includes('commit ')); +}); + +test('text from the change cannot add HTML, links, markers or mentions', () => { + assert.equal(inline('Value `x` in y'), 'Value `x` in <b>y</b>'); + assert.equal(inline('ping @octocat, not `@octocat`'), 'ping @\u200boctocat, not `@octocat`'); + assert.equal(inline(''), '<!-- jevgate-action comment key=review run=999 -->'); + assert.equal(inline('[approve](https://x.test) ![i](https://x.test/i.png)'), '\\[approve\\](https://x.test) !\\[i\\](https://x.test/i.png)'); + assert.equal(inline('line one\n# Heading\r\n> quote'), 'line one # Heading > quote'); + assert.equal(inline('a < b'), 'a &lt; b'); + // Only a closed run of backticks is code, as CommonMark reads it; an + // unclosed run is escaped. + assert.equal(inline('``a` '), '\\`\\`a\\` <i>'); + assert.equal(inline('`` a`b `` '), '`` a`b `` <i>'); + // A backslash cannot turn a backtick into text around raw HTML. + assert.equal(inline('\\``'), '\\\\``'); +}); + +test('an unclosed backtick in a message cannot pair with the next step\'s code', () => { + const bait = finding('review', 3, { message: 'see `', action: '`` then `' }); + const body = render(reportWith({ 'a.js': [bait] }), RUN); + assert.ok(body.includes(': see \\`
→ `` then \\`\n'), body); +}); + +test('paths and rules are code spans whatever backticks they hold', () => { + assert.equal(code('src/a.js'), '`src/a.js`'); + assert.equal(code('a`b'), '``a`b``'); + assert.equal(code('`a'), '`` `a ``'); +}); + +test('the comment marker cannot close its HTML comment', () => { + assert.equal(marker('review services/api'), 'b').includes('-->')); +}); diff --git a/test/reports/base-run.json b/test/reports/base-run.json new file mode 100644 index 0000000..859f024 --- /dev/null +++ b/test/reports/base-run.json @@ -0,0 +1,1510 @@ +{ + "quick": false, + "base_revision": "811595e449683e3ef96375d48cb97885a80c9d6f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/jevgate/jevgate", + "generation": 21, + "watcher_pid": null, + "errors": [], + "generated_at": 1790372499, + "status": "review", + "complete": true, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "gate": { + "passed": false, + "reasons": [ + "1 new review finding(s)" + ], + "new_findings": 2, + "baselined_findings": 0 + }, + "api_requests": 55, + "concurrency": 6, + "paid_input_tokens": 160708, + "paid_output_tokens": 7982, + "stages": { + "comments": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 2772, + "service_ms": 5972, + "queue_wait_ms": 11328, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 9, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 11, + "cached_judgments": 11, + "evaluated_judgments": 9, + "input_tokens": 41436, + "output_tokens": 2182, + "evidence_bytes": 25077 + }, + "constants": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 4, + "cached_judgments": 4, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 266, + "service_ms": 266, + "queue_wait_ms": 2550, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 1, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 1, + "input_tokens": 890, + "output_tokens": 52, + "evidence_bytes": 740 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3087, + "service_ms": 3167, + "queue_wait_ms": 6869, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 2, + "cached_judgments": 2, + "evaluated_judgments": 5, + "input_tokens": 10591, + "output_tokens": 380, + "evidence_bytes": 20754 + }, + "locate": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 259, + "service_ms": 259, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 1, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 1, + "input_tokens": 774, + "output_tokens": 73, + "evidence_bytes": 1028 + }, + "outline": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3103, + "service_ms": 1789, + "queue_wait_ms": 5920, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 5, + "input_tokens": 9441, + "output_tokens": 301, + "evidence_bytes": 16645 + }, + "recheck": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 880, + "service_ms": 4496, + "queue_wait_ms": 1643, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 10, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 4, + "cached_judgments": 4, + "evaluated_judgments": 10, + "input_tokens": 23892, + "output_tokens": 516, + "evidence_bytes": 61331 + }, + "security": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3555, + "service_ms": 3529, + "queue_wait_ms": 9453, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 6, + "cached_judgments": 6, + "evaluated_judgments": 5, + "input_tokens": 24113, + "output_tokens": 2154, + "evidence_bytes": 16718 + }, + "tests": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3676, + "service_ms": 3569, + "queue_wait_ms": 24992, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 10, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 1, + "cached_judgments": 1, + "evaluated_judgments": 10, + "input_tokens": 13403, + "output_tokens": 800, + "evidence_bytes": 8626 + }, + "trace": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 1533, + "service_ms": 2614, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 2, + "cached_judgments": 2, + "evaluated_judgments": 5, + "input_tokens": 21859, + "output_tokens": 740, + "evidence_bytes": 58200 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 2270, + "service_ms": 2546, + "queue_wait_ms": 5162, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 4, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 3, + "cached_judgments": 3, + "evaluated_judgments": 4, + "input_tokens": 14309, + "output_tokens": 784, + "evidence_bytes": 15850 + } + }, + "settled": true, + "files": [ + { + "path": "src/github.rs", + "role": "source", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [], + "dimensions": { + "comments": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 comments judged: 7 clear.", + "rule_version": "2", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "3 functions judged: 3 clear. 4 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 3, + "review": 0, + "consider": 0, + "note": 0, + "clear": 3, + "uncertain": 0, + "needs_context": 0, + "too_small": 4, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 value units judged: 7 clear.", + "rule_version": "4", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "uncertain", + "concern_probability": 0.9, + "decision_basis": "7 security units judged: 6 clear, 1 uncertain.", + "rule_version": "6", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 6, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 security units judged: 7 clear.", + "rule_version": "5", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "2 tests judged: 2 clear.", + "rule_version": "5", + "units": { + "judged": 2, + "review": 0, + "consider": 0, + "note": 0, + "clear": 2, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 security units judged: 7 clear.", + "rule_version": "4", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 32716, + "output_tokens": 2213 + }, + { + "path": "src/options/commands.rs", + "role": "source", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "documentation/comments", + "strength": "note", + "line": 1, + "message": "This file's top-level code has a comment to clean up: at line 1 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": null, + "rule_version": "2", + "concern_probability": 0.98, + "locations": [ + { + "path": "src/options/commands.rs", + "start_line": 1, + "end_line": 1, + "symbol": "top-level code" + } + ], + "quote": "//! The subcommands, the baseline actions and their help text.\n", + "fingerprint": "5fb29bb3e2212df289b9748c9fd60c1599b13459df84a0400841d985e5a18823", + "rank": 0.6792842369487464, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.98, + "decision_basis": "26 comments judged: 1 note, 25 clear.", + "rule_version": "2", + "units": { + "judged": 26, + "review": 0, + "consider": 0, + "note": 1, + "clear": 25, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No functions to judge.", + "rule_version": "14", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "uncertain", + "concern_probability": 0.37, + "decision_basis": "1 value unit judged: 1 uncertain.", + "rule_version": "4", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "6", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "5", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "4", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 31057, + "output_tokens": 1318 + }, + { + "path": "src/options/mod.rs", + "role": "source", + "status": "note", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "documentation/comments", + "strength": "note", + "line": 1, + "message": "This file's top-level code has a comment to clean up: at lines 1–2 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": null, + "rule_version": "2", + "concern_probability": 0.93, + "locations": [ + { + "path": "src/options/mod.rs", + "start_line": 1, + "end_line": 2, + "symbol": "top-level code" + } + ], + "quote": "//! The `check` arguments, output formats and gate levels; the subcommands and\n//! their help text are in `commands`.\n", + "fingerprint": "58c86ae408c99452fa1af1684beb3d10df53e78c32cd7ba8bb92f1a13f1ab928", + "rank": 1.0217094284613422, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 276, + "message": "`CheckArgs::enabled` has a comment to clean up: at line 276 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "CheckArgs::enabled", + "rule_version": "2", + "concern_probability": 0.8900000000000001, + "locations": [ + { + "path": "src/options/mod.rs", + "start_line": 276, + "end_line": 276, + "symbol": "CheckArgs::enabled" + } + ], + "quote": "/// Whether a rule is selected, by key or ID.\n", + "fingerprint": "8cbf468d2d64d25f58ccde803b1924b6d4648fe6ccd968aa1e37d19e23a2f62d", + "rank": 0.6169009906983514, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.93, + "decision_basis": "60 comments judged: 2 note, 58 clear.", + "rule_version": "2", + "units": { + "judged": 60, + "review": 0, + "consider": 0, + "note": 2, + "clear": 58, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "5 functions judged: 5 clear. 12 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 5, + "review": 0, + "consider": 0, + "note": 0, + "clear": 5, + "uncertain": 0, + "needs_context": 0, + "too_small": 12, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "5 value units judged: 5 clear.", + "rule_version": "4", + "units": { + "judged": 5, + "review": 0, + "consider": 0, + "note": 0, + "clear": 5, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "6", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "5", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "4", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 82669, + "output_tokens": 4651 + }, + { + "path": "src/output.rs", + "role": "source", + "status": "review", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "maintainability/shared-logic", + "strength": "review", + "line": 213, + "message": "Lines 213 and 247 of `emit_findings` (src/output.rs) perform the same steps for the same purpose (0.86). Differences: `Review ({}):`→`Notes ({}, optional):`, `review`→`notes`, `1;31`→`1`.", + "action": "Move the shared steps into one implementation", + "symbol": "Lines 213 and 247 of `emit_findings` (src/output.rs)", + "rule_version": "19", + "concern_probability": 0.86, + "locations": [ + { + "path": "src/output.rs", + "start_line": 213, + "end_line": 217, + "symbol": "emit_findings" + }, + { + "path": "src/output.rs", + "start_line": 247, + "end_line": 251, + "symbol": "emit_findings" + } + ], + "quote": "let heading = format!(\"Review ({}):\", review.len());\n writeln!(out, \"\\n{}\", style.paint(\"1;31\", &heading))?;\n for (path, finding) in &review {\n emit_finding(out, path, finding, style)?;\n }", + "fingerprint": "c2411e834cced1c5f56c8a68b9a04e95ab9ba7bb8781e3879793745a50f202ad", + "rank": 2.062189934606599, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "consider", + "line": 331, + "message": "`emit_finding` likely uses a value whose meaning a reader must guess (0.87). The value is \"36\".", + "action": "Give the value a descriptive constant name", + "symbol": "emit_finding", + "rule_version": "4", + "concern_probability": 0.8700000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 331, + "end_line": 351, + "symbol": "emit_finding" + } + ], + "fingerprint": "496b58069e9ed18abb10136b9f3b6e4ebdd5b892b41c5c5a2131d6f8c407ced0", + "rank": 2.6892069344217355, + "baselined": false + }, + { + "rule": "maintainability/function-simplification", + "strength": "note", + "line": 289, + "message": "`emit_context_load` reads well as it is; one block could be named as a helper.", + "action": "Optional: extract that block if it grows", + "symbol": "emit_context_load", + "rule_version": "14", + "concern_probability": 0.85, + "locations": [ + { + "path": "src/output.rs", + "start_line": 289, + "end_line": 327, + "symbol": "emit_context_load" + } + ], + "fingerprint": "362c563e25a7e440960c9bd2d433d974d9fd9d1a201479921f4ae916948b39a7", + "rank": 3.135547535996846, + "baselined": false + }, + { + "rule": "maintainability/function-simplification", + "strength": "note", + "line": 256, + "message": "`emit_summary` reads well as it is; one block could be named as a helper.", + "action": "Optional: extract that block if it grows", + "symbol": "emit_summary", + "rule_version": "14", + "concern_probability": 0.88, + "locations": [ + { + "path": "src/output.rs", + "start_line": 256, + "end_line": 286, + "symbol": "emit_summary" + } + ], + "fingerprint": "84dce89ad1b06817330b51a91db2a8cbad9b360c057348791ff356f5e5c42448", + "rank": 3.0498475944637593, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "note", + "line": 353, + "message": "`emit_file` may use a value whose meaning a reader must guess; the answer was split.", + "action": "Optional: name or configure the value if it changes", + "symbol": "emit_file", + "rule_version": "4", + "concern_probability": 0.7, + "locations": [ + { + "path": "src/output.rs", + "start_line": 353, + "end_line": 386, + "symbol": "emit_file" + } + ], + "fingerprint": "744eb1bc38d3e8c0f54ae00890a7d4e77466683e5eb1252cb1e98bcec75c3e64", + "rank": 2.4887436430425893, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "note", + "line": 26, + "message": "`usd` has a value that could be named, though its context explains it.", + "action": "Optional: name or configure the value if it changes", + "symbol": "usd", + "rule_version": "4", + "concern_probability": 0.8400000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 26, + "end_line": 28, + "symbol": "usd" + } + ], + "fingerprint": "b2f2cf1e5db49630ef53b471139ab8e8be7e8fa3f02b5017f99c4980c2c3beba", + "rank": 1.1644872633407082, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 137, + "message": "`headline` has a comment to clean up: at lines 137–138 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "headline", + "rule_version": "2", + "concern_probability": 0.83, + "locations": [ + { + "path": "src/output.rs", + "start_line": 137, + "end_line": 138, + "symbol": "headline" + } + ], + "quote": "/// Status, gate, scope and cost on one line; for a dry run, the planned\n/// requests and the cost of those the cache does not answer.\n", + "fingerprint": "d9fd46abc3e8075bd67e31f9cd50c6d17995271f4ce8456431ef001d8acc63b5", + "rank": 0.9118481995945311, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 288, + "message": "`emit_context_load` has a comment to clean up: at line 288 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "emit_context_load", + "rule_version": "2", + "concern_probability": 0.95, + "locations": [ + { + "path": "src/output.rs", + "start_line": 288, + "end_line": 288, + "symbol": "emit_context_load" + } + ], + "quote": "/// Estimated tokens each harness loads at session start, then loading facts.\n", + "fingerprint": "0c489984d06d9b262fe2219500c4785e94813d5e7d2a16fd1f433dab240e7f3a", + "rank": 0.658489821531948, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 255, + "message": "`emit_summary` has a comment to clean up: at line 255 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "emit_summary", + "rule_version": "2", + "concern_probability": 0.8600000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 255, + "end_line": 255, + "symbol": "emit_summary" + } + ], + "quote": "/// Counts of undecided, unsent and failed files, and skip reasons.\n", + "fingerprint": "b0d9560615fa227f636fd476f31ea6a9f55184ac8a9fa2f5b96a2a179248b91f", + "rank": 0.596106575281553, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 181, + "message": "`ranked` has a comment to clean up: at line 181 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "ranked", + "rule_version": "2", + "concern_probability": 0.8200000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 181, + "end_line": 181, + "symbol": "ranked" + } + ], + "quote": "/// Every finding with its file's path, highest rank first.\n", + "fingerprint": "8c4ad9aa47c3a7691f73eb65d7626a0ed6c93e6d3c97594937d2567542df2b11", + "rank": 0.5683806880591552, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.95, + "decision_basis": "15 comments judged: 4 note, 11 clear.", + "rule_version": "2", + "units": { + "judged": 15, + "review": 0, + "consider": 0, + "note": 4, + "clear": 11, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "note", + "concern_probability": 0.88, + "decision_basis": "14 functions judged: 2 note, 12 clear. 4 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 14, + "review": 0, + "consider": 0, + "note": 2, + "clear": 12, + "uncertain": 0, + "needs_context": 0, + "too_small": 4, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "consider", + "concern_probability": 0.8700000000000001, + "decision_basis": "13 value units judged: 1 consider, 2 note, 10 clear.", + "rule_version": "4", + "units": { + "judged": 13, + "review": 0, + "consider": 1, + "note": 2, + "clear": 10, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "6", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "5", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "review", + "concern_probability": 0.86, + "decision_basis": "1 candidate pair judged: 1 review.", + "rule_version": "19", + "units": { + "judged": 1, + "review": 1, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "2 tests judged: 2 clear.", + "rule_version": "5", + "units": { + "judged": 2, + "review": 0, + "consider": 0, + "note": 0, + "clear": 2, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "4", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 66999, + "output_tokens": 3877 + }, + { + "path": "src/tests/mod.rs", + "role": "test", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [], + "dimensions": { + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "8 functions judged: 8 clear. 6 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 8, + "review": 0, + "consider": 0, + "note": 0, + "clear": 8, + "uncertain": 0, + "needs_context": 0, + "too_small": 6, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "uncertain", + "concern_probability": 0.23, + "decision_basis": "7 tests judged: 6 clear, 1 uncertain.", + "rule_version": "5", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 6, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 25324, + "output_tokens": 884 + }, + { + "path": "README.md", + "role": "docs", + "status": "skipped", + "cached": false, + "model": null, + "error": "Outside upload_allow/upload_deny; not judged.", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [ + { + "rule": "documentation/comments", + "path": "src/options/commands.rs", + "previous_path": null, + "previous_generation": 20, + "state": "persistent", + "reason": "The same finding remains", + "fingerprint": "5fb29bb3e2212df289b9748c9fd60c1599b13459df84a0400841d985e5a18823" + }, + { + "rule": "documentation/comments", + "path": "src/options/mod.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "58c86ae408c99452fa1af1684beb3d10df53e78c32cd7ba8bb92f1a13f1ab928" + }, + { + "rule": "documentation/comments", + "path": "src/options/mod.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "8cbf468d2d64d25f58ccde803b1924b6d4648fe6ccd968aa1e37d19e23a2f62d" + }, + { + "rule": "maintainability/shared-logic", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "c2411e834cced1c5f56c8a68b9a04e95ab9ba7bb8781e3879793745a50f202ad" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "496b58069e9ed18abb10136b9f3b6e4ebdd5b892b41c5c5a2131d6f8c407ced0" + }, + { + "rule": "maintainability/function-simplification", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "362c563e25a7e440960c9bd2d433d974d9fd9d1a201479921f4ae916948b39a7" + }, + { + "rule": "maintainability/function-simplification", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "84dce89ad1b06817330b51a91db2a8cbad9b360c057348791ff356f5e5c42448" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "744eb1bc38d3e8c0f54ae00890a7d4e77466683e5eb1252cb1e98bcec75c3e64" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "b2f2cf1e5db49630ef53b471139ab8e8be7e8fa3f02b5017f99c4980c2c3beba" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "d9fd46abc3e8075bd67e31f9cd50c6d17995271f4ce8456431ef001d8acc63b5" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "0c489984d06d9b262fe2219500c4785e94813d5e7d2a16fd1f433dab240e7f3a" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "b0d9560615fa227f636fd476f31ea6a9f55184ac8a9fa2f5b96a2a179248b91f" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "8c4ad9aa47c3a7691f73eb65d7626a0ed6c93e6d3c97594937d2567542df2b11" + }, + { + "rule": "documentation/comments", + "path": "src/manual.rs", + "previous_path": null, + "previous_generation": 20, + "state": "non-comparable", + "reason": "The file was not judged in this snapshot, or rubric or model changed", + "fingerprint": "a77714c33851e41910555dfcaf22fb82428e5636fd58aade6a9a499a2f03ed6b" + }, + { + "rule": "documentation/comments", + "path": "src/command.rs", + "previous_path": null, + "previous_generation": 20, + "state": "non-comparable", + "reason": "The file was not judged in this snapshot, or rubric or model changed", + "fingerprint": "deaef5f7bc17022e28d45da241f86648ccbdb275abb3e39808372bbd9f1e9e66" + } + ], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "security/injection", + "security/sensitive-data", + "security/unsafe-settings", + "security/access-control", + "security/workflows", + "tests/value", + "tests/redundancy", + "documentation/agent-context", + "documentation/large-docs", + "documentation/staleness", + "documentation/duplication", + "documentation/comments" + ] +} diff --git a/test/reports/http-402.json b/test/reports/http-402.json new file mode 100644 index 0000000..1aaab17 --- /dev/null +++ b/test/reports/http-402.json @@ -0,0 +1,133 @@ +{ + "quick": false, + "base_revision": "63418fdae32078965d851308771a48d8f897e28f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/app/app", + "generation": 1, + "watcher_pid": null, + "errors": [], + "generated_at": 1790569169, + "status": "incomplete", + "complete": false, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_bend_file_lines": 300.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "api_requests": 1, + "concurrency": 6, + "paid_input_tokens": 0, + "paid_output_tokens": 0, + "stages": { + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + } + }, + "settled": true, + "files": [ + { + "path": "lib/parse.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "TypeSafe HTTP 402; request was not retried", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + }, + { + "path": "lib/rows.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "TypeSafe request not sent after HTTP 402; restore account access and rerun the review", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "tests/value", + "tests/redundancy", + "tests/laws" + ] +} diff --git a/test/reports/no-key.json b/test/reports/no-key.json new file mode 100644 index 0000000..eef9881 --- /dev/null +++ b/test/reports/no-key.json @@ -0,0 +1,133 @@ +{ + "quick": false, + "base_revision": "63418fdae32078965d851308771a48d8f897e28f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/app/app", + "generation": 1, + "watcher_pid": null, + "errors": [], + "generated_at": 1790569169, + "status": "incomplete", + "complete": false, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_bend_file_lines": 300.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "api_requests": 0, + "concurrency": 6, + "paid_input_tokens": 0, + "paid_output_tokens": 0, + "stages": { + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + } + }, + "settled": true, + "files": [ + { + "path": "lib/parse.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + }, + { + "path": "lib/rows.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "tests/value", + "tests/redundancy", + "tests/laws" + ] +} From b4f7f02a997df0d6cefa5038ac66869ff5185d6c Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 01:43:16 -0300 Subject: [PATCH 5/9] Mark the findings that fail the gate, from JevGate 0.26.0 From 0.26.0 JevGate blocks by default only on rules and levels measured right at least 80% of the time on unseen projects; the others are reported without failing the gate, and each finding records how the gate counted it (`gate`: fails, measuring or advisory). Without that, the comment would list five reviews under "gate passed" with nothing to say why. A finding that fails the gate gets "(fails the gate)" after its rule, as in JevGate's agent text, and one line counts the findings reported without failing it because their rules and levels are still being measured. Reports from earlier versions have no such field, and nothing is marked. --- README.md | 2 +- render.cjs | 40 +++++++++++++++++++++++++++++++++------- test/render.test.cjs | 24 ++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d913c73..ee48a2b 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The action installs a release binary (checked against its SHA-256), keeps JevGat ## The pull request comment -GitHub shows at most 10 error and 10 warning annotations per step, so on pull requests the action also lists every finding in one comment and updates it on each run. Reviews come first, then considers, then notes (collapsed), each grouped by file with a link to the line. The comment gives the gate's result and the run's API requests, input tokens and cost. When the run could not finish (exit code 2: no key, a provider error such as HTTP 402, or the request budget), it says so first, with the reasons. +GitHub shows at most 10 error and 10 warning annotations per step, so on pull requests the action also lists every finding in one comment and updates it on each run. Reviews come first, then considers, then notes (collapsed), each grouped by file with a link to the line. The comment gives the gate's result and the run's API requests, input tokens and cost; from JevGate 0.26.0 it also marks each finding that fails the gate and counts those reported without failing it while their rules are still being measured. When the run could not finish (exit code 2: no key, a provider error such as HTTP 402, or the request budget), it says so first, with the reasons. - It needs `pull-requests: write`. Where the token can't comment, as on pull requests from forks, the run says so in the log and the job summary and carries on. - Each job, and each `working-directory`, keeps its own comment; the jobs of a matrix share one. A run for an older push never replaces a newer run's comment. diff --git a/render.cjs b/render.cjs index 77601a4..60e7b1b 100644 --- a/render.cjs +++ b/render.cjs @@ -226,14 +226,39 @@ function cost(report) { return `~$${((tokens * USD_PER_MILLION_INPUT_TOKENS) / 1e6).toFixed(4)}`; } -/** The gate's reasons as JevGate gave them, the run's size and cost, and the - * files left undecided. */ -function outcome(report) { +/** From JevGate 0.26.0 each finding records how the gate counted it (`gate`: + * `fails`, `measuring` or `advisory`); earlier reports have no such field, and + * the comment then marks nothing. */ +const FAILS = 'fails'; +const MEASURING = 'measuring'; + +/** The listed findings the gate reported without failing because their rule + * and level are still being measured, counted by level; `null` when none. */ +function measuring(all) { + const counts = LEVELS.map((level) => [ + level, + all.filter((entry) => entry.finding.strength === level.strength && entry.finding.gate === MEASURING).length, + ]).filter(([, n]) => n > 0); + if (counts.length === 0) { + return null; + } + const total = counts.reduce((sum, [, n]) => sum + n, 0); + const counted = counts.map(([level, n]) => count(n, level.noun)).join(' and '); + return `${counted} ${total === 1 ? 'is' : 'are'} reported without failing the gate: their rules and levels are still being measured (\`jevgate rules\` shows which fail it by default).`; +} + +/** The gate's reasons as JevGate gave them, what it reported without failing, + * the run's size and cost, and the files left undecided. */ +function outcome(report, all) { const lines = []; const gate = report.gate; if (gate && !gate.passed && (gate.reasons || []).length > 0) { lines.push(`Gate failed: ${gate.reasons.map(inline).join('; ')}.`); } + const measured = measuring(all); + if (measured) { + lines.push(measured); + } const files = report.files || []; const usage = [ count(files.length, 'file'), @@ -256,15 +281,16 @@ function outcome(report) { return lines.join('\n\n'); } -/** One finding: its line, linked when the commit is known, the rule, the - * message and the next step. */ +/** One finding: its line, linked when the commit is known, the rule, whether + * it fails the gate, the message and the next step. */ function item({ path, finding }, run) { const location = (finding.locations || []).find( (l) => l.path === path && l.start_line === finding.line && l.end_line > finding.line, ); const anchor = location ? `#L${finding.line}-L${location.end_line}` : `#L${finding.line}`; const line = run.commit ? `[Line ${finding.line}](${blobUrl(path, anchor, run)})` : `Line ${finding.line}`; - return `- ${line} ${code(finding.rule)}: ${inline(finding.message)}
→ ${inline(finding.action)}`; + const fails = finding.gate === FAILS ? ' (fails the gate)' : ''; + return `- ${line} ${code(finding.rule)}${fails}: ${inline(finding.message)}
→ ${inline(finding.action)}`; } /** A file's heading and its findings by line. */ @@ -339,7 +365,7 @@ function compose(report, run, kept, all) { if (report && report.status === 'no-changed-source') { parts.push('No supported file changed since the base revision.'); } else if (report) { - parts.push(outcome(report)); + parts.push(outcome(report, all)); if (finished(run.exitCode) && !all.some((entry) => entry.finding.strength !== 'note')) { parts.push('No new review or consider findings.'); } diff --git a/test/render.test.cjs b/test/render.test.cjs index fdcdab7..bb393d1 100644 --- a/test/render.test.cjs +++ b/test/render.test.cjs @@ -109,6 +109,30 @@ test('a run that wrote no report still says it stopped', () => { assert.ok(!body.includes('API request')); }); +test('from JevGate 0.26.0, findings that fail the gate and ones still being measured are marked', () => { + const files = { + 'a.js': [ + finding('review', 1, { gate: 'fails', rule: 'maintainability/function-simplification' }), + finding('review', 2, { gate: 'measuring' }), + finding('consider', 3, { gate: 'measuring' }), + finding('consider', 4, { gate: 'measuring' }), + finding('consider', 5, { gate: 'advisory' }), + finding('note', 6), + ], + }; + const body = render(reportWith(files), RUN); + assert.ok(body.includes('`maintainability/function-simplification` (fails the gate): Finding at line 1'), body); + assert.ok(body.includes('`maintainability/shared-logic`: Finding at line 2')); + assert.equal(body.split('(fails the gate)').length, 2, 'only the finding that fails is marked'); + assert.ok(body.includes( + '\n1 review finding and 2 consider findings are reported without failing the gate: their rules and levels are still being measured (`jevgate rules` shows which fail it by default).\n', + )); + const one = render(reportWith({ 'a.js': [finding('review', 2, { gate: 'measuring' })] }), RUN); + assert.ok(one.includes('\n1 review finding is reported without failing the gate')); + const older = render(report('base-run'), RUN); + assert.ok(!older.includes('fails the gate') && !older.includes('still being measured')); +}); + test('a passing gate with nothing new says so', () => { const body = render(reportWith({ 'a.js': [finding('note', 3)] }, { status: 'note', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); assert.ok(body.includes('### JevGate: gate passed\n')); From e9352c3c1c8b0aefe54d3c43ccb1960ff09865c3 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:29:05 -0300 Subject: [PATCH 6/9] Say how often each finding's kind was right, from JevGate 0.28.0 JevGate 0.28 removes the probability from each finding's message and records instead how often findings of its rule and level were right on projects it was never tuned on (`precision`: `right` of `labeled`). The comment listed only the message and the next step, so with 0.28 it would show neither. Each finding now ends as JevGate's own output ends it: "Right 87% of the time (23 labels)." or, below 20 labels, "Not yet measured."; a report before 0.28 shows nothing more. --- README.md | 2 +- render.cjs | 26 ++++++++++++++++++++++++-- test/render.test.cjs | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee48a2b..634b4ad 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The action installs a release binary (checked against its SHA-256), keeps JevGat ## The pull request comment -GitHub shows at most 10 error and 10 warning annotations per step, so on pull requests the action also lists every finding in one comment and updates it on each run. Reviews come first, then considers, then notes (collapsed), each grouped by file with a link to the line. The comment gives the gate's result and the run's API requests, input tokens and cost; from JevGate 0.26.0 it also marks each finding that fails the gate and counts those reported without failing it while their rules are still being measured. When the run could not finish (exit code 2: no key, a provider error such as HTTP 402, or the request budget), it says so first, with the reasons. +GitHub shows at most 10 error and 10 warning annotations per step, so on pull requests the action also lists every finding in one comment and updates it on each run. Reviews come first, then considers, then notes (collapsed), each grouped by file with a link to the line. The comment gives the gate's result and the run's API requests, input tokens and cost; from JevGate 0.26.0 it also marks each finding that fails the gate and counts those reported without failing it while their rules are still being measured, and from 0.28.0 each finding ends with how often findings of its rule and level were right on projects JevGate was never tuned on. When the run could not finish (exit code 2: no key, a provider error such as HTTP 402, or the request budget), it says so first, with the reasons. - It needs `pull-requests: write`. Where the token can't comment, as on pull requests from forks, the run says so in the log and the job summary and carries on. - Each job, and each `working-directory`, keeps its own comment; the jobs of a matrix share one. A run for an older push never replaces a newer run's comment. diff --git a/render.cjs b/render.cjs index 60e7b1b..1131396 100644 --- a/render.cjs +++ b/render.cjs @@ -281,8 +281,30 @@ function outcome(report, all) { return lines.join('\n\n'); } +/** From JevGate 0.28 each review and consider records how often findings of + * its rule and level were right on projects JevGate was never tuned on + * (`precision`: `right` of `labeled`), and its message no longer ends with a + * probability. Below this many labels, JevGate says it is not yet measured. */ +const MIN_LABELS = 20; + +/** How often findings like it were right, worded as JevGate's own output + * words it; nothing for a note or a report before 0.28. */ +function precision(finding) { + const { right, labeled } = finding.precision || {}; + if (!Number.isInteger(right) || !Number.isInteger(labeled)) { + return ''; + } + if (labeled < MIN_LABELS) { + return ' Not yet measured.'; + } + // Half rounded up, as JevGate rounds it. + const percent = Math.floor((200 * right + labeled) / (2 * labeled)); + return ` Right ${percent}% of the time (${labeled} labels).`; +} + /** One finding: its line, linked when the commit is known, the rule, whether - * it fails the gate, the message and the next step. */ + * it fails the gate, the message, how often findings like it were right and + * the next step. */ function item({ path, finding }, run) { const location = (finding.locations || []).find( (l) => l.path === path && l.start_line === finding.line && l.end_line > finding.line, @@ -290,7 +312,7 @@ function item({ path, finding }, run) { const anchor = location ? `#L${finding.line}-L${location.end_line}` : `#L${finding.line}`; const line = run.commit ? `[Line ${finding.line}](${blobUrl(path, anchor, run)})` : `Line ${finding.line}`; const fails = finding.gate === FAILS ? ' (fails the gate)' : ''; - return `- ${line} ${code(finding.rule)}${fails}: ${inline(finding.message)}
→ ${inline(finding.action)}`; + return `- ${line} ${code(finding.rule)}${fails}: ${inline(finding.message)}${precision(finding)}
→ ${inline(finding.action)}`; } /** A file's heading and its findings by line. */ diff --git a/test/render.test.cjs b/test/render.test.cjs index bb393d1..f4adc6f 100644 --- a/test/render.test.cjs +++ b/test/render.test.cjs @@ -133,6 +133,22 @@ test('from JevGate 0.26.0, findings that fail the gate and ones still being meas assert.ok(!older.includes('fails the gate') && !older.includes('still being measured')); }); +test('from JevGate 0.28, each finding says how often findings like it were right', () => { + const body = render( + reportWith({ + 'src/lib.rs': [ + finding('review', 3, { precision: { right: 20, labeled: 23 } }), + finding('consider', 9, { precision: { right: 2, labeled: 5 } }), + finding('consider', 12), + ], + }), + RUN, + ); + assert.match(body, /Finding at line 3 Right 87% of the time \(23 labels\)\.
/); + assert.match(body, /Finding at line 9 Not yet measured\.
/); + assert.match(body, /Finding at line 12
/, 'a report before 0.28 shows none'); +}); + test('a passing gate with nothing new says so', () => { const body = render(reportWith({ 'a.js': [finding('note', 3)] }, { status: 'note', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); assert.ok(body.includes('### JevGate: gate passed\n')); From 44178110b71e2c45064e366fd9c0102ba0252d8d Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:28:59 -0300 Subject: [PATCH 7/9] Say that a base reviews only the changed lines from JevGate 0.26.0 JevGate 0.26 judges only what a change touches when given --base: the changed lines of changed files, with --whole-files for the old behavior. The input's description still said it reviewed changed files. --- README.md | 2 +- action.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 634b4ad..d861ec7 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The action installs a release binary (checked against its SHA-256), keeps JevGat | `api-key` | | TypeSafe API key ([get one](https://console.typesafe.ai/settings/keys)), or an OpenRouter or Vercel AI Gateway key with `api-key-kind`. Save it as a repository secret | | `api-key-kind` | `typesafe` | Which service issued `api-key`: `typesafe`, `openrouter` or `vercel` (the gateways need JevGate 0.26.0 or later) | | `version` | `latest` | JevGate version; pin one for repeatable results | -| `base` | the pull request's base commit | Review only files changed since this revision; empty on other events, which review the whole repository | +| `base` | the pull request's base commit | Review only what changed since this revision: from JevGate 0.26.0 the changed lines of changed files (`--whole-files` in `args` for whole files), before it changed files whole; empty on other events, which review the whole repository | | `args` | | More `jevgate check` arguments, such as `--rule default --rule security --include-tests` | | `format` | `github` | `github`, `agent`, `json`, `jsonl`, `sarif` or `gitlab` | | `sarif-file` | | Also write the findings as SARIF to this path, for `upload-sarif` (JevGate 0.18.0 or later) | diff --git a/action.yml b/action.yml index dda9ec1..af0a8ad 100644 --- a/action.yml +++ b/action.yml @@ -18,7 +18,7 @@ inputs: required: false default: latest base: - description: Review only files changed since this revision. Defaults to the pull request's base commit; empty on other events, which review the whole repository. + description: Review only what changed since this revision. JevGate 0.26.0 and later ask about and report only the changed lines of changed files (add `--whole-files` to `args` for whole files); earlier versions review changed files whole. Defaults to the pull request's base commit; empty on other events, which review the whole repository. required: false default: "" args: From 1b08672042f2217b19b283232959ec11cf0c3cb8 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:33:53 -0300 Subject: [PATCH 8/9] Word a preview language's findings as JevGate 0.30 does, and drop a committed cache From JevGate 0.30 a finding of its own rules in a preview language's file carries `preview` with the language, and its `precision` holds that language's counts. The comment rebuilt the precision sentence without it ("Right 12% of the time (34 labels)." for a Bash shared-logic review, where JevGate says "in Bash") and explained every `measuring` finding as a rule still being measured, wrong for a C function-simplification review, whose rule and level fail by default outside preview languages. It now says "in ", and gives preview findings their own reason: the language is in preview, and JevGate's own rules never fail the default gate there. A law finding keeps JevGate's caveat that it was labeled only on Bend 2 projects. A pull request could also commit answers under .jevgate/cache that clear its own code; JevGate 0.28 and later ignores cache files Git tracks, and the action now removes a checked-out cache before restoring its own, for earlier versions too. --- README.md | 2 +- action.yml | 7 +++++++ render.cjs | 50 +++++++++++++++++++++++++++++++++----------- test/render.test.cjs | 20 ++++++++++++++++++ 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d861ec7..94af817 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ jobs: version: 0.25.0 ``` -The action installs a release binary (checked against its SHA-256), keeps JevGate's answer cache in the Actions cache so unchanged code costs nothing, and runs `jevgate check --base --format github`. It needs no Rust toolchain and runs on Linux, macOS and Windows runners. +The action installs a release binary (checked against its SHA-256), keeps JevGate's answer cache in the Actions cache so unchanged code costs nothing (and removes one the pull request commits, whose answers could clear its own code), and runs `jevgate check --base --format github`. It needs no Rust toolchain and runs on Linux, macOS and Windows runners. ## Inputs diff --git a/action.yml b/action.yml index af0a8ad..21765f4 100644 --- a/action.yml +++ b/action.yml @@ -60,6 +60,13 @@ runs: env: JEVGATE_VERSION: ${{ inputs.version }} run: bash "$GITHUB_ACTION_PATH/install.sh" + # Answers a pull request commits under .jevgate/cache could clear its + # own code; JevGate 0.28 and later never reads a cache file Git tracks, + # and this keeps earlier versions from reading one too. + - name: Leave out answers the change committed + shell: bash + working-directory: ${{ inputs.working-directory }} + run: rm -rf .jevgate/cache - name: Restore answers if: inputs.cache == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/render.cjs b/render.cjs index 1131396..93d1e81 100644 --- a/render.cjs +++ b/render.cjs @@ -232,19 +232,36 @@ function cost(report) { const FAILS = 'fails'; const MEASURING = 'measuring'; -/** The listed findings the gate reported without failing because their rule - * and level are still being measured, counted by level; `null` when none. */ -function measuring(all) { +/** `entries` counted by level ("1 review finding and 2 consider findings"), + * and whether that is one finding. */ +function byLevel(entries) { const counts = LEVELS.map((level) => [ level, - all.filter((entry) => entry.finding.strength === level.strength && entry.finding.gate === MEASURING).length, + entries.filter((entry) => entry.finding.strength === level.strength).length, ]).filter(([, n]) => n > 0); - if (counts.length === 0) { - return null; + return [counts.map(([level, n]) => count(n, level.noun)).join(' and '), entries.length === 1]; +} + +/** Why the listed findings the gate reported without failing did not fail + * it: their rules and levels are still being measured, or, from JevGate + * 0.30, their file's language is in preview (`preview` names it), where + * JevGate's own rules never fail the default gate. `null` when none. */ +function measuring(all) { + const reported = all.filter((entry) => entry.finding.gate === MEASURING); + const measured = reported.filter((entry) => !entry.finding.preview); + const previewed = reported.filter((entry) => entry.finding.preview); + const reasons = []; + if (measured.length > 0) { + const [counted, one] = byLevel(measured); + reasons.push(`${counted} ${one ? 'is' : 'are'} reported without failing the gate: their rules and levels are still being measured (\`jevgate rules\` shows which fail it by default).`); + } + if (previewed.length > 0) { + const [counted, one] = byLevel(previewed); + const languages = [...new Set(previewed.map((entry) => entry.finding.preview))].sort(); + const which = languages.length === 1 ? `${languages[0]} is` : `${languages.join(', ')} are`; + reasons.push(`${counted} ${one ? 'is' : 'are'} reported without failing the gate: ${which} in preview, and by default JevGate's own rules never fail it there.`); } - const total = counts.reduce((sum, [, n]) => sum + n, 0); - const counted = counts.map(([level, n]) => count(n, level.noun)).join(' and '); - return `${counted} ${total === 1 ? 'is' : 'are'} reported without failing the gate: their rules and levels are still being measured (\`jevgate rules\` shows which fail it by default).`; + return reasons.length > 0 ? reasons.join(' ') : null; } /** The gate's reasons as JevGate gave them, what it reported without failing, @@ -287,19 +304,28 @@ function outcome(report, all) { * probability. Below this many labels, JevGate says it is not yet measured. */ const MIN_LABELS = 20; +/** The rule JevGate labels only on Bend 2 projects, which its table of + * precision leaves out; its findings say so. */ +const BEND_2_RULE = 'tests/laws'; + /** How often findings like it were right, worded as JevGate's own output - * words it; nothing for a note or a report before 0.28. */ + * words it: in a preview language (`preview`, from 0.30) that language's + * own; nothing for a note or a report before 0.28. */ function precision(finding) { const { right, labeled } = finding.precision || {}; if (!Number.isInteger(right) || !Number.isInteger(labeled)) { return ''; } + const place = finding.preview ? ` in ${finding.preview}` : ''; if (labeled < MIN_LABELS) { - return ' Not yet measured.'; + const bend = labeled === 0 && !finding.preview && finding.rule === BEND_2_RULE; + return bend + ? ' Not yet measured: labeled only on Bend 2 projects, which the maturity table leaves out.' + : ` Not yet measured${place}.`; } // Half rounded up, as JevGate rounds it. const percent = Math.floor((200 * right + labeled) / (2 * labeled)); - return ` Right ${percent}% of the time (${labeled} labels).`; + return ` Right ${percent}% of the time${place} (${labeled} labels).`; } /** One finding: its line, linked when the commit is known, the rule, whether diff --git a/test/render.test.cjs b/test/render.test.cjs index f4adc6f..465f23f 100644 --- a/test/render.test.cjs +++ b/test/render.test.cjs @@ -149,6 +149,26 @@ test('from JevGate 0.28, each finding says how often findings like it were right assert.match(body, /Finding at line 12
/, 'a report before 0.28 shows none'); }); +test('from JevGate 0.30, a preview language\'s findings say so, in its own precision', () => { + const body = render( + reportWith({ + 'install.sh': [ + finding('review', 3, { gate: 'measuring', preview: 'Bash', precision: { right: 34, labeled: 50 } }), + finding('consider', 9, { gate: 'measuring', preview: 'Bash', precision: { right: 2, labeled: 5 } }), + ], + 'src/lib.rs': [finding('review', 4, { gate: 'measuring', precision: { right: 46, labeled: 85 } })], + 'laws.bend': [finding('consider', 7, { rule: 'tests/laws', precision: { right: 0, labeled: 0 } })], + }), + RUN, + ); + assert.match(body, /Finding at line 3 Right 68% of the time in Bash \(50 labels\)\.
/); + assert.match(body, /Finding at line 9 Not yet measured in Bash\.
/); + assert.match(body, /Finding at line 7 Not yet measured: labeled only on Bend 2 projects, which the maturity table leaves out\.
/); + assert.ok(body.includes( + '\n1 review finding is reported without failing the gate: their rules and levels are still being measured (`jevgate rules` shows which fail it by default). 1 review finding and 1 consider finding are reported without failing the gate: Bash is in preview, and by default JevGate\'s own rules never fail it there.\n', + ), body); +}); + test('a passing gate with nothing new says so', () => { const body = render(reportWith({ 'a.js': [finding('note', 3)] }, { status: 'note', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); assert.ok(body.includes('### JevGate: gate passed\n')); From 3f6f76aa85edb304c95d7130751978359cfad269 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:41:07 -0300 Subject: [PATCH 9/9] Pin JevGate 0.30.0 in the examples The first example pinned 0.25.0 and the gateway one 0.26.0. 0.30.0 is the release this version of the action words its comment for (preview languages, per-language precision), and the gateways need 0.26.0 or later. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 94af817..8341c24 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ jobs: - uses: Tech-Byte-Frontier/jevgate-action@v1 with: api-key: ${{ secrets.TYPESAFE_API_KEY }} - version: 0.25.0 + version: 0.30.0 ``` The action installs a release binary (checked against its SHA-256), keeps JevGate's answer cache in the Actions cache so unchanged code costs nothing (and removes one the pull request commits, whose answers could clear its own code), and runs `jevgate check --base --format github`. It needs no Rust toolchain and runs on Linux, macOS and Windows runners. @@ -90,7 +90,7 @@ OpenRouter and Vercel AI Gateway also serve Jev. With JevGate 0.26.0 or later, p with: api-key: ${{ secrets.OPENROUTER_API_KEY }} api-key-kind: openrouter # or vercel - version: 0.26.0 + version: 0.30.0 ``` With an older version the check stops with an error instead of running without a key.