List every finding in one pull request comment

.github/workflows/test.yml

@@ -8,6 +8,20 @@ on:
 permissions:
   contents: read
 jobs:
+  unit:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Key kinds
+        shell: bash
+        run: bash test/key-kinds.sh
+      - name: Comment
+        shell: bash
+        run: node --test test/render.test.cjs test/comment.test.cjs
   dry-run:
     strategy:
       fail-fast: false
@@ -34,3 +48,78 @@ jobs:
           [ "$CODE" = 0 ] || { echo "::error::exit code $CODE"; exit 1; }
           jevgate --version
           grep -q '"version": "2.1.0"' jevgate.sarif || { echo "::error::no SARIF log"; exit 1; }
+  fixture:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # A change whose answers no cache holds: --cache-only writes a report
+      # and ends incomplete, with no key and nothing sent.
+      - name: Create a repository to check
+        shell: bash
+        run: bash test/fixture-repository.sh fixture
+      - id: incomplete
+        uses: ./
+        continue-on-error: true
+        with:
+          working-directory: fixture
+          base: HEAD~1
+          args: --cache-only
+          cache: "false"
+      # Nothing changed since HEAD: the gate passes. On a pull request the
+      # read-only token cannot comment, which must not fail the step.
+      - id: passed
+        uses: ./
+        with:
+          working-directory: fixture
+          base: HEAD
+          cache: "false"
+      - name: Check the outputs
+        shell: bash
+        env:
+          INCOMPLETE: ${{ steps.incomplete.outputs.exit-code }}
+          PASSED: ${{ steps.passed.outputs.exit-code }}
+          REPORT: ${{ steps.incomplete.outputs.report }}
+        run: |
+          [ "$INCOMPLETE" = 2 ] || { echo "::error::exit code $INCOMPLETE, not 2"; exit 1; }
+          [ "$PASSED" = 0 ] || { echo "::error::exit code $PASSED, not 0"; exit 1; }
+          # The path must open outside bash too, as upload-artifact opens it.
+          node -e 'require("fs").accessSync(process.argv[1])' "$REPORT" || { echo "::error::no report at $REPORT"; exit 1; }
+  comment:
+    # It writes to the pull request, so only where the token may.
+    if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Create a repository to check
+        shell: bash
+        run: bash test/fixture-repository.sh fixture
+      # Two checks in one job share one comment: the second replaces the first.
+      - uses: ./
+        with:
+          working-directory: fixture
+          base: HEAD
+          cache: "false"
+      - uses: ./
+        continue-on-error: true
+        with:
+          working-directory: fixture
+          base: HEAD~1
+          args: --cache-only
+          cache: "false"
+      - name: Check the comment
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PULL_REQUEST: ${{ github.event.pull_request.number }}
+        run: |
+          bodies=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST/comments" \
+            --jq '.[] | select(.user.type == "Bot" and (.body | startswith("<!-- jevgate-action comment key=comment%20fixture "))) | .body | @json')
+          [ "$(grep -c . <<< "$bodies")" = 1 ] || { echo "::error::expected one comment: $bodies"; exit 1; }
+          grep -q 'JevGate could not finish this run (exit code 2)' <<< "$bodies" || { echo "::error::no banner: $bodies"; exit 1; }

README.md

@@ -1,12 +1,13 @@
 # JevGate action
 
-Runs [JevGate](https://github.com/Tech-Byte-Frontier/jevgate), a code-review gate, on pull requests. It reviews only the files a pull request changes, annotates the changed lines with each finding, writes a job summary and fails the job when the gate fails.
+Runs [JevGate](https://github.com/Tech-Byte-Frontier/jevgate), a code-review gate, on pull requests. It reviews only the files a pull request changes, annotates the changed lines with each finding, lists them all in one pull request comment, writes a job summary and fails the job when the gate fails.
 
 ```yaml
 name: JevGate
 on: pull_request
 permissions:
   contents: read
+  pull-requests: write # the comment
 jobs:
   review:
     runs-on: ubuntu-latest
@@ -17,21 +18,23 @@ jobs:
       - uses: Tech-Byte-Frontier/jevgate-action@v1
         with:
           api-key: ${{ secrets.TYPESAFE_API_KEY }}
-          version: 0.25.0
+          version: 0.30.0
 ```
 
-The action installs a release binary (checked against its SHA-256), keeps JevGate's answer cache in the Actions cache so unchanged code costs nothing, and runs `jevgate check --base <pull request base> --format github`. It needs no Rust toolchain and runs on Linux, macOS and Windows runners.
+The action installs a release binary (checked against its SHA-256), keeps JevGate's answer cache in the Actions cache so unchanged code costs nothing (and removes one the pull request commits, whose answers could clear its own code), and runs `jevgate check --base <pull request base> --format github`. It needs no Rust toolchain and runs on Linux, macOS and Windows runners.
 
 ## Inputs
 
 | Input | Default | Meaning |
 |---|---|---|
-| `api-key` | | TypeSafe API key. [Get one](https://console.typesafe.ai/settings/keys) and save it as a repository secret |
+| `api-key` | | TypeSafe API key ([get one](https://console.typesafe.ai/settings/keys)), or an OpenRouter or Vercel AI Gateway key with `api-key-kind`. Save it as a repository secret |
+| `api-key-kind` | `typesafe` | Which service issued `api-key`: `typesafe`, `openrouter` or `vercel` (the gateways need JevGate 0.26.0 or later) |
 | `version` | `latest` | JevGate version; pin one for repeatable results |
-| `base` | the pull request's base commit | Review only files changed since this revision; empty on other events, which review the whole repository |
+| `base` | the pull request's base commit | Review only what changed since this revision: from JevGate 0.26.0 the changed lines of changed files (`--whole-files` in `args` for whole files), before it changed files whole; empty on other events, which review the whole repository |
 | `args` | | More `jevgate check` arguments, such as `--rule default --rule security --include-tests` |
 | `format` | `github` | `github`, `agent`, `json`, `jsonl`, `sarif` or `gitlab` |
 | `sarif-file` | | Also write the findings as SARIF to this path, for `upload-sarif` (JevGate 0.18.0 or later) |
+| `comment` | `true` | On pull requests, list every finding in [one comment](#the-pull-request-comment), updated on each run |
 | `cache` | `true` | Keep answers in the Actions cache |
 | `working-directory` | `.` | Repository root to check |
 
@@ -42,13 +45,23 @@ The action installs a release binary (checked against its SHA-256), keeps JevGat
 | `exit-code` | `0` gate passed, `1` gate failed, `2` run incomplete |
 | `report` | Path of the full JSON report (`.jevgate/latest.json`), to upload as an artifact |
 
+## The pull request comment
+
+GitHub shows at most 10 error and 10 warning annotations per step, so on pull requests the action also lists every finding in one comment and updates it on each run. Reviews come first, then considers, then notes (collapsed), each grouped by file with a link to the line. The comment gives the gate's result and the run's API requests, input tokens and cost; from JevGate 0.26.0 it also marks each finding that fails the gate and counts those reported without failing it while their rules are still being measured, and from 0.28.0 each finding ends with how often findings of its rule and level were right on projects JevGate was never tuned on. When the run could not finish (exit code 2: no key, a provider error such as HTTP 402, or the request budget), it says so first, with the reasons.
+
+- It needs `pull-requests: write`. Where the token can't comment, as on pull requests from forks, the run says so in the log and the job summary and carries on.
+- Each job, and each `working-directory`, keeps its own comment; the jobs of a matrix share one. A run for an older push never replaces a newer run's comment.
+- A run too large for one comment (GitHub's limit is 65,536 characters) leaves out notes first, then the lowest-ranked considers, and says how many; the JSON report (the `report` output) keeps them all.
+- `comment: false` turns it off.
+
 ## Code scanning
 
 `sarif-file` also writes the findings as SARIF, replayed from the answers the check just cached, so it costs nothing. Upload it to show them in the repository's Security tab and on pull requests (needs JevGate 0.18.0 or later):
 
 ```yaml
 permissions:
   contents: read
+  pull-requests: write
   security-events: write
 jobs:
   review:
@@ -68,6 +81,20 @@ jobs:
           category: jevgate
 ```
 
+## Keys from OpenRouter or Vercel AI Gateway
+
+OpenRouter and Vercel AI Gateway also serve Jev. With JevGate 0.26.0 or later, pass their key and say which it is; the action gives it to JevGate as `OPENROUTER_API_KEY` or `AI_GATEWAY_API_KEY`, and no other kind's key, so a key the job holds for something else is never used:
+
+```yaml
+      - uses: Tech-Byte-Frontier/jevgate-action@v1
+        with:
+          api-key: ${{ secrets.OPENROUTER_API_KEY }}
+          api-key-kind: openrouter # or vercel
+          version: 0.30.0
+```
+
+With an older version the check stops with an error instead of running without a key.
+
 ## Pull requests from forks
 
 GitHub doesn't give secrets to pull requests from forks, so the run there ends incomplete with "No API key configured". Skip the job for them:

action.yml

@@ -6,15 +6,19 @@ branding:
   color: blue
 inputs:
   api-key:
-    description: TypeSafe API key, usually secrets.TYPESAFE_API_KEY. Pull requests from forks don't receive secrets.
+    description: API key, usually secrets.TYPESAFE_API_KEY; `api-key-kind` says which service issued it. Pull requests from forks don't receive secrets.
     required: false
     default: ""
+  api-key-kind:
+    description: Which service issued `api-key`, `typesafe`, `openrouter` (OpenRouter) or `vercel` (Vercel AI Gateway). The gateways need JevGate 0.26.0 or later.
+    required: false
+    default: typesafe
   version:
     description: JevGate version to install, such as 0.17.0, or `latest`. Pin one for repeatable results.
     required: false
     default: latest
   base:
-    description: Review only files changed since this revision. Defaults to the pull request's base commit; empty on other events, which review the whole repository.
+    description: Review only what changed since this revision. JevGate 0.26.0 and later ask about and report only the changed lines of changed files (add `--whole-files` to `args` for whole files); earlier versions review changed files whole. Defaults to the pull request's base commit; empty on other events, which review the whole repository.
     required: false
     default: ""
   args:
@@ -29,6 +33,10 @@ inputs:
     description: Also write the findings as SARIF to this path, for github/codeql-action/upload-sarif. Replays the check from its cached answers, so it costs nothing. Needs JevGate 0.18.0 or later.
     required: false
     default: ""
+  comment:
+    description: "On pull requests, list every finding in one comment, updated in place on each run. Needs `pull-requests: write`; without it, as on pull requests from forks, the run says so and carries on."
+    required: false
+    default: "true"
   cache:
     description: Keep answers in the Actions cache, so unchanged code costs nothing on the next run.
     required: false
@@ -52,6 +60,13 @@ runs:
       env:
         JEVGATE_VERSION: ${{ inputs.version }}
       run: bash "$GITHUB_ACTION_PATH/install.sh"
+    # Answers a pull request commits under .jevgate/cache could clear its
+    # own code; JevGate 0.28 and later never reads a cache file Git tracks,
+    # and this keeps earlier versions from reading one too.
+    - name: Leave out answers the change committed
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      run: rm -rf .jevgate/cache
     - name: Restore answers
       if: inputs.cache == 'true'
       uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -64,9 +79,27 @@ runs:
       shell: bash
       working-directory: ${{ inputs.working-directory }}
       env:
-        TYPESAFE_API_KEY: ${{ inputs.api-key }}
+        API_KEY: ${{ inputs.api-key }}
+        API_KEY_KIND: ${{ inputs.api-key-kind }}
         BASE: ${{ inputs.base || github.event.pull_request.base.sha }}
         FORMAT: ${{ inputs.format }}
         ARGS: ${{ inputs.args }}
         SARIF_FILE: ${{ inputs.sarif-file }}
       run: bash "$GITHUB_ACTION_PATH/check.sh"
+    # After a failed gate or an incomplete run too; not when the check never started.
+    - name: Comment
+      if: ${{ !cancelled() && inputs.comment == 'true' && steps.check.outputs.exit-code != '' }}
+      uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+      env:
+        JEVGATE_ACTION_PATH: ${{ github.action_path }}
+        JEVGATE_EXIT_CODE: ${{ steps.check.outputs.exit-code }}
+        JEVGATE_REPORT: ${{ steps.check.outputs.comment-report }}
+        JEVGATE_COMMIT: ${{ steps.check.outputs.commit }}
+        JEVGATE_PREFIX: ${{ steps.check.outputs.prefix }}
+        JEVGATE_VERSION: ${{ steps.check.outputs.version }}
+        JEVGATE_WORKING_DIRECTORY: ${{ inputs.working-directory }}
+      with:
+        retries: 3
+        script: |
+          const comment = require(require('node:path').join(process.env.JEVGATE_ACTION_PATH, 'comment.cjs'));
+          await comment.run({ github, context, core });

check.sh

@@ -2,20 +2,79 @@
 # Run `jevgate check` and pass its exit code on: 0 passed, 1 failed, 2 incomplete.
 set -uo pipefail
 
+report=.jevgate/latest.json
+version=$(jevgate --version 2> /dev/null)
+# This run's report for the comment step, empty when the check wrote none.
+comment_report=
+
+# A path Node can open: on Windows, Git Bash's $PWD (/d/a/...) is not one.
+native() {
+    if command -v cygpath > /dev/null; then cygpath -m "$1"; else echo "$1"; fi
+}
+
+# Write the step's outputs, then exit with CODE.
+finish() {
+    {
+        echo "exit-code=$1"
+        echo "report=$(native "$PWD")/$report"
+        echo "comment-report=$comment_report"
+        echo "commit=$(git rev-parse HEAD 2> /dev/null)"
+        echo "prefix=$(git rev-parse --show-prefix 2> /dev/null)"
+        echo "version=$version"
+    } >> "$GITHUB_OUTPUT"
+    exit "$1"
+}
+
+# The key goes in the variable JevGate reads for its kind.
+case "$API_KEY_KIND" in
+    typesafe) key_variable=TYPESAFE_API_KEY ;;
+    openrouter) key_variable=OPENROUTER_API_KEY ;;
+    vercel) key_variable=AI_GATEWAY_API_KEY ;;
+    *)
+        echo "::error::api-key-kind is typesafe, openrouter or vercel, not $API_KEY_KIND."
+        finish 2
+        ;;
+esac
+if [ "$API_KEY_KIND" != typesafe ]; then
+    # Older versions read only TYPESAFE_API_KEY and would say no key is configured.
+    IFS=. read -r major minor _ <<< "${version##* }"
+    if [ "$major" = 0 ] && [ "${minor:-0}" -lt 26 ] 2> /dev/null; then
+        echo "::error::api-key-kind: $API_KEY_KIND needs JevGate 0.26.0 or later; this is ${version##* }."
+        finish 2
+    fi
+fi
+# Only a key that was given, so an empty input leaves one set in the job's env.
+if [ -n "$API_KEY" ]; then
+    export "$key_variable=$API_KEY"
+fi
+# And only that kind's: a key the job holds for another service is never spent.
+for variable in TYPESAFE_API_KEY OPENROUTER_API_KEY AI_GATEWAY_API_KEY; do
+    if [ "$variable" != "$key_variable" ]; then unset "$variable"; fi
+done
+
 command=(jevgate check)
 if [ -n "$BASE" ]; then
     if ! git cat-file -e "$BASE^{commit}" 2> /dev/null; then
         echo "::error::The base revision $BASE is not in the checkout. Check out with fetch-depth: 0 so --base can find the fork point."
-        exit 2
+        finish 2
     fi
     command+=(--base "$BASE")
 fi
 # Extra arguments are split on spaces, as written in the workflow.
 read -r -a extra <<< "$ARGS"
 command+=(${extra[@]+"${extra[@]}"})
 
+# Each report JevGate writes is a new generation, so its checksum changes;
+# one left from an earlier check is not this run's.
+before=
+if [ -f "$report" ]; then before=$(cksum < "$report"); fi
 "${command[@]}" --format "$FORMAT"
 code=$?
+if [ -f "$report" ] && [ "$(cksum < "$report")" != "$before" ]; then
+    # Saved before the SARIF replay publishes its own, which sent no request.
+    saved="$RUNNER_TEMP/jevgate-report-$$.json"
+    cp "$report" "$saved" && comment_report=$saved
+fi
 
 # The same check again from the answers just cached: no request is sent.
 if [ -n "${SARIF_FILE:-}" ]; then
@@ -26,9 +85,7 @@ if [ -n "${SARIF_FILE:-}" ]; then
     fi
 fi
 
-echo "exit-code=$code" >> "$GITHUB_OUTPUT"
-echo "report=$PWD/.jevgate/latest.json" >> "$GITHUB_OUTPUT"
-if [ "$code" = 2 ] && [ -z "${TYPESAFE_API_KEY:-}" ]; then
-    echo "::error::No TypeSafe API key. Pass api-key: \${{ secrets.TYPESAFE_API_KEY }}. Pull requests from forks don't receive secrets; skip the job for them."
+if [ "$code" = 2 ] && [ -z "${!key_variable:-}" ]; then
+    echo "::error::No API key. Pass api-key: \${{ secrets.$key_variable }}. Pull requests from forks don't receive secrets; skip the job for them."
 fi
-exit "$code"
+finish "$code"