diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2474fe4..3714407 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,20 @@ on: permissions: contents: read jobs: + unit: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Key kinds + shell: bash + run: bash test/key-kinds.sh + - name: Comment + shell: bash + run: node --test test/render.test.cjs test/comment.test.cjs dry-run: strategy: fail-fast: false @@ -34,3 +48,78 @@ jobs: [ "$CODE" = 0 ] || { echo "::error::exit code $CODE"; exit 1; } jevgate --version grep -q '"version": "2.1.0"' jevgate.sarif || { echo "::error::no SARIF log"; exit 1; } + fixture: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # A change whose answers no cache holds: --cache-only writes a report + # and ends incomplete, with no key and nothing sent. + - name: Create a repository to check + shell: bash + run: bash test/fixture-repository.sh fixture + - id: incomplete + uses: ./ + continue-on-error: true + with: + working-directory: fixture + base: HEAD~1 + args: --cache-only + cache: "false" + # Nothing changed since HEAD: the gate passes. On a pull request the + # read-only token cannot comment, which must not fail the step. + - id: passed + uses: ./ + with: + working-directory: fixture + base: HEAD + cache: "false" + - name: Check the outputs + shell: bash + env: + INCOMPLETE: ${{ steps.incomplete.outputs.exit-code }} + PASSED: ${{ steps.passed.outputs.exit-code }} + REPORT: ${{ steps.incomplete.outputs.report }} + run: | + [ "$INCOMPLETE" = 2 ] || { echo "::error::exit code $INCOMPLETE, not 2"; exit 1; } + [ "$PASSED" = 0 ] || { echo "::error::exit code $PASSED, not 0"; exit 1; } + # The path must open outside bash too, as upload-artifact opens it. + node -e 'require("fs").accessSync(process.argv[1])' "$REPORT" || { echo "::error::no report at $REPORT"; exit 1; } + comment: + # It writes to the pull request, so only where the token may. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Create a repository to check + shell: bash + run: bash test/fixture-repository.sh fixture + # Two checks in one job share one comment: the second replaces the first. + - uses: ./ + with: + working-directory: fixture + base: HEAD + cache: "false" + - uses: ./ + continue-on-error: true + with: + working-directory: fixture + base: HEAD~1 + args: --cache-only + cache: "false" + - name: Check the comment + shell: bash + env: + GH_TOKEN: ${{ github.token }} + PULL_REQUEST: ${{ github.event.pull_request.number }} + run: | + bodies=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PULL_REQUEST/comments" \ + --jq '.[] | select(.user.type == "Bot" and (.body | startswith("/.exec(body); + return run ? BigInt(run[1]) : 0n; +} + +/** Text outside code spans, escaped so it stays text: no links, HTML or mentions. */ +function escapeText(text) { + return text + .replace(/[\\[\]]/g, '\\$&') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/@(?=\w)/g, '@\u200b'); +} + +/** The length of the run of backticks at `index`. */ +function backticks(text, index) { + let end = index; + while (text[end] === '`') { + end += 1; + } + return end - index; +} + +/** Where a run of exactly `width` backticks starts at or after `from`, or -1. */ +function closing(text, from, width) { + for (let at = text.indexOf('`', from); at >= 0; ) { + const run = backticks(text, at); + if (run === width) { + return at; + } + at = text.indexOf('`', at + run); + } + return -1; +} + +/** Report text on one line of Markdown. Messages quote code from the change, so + * outside code spans nothing can add HTML, links, a comment marker or mentions; + * code spans are kept as JevGate wrote them. They are found as CommonMark finds + * them, a run of backticks closed by a run of the same length, so text can + * never pass as code. */ +function inline(text) { + const flat = String(text).replace(/\s*[\r\n]+\s*/g, ' '); + let out = ''; + let at = 0; + for (let open = flat.indexOf('`'); open >= 0; open = flat.indexOf('`', at)) { + const width = backticks(flat, open); + const close = closing(flat, open + width, width); + if (close < 0) { + // An unclosed run is escaped: left as it is, it could close on a + // backtick of the next text in the same line and expose that + // text's code span as Markdown and HTML. + out += escapeText(flat.slice(at, open)) + '\\`'.repeat(width); + at = open + width; + } else { + out += escapeText(flat.slice(at, open)) + flat.slice(open, close + width); + at = close + width; + } + } + return out + escapeText(flat.slice(at)); +} + +/** `text` as one code span, fenced by more backticks than any run it holds. */ +function code(text) { + const flat = String(text).replace(/[\r\n]+/g, ' '); + const longest = Math.max(0, ...(flat.match(/`+/g) || []).map((run) => run.length)); + const fence = '`'.repeat(longest + 1); + const pad = flat.startsWith('`') || flat.endsWith('`') ? ' ' : ''; + return `${fence}${pad}${flat}${pad}${fence}`; +} + +/** A path in a URL: each segment encoded, parentheses too, so it cannot end a + * Markdown link early. */ +function urlPath(path) { + const encode = (segment) => + encodeURIComponent(segment).replace(/[()]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`); + return path.split('/').map(encode).join('/'); +} + +/** A report path as a repository path, with forward slashes. */ +function repositoryPath(path, run) { + return run.prefix + (run.windows ? path.replace(/\\/g, '/') : path); +} + +/** A link to `path` at the checked commit, with a line anchor or none. */ +function blobUrl(path, anchor, run) { + return `${run.serverUrl}/${run.repository}/blob/${run.commit}/${urlPath(repositoryPath(path, run))}${anchor}`; +} + +/** Exit codes 0 and 1 mean the gate was applied; any other code stopped the run. */ +function finished(exitCode) { + return exitCode === 0 || exitCode === 1; +} + +/** Findings nobody accepted, at a known level, with their file's path. */ +function listed(report) { + const known = new Set(LEVELS.map((level) => level.strength)); + return (report.files || []).flatMap((file) => + (file.findings || []) + .filter((finding) => known.has(finding.strength) && !finding.baselined && !finding.suppressed) + .map((finding) => ({ path: file.path, finding })), + ); +} + +function levelIndex(strength) { + return LEVELS.findIndex((level) => level.strength === strength); +} + +/** Findings in the order they are kept when the comment must be cut: by level, + * then by JevGate's rank. */ +function byPriority(findings) { + return [...findings].sort( + (a, b) => + levelIndex(a.finding.strength) - levelIndex(b.finding.strength) || + (b.finding.rank || 0) - (a.finding.rank || 0), + ); +} + +function title(exitCode) { + if (!finished(exitCode)) { + return '### JevGate: run incomplete'; + } + return exitCode === 1 ? '### JevGate: gate failed' : '### JevGate: gate passed'; +} + +/** Distinct reasons the run could not finish, with the number of files each + * stopped: run errors first (no files), then the errors of files not judged. */ +function reasons(report) { + const files = new Map(); + for (const error of report.errors || []) { + files.set(error, 0); + } + for (const file of report.files || []) { + if (file.status === 'error') { + const error = file.error || 'Not judged'; + files.set(error, (files.get(error) || 0) + 1); + } + } + return [...files]; +} + +/** The loud part of an incomplete run: what it means, then why. */ +function banner(report, run) { + const lines = [ + '> [!CAUTION]', + `> **JevGate could not finish this run (exit code ${run.exitCode}).** The gate was not applied, and findings may be missing.`, + ]; + const why = report ? reasons(report) : []; + for (const [reason, files] of why.slice(0, MAX_REASONS)) { + lines.push(`> - ${inline(reason)}${files ? ` (${count(files, 'file')})` : ''}`); + } + const others = why.slice(MAX_REASONS).reduce((sum, [, files]) => sum + files, 0); + if (others > 0) { + lines.push(`> - ${count(others, 'more file')} stopped for other reasons.`); + } + if (!report) { + lines.push(`>\n> JevGate stopped before writing a report; the [job log](${run.runUrl}) says why.`); + } else if (why.length === 0) { + lines.push(`>\n> The [job log](${run.runUrl}) says why.`); + } + return lines.join('\n'); +} + +/** This run's cost: paid input tokens in dollars for the priced model, "cost + * unknown" when requests were answered without a priced token count (a + * gateway may answer without usage), or `null` when nothing was paid for. */ +function cost(report) { + const tokens = report.paid_input_tokens || 0; + if (tokens === 0) { + const answered = Object.values(report.stages || {}).reduce( + (sum, stage) => sum + (stage.successful_requests || 0), + 0, + ); + return answered > 0 ? 'cost unknown' : null; + } + // The model that answered, else the one asked for. + const models = new Set((report.files || []).map((file) => file.model).filter(Boolean)); + const model = models.size > 0 ? [...models].join(', ') : report.requested_model; + if (model !== PRICED_MODEL) { + return 'cost unknown'; + } + return `~$${((tokens * USD_PER_MILLION_INPUT_TOKENS) / 1e6).toFixed(4)}`; +} + +/** From JevGate 0.26.0 each finding records how the gate counted it (`gate`: + * `fails`, `measuring` or `advisory`); earlier reports have no such field, and + * the comment then marks nothing. */ +const FAILS = 'fails'; +const MEASURING = 'measuring'; + +/** `entries` counted by level ("1 review finding and 2 consider findings"), + * and whether that is one finding. */ +function byLevel(entries) { + const counts = LEVELS.map((level) => [ + level, + entries.filter((entry) => entry.finding.strength === level.strength).length, + ]).filter(([, n]) => n > 0); + return [counts.map(([level, n]) => count(n, level.noun)).join(' and '), entries.length === 1]; +} + +/** Why the listed findings the gate reported without failing did not fail + * it: their rules and levels are still being measured, or, from JevGate + * 0.30, their file's language is in preview (`preview` names it), where + * JevGate's own rules never fail the default gate. `null` when none. */ +function measuring(all) { + const reported = all.filter((entry) => entry.finding.gate === MEASURING); + const measured = reported.filter((entry) => !entry.finding.preview); + const previewed = reported.filter((entry) => entry.finding.preview); + const reasons = []; + if (measured.length > 0) { + const [counted, one] = byLevel(measured); + reasons.push(`${counted} ${one ? 'is' : 'are'} reported without failing the gate: their rules and levels are still being measured (\`jevgate rules\` shows which fail it by default).`); + } + if (previewed.length > 0) { + const [counted, one] = byLevel(previewed); + const languages = [...new Set(previewed.map((entry) => entry.finding.preview))].sort(); + const which = languages.length === 1 ? `${languages[0]} is` : `${languages.join(', ')} are`; + reasons.push(`${counted} ${one ? 'is' : 'are'} reported without failing the gate: ${which} in preview, and by default JevGate's own rules never fail it there.`); + } + return reasons.length > 0 ? reasons.join(' ') : null; +} + +/** The gate's reasons as JevGate gave them, what it reported without failing, + * the run's size and cost, and the files left undecided. */ +function outcome(report, all) { + const lines = []; + const gate = report.gate; + if (gate && !gate.passed && (gate.reasons || []).length > 0) { + lines.push(`Gate failed: ${gate.reasons.map(inline).join('; ')}.`); + } + const measured = measuring(all); + if (measured) { + lines.push(measured); + } + const files = report.files || []; + const usage = [ + count(files.length, 'file'), + count(report.api_requests || 0, 'API request'), + count(report.paid_input_tokens || 0, 'input token'), + cost(report), + ]; + lines.push(usage.filter(Boolean).join(' · ')); + const uncertain = files.filter((file) => + Object.values(file.dimensions || {}).some((dimension) => dimension.status === 'uncertain'), + ).length; + const context = files.filter((file) => file.status === 'needs-context').length; + const open = [ + uncertain > 0 ? `${count(uncertain, 'file')} with uncertain units` : '', + context > 0 ? `${count(context, 'file')} needing context` : '', + ].filter(Boolean); + if (open.length > 0) { + lines.push(`${open.join(' · ')}.`); + } + return lines.join('\n\n'); +} + +/** From JevGate 0.28 each review and consider records how often findings of + * its rule and level were right on projects JevGate was never tuned on + * (`precision`: `right` of `labeled`), and its message no longer ends with a + * probability. Below this many labels, JevGate says it is not yet measured. */ +const MIN_LABELS = 20; + +/** The rule JevGate labels only on Bend 2 projects, which its table of + * precision leaves out; its findings say so. */ +const BEND_2_RULE = 'tests/laws'; + +/** How often findings like it were right, worded as JevGate's own output + * words it: in a preview language (`preview`, from 0.30) that language's + * own; nothing for a note or a report before 0.28. */ +function precision(finding) { + const { right, labeled } = finding.precision || {}; + if (!Number.isInteger(right) || !Number.isInteger(labeled)) { + return ''; + } + const place = finding.preview ? ` in ${finding.preview}` : ''; + if (labeled < MIN_LABELS) { + const bend = labeled === 0 && !finding.preview && finding.rule === BEND_2_RULE; + return bend + ? ' Not yet measured: labeled only on Bend 2 projects, which the maturity table leaves out.' + : ` Not yet measured${place}.`; + } + // Half rounded up, as JevGate rounds it. + const percent = Math.floor((200 * right + labeled) / (2 * labeled)); + return ` Right ${percent}% of the time${place} (${labeled} labels).`; +} + +/** One finding: its line, linked when the commit is known, the rule, whether + * it fails the gate, the message, how often findings like it were right and + * the next step. */ +function item({ path, finding }, run) { + const location = (finding.locations || []).find( + (l) => l.path === path && l.start_line === finding.line && l.end_line > finding.line, + ); + const anchor = location ? `#L${finding.line}-L${location.end_line}` : `#L${finding.line}`; + const line = run.commit ? `[Line ${finding.line}](${blobUrl(path, anchor, run)})` : `Line ${finding.line}`; + const fails = finding.gate === FAILS ? ' (fails the gate)' : ''; + return `- ${line} ${code(finding.rule)}${fails}: ${inline(finding.message)}${precision(finding)}
→ ${inline(finding.action)}`; +} + +/** A file's heading and its findings by line. */ +function fileBlock(path, entries, run) { + const name = code(repositoryPath(path, run)); + const heading = run.commit ? `[${name}](${blobUrl(path, '', run)})` : name; + const items = [...entries].sort((a, b) => a.finding.line - b.finding.line).map((entry) => item(entry, run)); + return [`**${heading}**`, ...items].join('\n'); +} + +/** One level's findings grouped by file, the files by path as the pull + * request lists them. Reviews are open; notes, and considers past + * `OPEN_CONSIDERS`, are collapsed. */ +function section(level, shown, total, run) { + const byPath = new Map(); + for (const entry of shown) { + const group = byPath.get(entry.path); + if (group) { + group.push(entry); + } else { + byPath.set(entry.path, [entry]); + } + } + const blocks = [...byPath.keys()].sort().map((path) => fileBlock(path, byPath.get(path), run)); + const counted = shown.length === total ? number(total) : `${number(shown.length)} of ${number(total)}`; + if (level.strength === 'review' || (level.strength === 'consider' && total <= OPEN_CONSIDERS)) { + return [`#### ${level.title} (${counted})`, ...blocks].join('\n\n'); + } + const optional = level.strength === 'note' ? ', optional' : ''; + return [`
${level.title} (${counted}${optional})`, ...blocks, '
'].join('\n\n'); +} + +/** A section for each level with findings kept, then a line for what was cut. */ +function sections(kept, all, run) { + const parts = []; + const cut = []; + for (const level of LEVELS) { + const atLevel = (entry) => entry.finding.strength === level.strength; + const shown = kept.filter(atLevel); + const total = all.filter(atLevel).length; + if (shown.length > 0) { + parts.push(section(level, shown, total, run)); + } + if (total > shown.length) { + cut.push(count(total - shown.length, level.noun)); + } + } + if (cut.length > 0) { + parts.push( + `**${count(all.length - kept.length, 'more finding')}** did not fit in this comment: ${cut.join(', ')}. The JSON report (the action's \`report\` output) lists them all.`, + ); + } + return parts; +} + +function footer(run) { + const parts = [run.version ? inline(run.version) : 'JevGate']; + if (run.commit) { + parts.push(`commit ${run.commit.slice(0, 7)}`); + } + parts.push(`[workflow run](${run.runUrl})`, 'updated on each run'); + return `${parts.join(' · ')}`; +} + +/** The comment for `report` (`null` when the run wrote none), listing the + * findings in `kept` of `all`. */ +function compose(report, run, kept, all) { + const parts = [`${marker(run.key)}run=${run.runId} -->\n${title(run.exitCode)}`]; + if (!finished(run.exitCode)) { + parts.push(banner(report, run)); + } + if (report && report.status === 'no-changed-source') { + parts.push('No supported file changed since the base revision.'); + } else if (report) { + parts.push(outcome(report, all)); + if (finished(run.exitCode) && !all.some((entry) => entry.finding.strength !== 'note')) { + parts.push('No new review or consider findings.'); + } + parts.push(...sections(kept, all, run)); + const accepted = (report.files || []) + .flatMap((file) => file.findings || []) + .filter((finding) => finding.baselined || finding.suppressed).length; + if (accepted > 0) { + const verb = accepted === 1 ? 'is' : 'are'; + parts.push(`${count(accepted, 'finding')} accepted by the baseline or an inline allow ${verb} not listed.`); + } + } + parts.push(footer(run)); + return `${parts.join('\n\n')}\n`; +} + +/** The comment for a report, or for a run that wrote none (`report` null). + * It lists as many findings as fit under GitHub's limit, cutting the lowest + * ranked first: notes before considers before reviews. */ +function render(report, run) { + const all = report ? byPriority(listed(report)) : []; + const body = (n) => compose(report, run, all.slice(0, n), all); + const fits = (n) => Buffer.byteLength(body(n), 'utf8') <= MAX_BODY_BYTES; + if (fits(all.length)) { + return body(all.length); + } + // Bisect over the counts that leave a "N more" line: each finding kept + // adds a line longer than the digits it saves there, so the size grows + // with the count. + let low = 0; + let high = all.length - 1; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if (fits(middle)) { + low = middle; + } else { + high = middle - 1; + } + } + return body(low); +} + +module.exports = { render, marker, runOf, finished, inline, code, MAX_BODY_BYTES }; diff --git a/test/comment.test.cjs b/test/comment.test.cjs new file mode 100644 index 0000000..9ba47b4 --- /dev/null +++ b/test/comment.test.cjs @@ -0,0 +1,194 @@ +// Tests for posting the comment: `node --test test/comment.test.cjs`, with an +// in-memory issues API standing in for GitHub. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { run, commentKey } = require('../comment.cjs'); +const { marker } = require('../render.cjs'); + +const COMMIT = '0123456789abcdef0123456789abcdef01234567'; + +test('each job and working directory has its own comment', () => { + assert.equal(commentKey('review', '.'), 'review'); + assert.equal(commentKey('review', './'), 'review'); + assert.equal(commentKey('review', 'services/api'), 'review services/api'); + assert.equal(commentKey('review', './services/api/'), 'review services/api'); +}); + +const BOT = { login: 'github-actions[bot]', type: 'Bot' }; + +function refusal(status, message) { + return Object.assign(new Error(message), { status }); +} + +function fakeGitHub(comments = [], failures = {}) { + const calls = []; + let next = 1000; + const url = (id) => `https://github.com/o/r/pull/7#issuecomment-${id}`; + const fail = (name) => { + if (failures[name]) { + throw failures[name]; + } + }; + const issues = { + async listComments(params) { + calls.push(['list', params.issue_number, params.per_page]); + fail('list'); + return { data: comments.map((c) => ({ ...c, html_url: url(c.id) })) }; + }, + async createComment({ issue_number, body }) { + calls.push(['create', issue_number]); + fail('create'); + const comment = { id: next++, body, user: BOT }; + comments.push(comment); + return { data: { ...comment, html_url: url(comment.id) } }; + }, + async updateComment({ comment_id, body }) { + calls.push(['update', comment_id]); + fail('update'); + comments.find((c) => c.id === comment_id).body = body; + return { data: { id: comment_id, html_url: url(comment_id) } }; + }, + async deleteComment({ comment_id }) { + calls.push(['delete', comment_id]); + fail('delete'); + comments.splice(comments.findIndex((c) => c.id === comment_id), 1); + return {}; + }, + }; + const github = { rest: { issues }, paginate: async (method, params) => (await method(params)).data }; + return { github, comments, calls }; +} + +function fakeCore() { + const log = { info: [], warning: [], summary: [] }; + const summary = { + addRaw(text) { + log.summary.push(text); + return summary; + }, + async write() { + return summary; + }, + }; + return { core: { info: (m) => log.info.push(m), warning: (m) => log.warning.push(m), summary }, log }; +} + +function context(overrides = {}) { + return { + payload: { pull_request: { number: 7, head: { repo: { full_name: 'o/r' } } } }, + repo: { owner: 'o', repo: 'r' }, + job: 'review', + runId: 123, + serverUrl: 'https://github.com', + ...overrides, + }; +} + +function env(name, exitCode, extra = {}) { + return { + JEVGATE_REPORT: name ? path.join(__dirname, 'reports', `${name}.json`) : '', + JEVGATE_EXIT_CODE: String(exitCode), + JEVGATE_COMMIT: COMMIT, + JEVGATE_PREFIX: '', + JEVGATE_VERSION: 'jevgate 0.25.0', + JEVGATE_WORKING_DIRECTORY: '.', + ...extra, + }; +} + +async function step({ comments, failures, ctx = context(), environment = env('base-run', 1) } = {}) { + const fake = fakeGitHub(comments, failures); + const { core, log } = fakeCore(); + await run({ github: fake.github, context: ctx, core, env: environment, platform: 'linux' }); + return { ...fake, log }; +} + +test('the first run creates the comment and links it from the job summary', async () => { + const { comments, calls, log } = await step(); + assert.deepEqual(calls, [['list', 7, 100], ['create', 7]]); + assert.equal(comments.length, 1); + assert.ok(comments[0].body.startsWith('\n### JevGate: gate failed')); + assert.deepEqual(log.warning, []); + assert.ok(log.summary[0].includes('[a pull request comment](https://github.com/o/r/pull/7#issuecomment-1000)')); +}); + +test('a later run updates its own comment in place and leaves others alone', async () => { + const others = [ + { id: 1, body: `${marker('review')}run=100 -->\nquoted by a person`, user: { login: 'someone', type: 'User' } }, + { id: 2, body: `${marker('security')}run=100 -->\nanother job`, user: BOT }, + { id: 3, body: `${marker('review')}run=100 -->\nold findings`, user: BOT }, + ]; + const { comments, calls } = await step({ comments: others }); + assert.deepEqual(calls.slice(1), [['update', 3]]); + assert.ok(comments.find((c) => c.id === 3).body.includes('### JevGate: gate failed')); + assert.ok(comments.find((c) => c.id === 1).body.includes('quoted by a person')); + assert.ok(comments.find((c) => c.id === 2).body.includes('another job')); +}); + +test('an unchanged comment is not written again', async () => { + const first = await step(); + const again = await step({ comments: first.comments }); + assert.deepEqual(again.calls, [['list', 7, 100]]); +}); + +test('copies left by racing first runs are deleted, keeping the oldest', async () => { + const racing = [4, 5, 6].map((id) => ({ id, body: `${marker('review')}run=123 -->\nrace`, user: BOT })); + const { comments, calls } = await step({ comments: racing }); + assert.deepEqual(calls.slice(1), [['update', 4], ['delete', 5], ['delete', 6]]); + assert.deepEqual(comments.map((c) => c.id), [4]); +}); + +test('a run older than the comment leaves it alone', async () => { + const newer = [{ id: 8, body: `${marker('review')}run=124 -->\nnewer`, user: BOT }]; + const { comments, calls, log } = await step({ comments: newer }); + assert.deepEqual(calls, [['list', 7, 100]]); + assert.ok(comments[0].body.endsWith('newer')); + assert.ok(log.info.some((m) => m.includes('newer run'))); +}); + +test('a read-only token is a warning in the log and the summary, not a failure', async () => { + const failures = { create: refusal(403, 'Resource not accessible by integration') }; + const { log } = await step({ failures }); + assert.equal(log.warning.length, 1); + assert.ok(log.warning[0].startsWith('The token cannot comment on this pull request: give the job `permissions: pull-requests: write`, or set `comment: false`.')); + assert.ok(log.warning[0].includes('(GitHub: Resource not accessible by integration)')); + assert.ok(log.summary[0].includes('pull-requests: write')); +}); + +test('a fork is told why it gets no comment', async () => { + const fork = context({ payload: { pull_request: { number: 7, head: { repo: { full_name: 'someone/r' } } } } }); + const { log } = await step({ ctx: fork, failures: { list: refusal(404, 'Not Found') } }); + assert.ok(log.warning[0].startsWith('GitHub gives workflows on pull requests from forks a read-only token')); +}); + +test('any other failure is a warning, never an exception', async () => { + const failed = await step({ failures: { update: refusal(500, 'Server Error') }, comments: [{ id: 3, body: `${marker('review')}run=1 -->`, user: BOT }] }); + assert.deepEqual(failed.log.warning, ['JevGate could not update its pull request comment: Server Error']); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'jevgate-comment-')); + const broken = path.join(directory, 'latest.json'); + fs.writeFileSync(broken, '{"files": ['); + const unreadable = await step({ environment: env('', 1, { JEVGATE_REPORT: broken }) }); + assert.equal(unreadable.calls.length, 0); + assert.ok(unreadable.log.warning[0].startsWith('JevGate could not update its pull request comment:')); +}); + +test('events without a pull request, and runs without a report, post nothing', async () => { + const push = await step({ ctx: context({ payload: {} }) }); + assert.equal(push.calls.length, 0); + const dryRun = await step({ environment: env('', 0) }); + assert.equal(dryRun.calls.length, 0); + assert.ok(dryRun.log.info[0].includes('no comment')); + const stopped = await step({ environment: env('', 2) }); + assert.deepEqual(stopped.calls, [['list', 7, 100], ['create', 7]]); + assert.ok(stopped.comments[0].body.includes('JevGate stopped before writing a report')); +}); + +test('a job checking a subdirectory keys its comment by it', async () => { + const { comments } = await step({ environment: env('base-run', 1, { JEVGATE_WORKING_DIRECTORY: 'services/api' }) }); + assert.ok(comments[0].body.startsWith('')); +}); diff --git a/test/fixture-repository.sh b/test/fixture-repository.sh new file mode 100644 index 0000000..5e42062 --- /dev/null +++ b/test/fixture-repository.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Create a repository at DIR whose last commit adds one function. Its answers +# are never in a cache, so `jevgate check --base HEAD~1 --cache-only` writes a +# report and ends incomplete (exit 2), with no key and nothing sent. +set -euo pipefail + +dir=$1 +if [ -e "$dir" ]; then + echo "$dir already exists" >&2 + exit 1 +fi +git -c init.defaultBranch=main init -q "$dir" +cd "$dir" +commit() { + git -c user.name=test -c user.email=test@example.com commit -q "$@" +} +commit --allow-empty -m "Start" +mkdir -p src +cat > src/records.js << 'EOF' +function parseRecord(line, options) { + const fields = line.split(options.separator || ","); + const record = {}; + for (let i = 0; i < fields.length; i++) { + const raw = fields[i].trim(); + if (raw === "") { + continue; + } + if (options.numbers && !isNaN(Number(raw))) { + record[options.columns[i]] = Number(raw); + } else if (raw === "true" || raw === "false") { + record[options.columns[i]] = raw === "true"; + } else { + record[options.columns[i]] = raw; + } + } + return record; +} +module.exports = { parseRecord }; +EOF +git add src +commit -m "Parse records" diff --git a/test/key-kinds.sh b/test/key-kinds.sh new file mode 100644 index 0000000..1d6cf54 --- /dev/null +++ b/test/key-kinds.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# check.sh gives JevGate api-key in the variable it reads for api-key-kind, +# and no other kind's key; it stops before checking when the installed +# JevGate predates gateway keys. A stand-in jevgate records the key +# variables it was given. +set -euo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir "$work/bin" +cat > "$work/bin/jevgate" << 'EOF' +#!/usr/bin/env bash +if [ "$1" = --version ]; then + echo "jevgate $STUB_VERSION" + exit 0 +fi +for name in TYPESAFE_API_KEY OPENROUTER_API_KEY AI_GATEWAY_API_KEY; do + if [ -n "${!name:-}" ]; then echo "$name=${!name}"; fi +done > "$STUB_SEEN" +EOF +chmod +x "$work/bin/jevgate" + +# The exit code of check.sh given api-key KEY of KIND with JevGate VERSION and +# the job's key variables VAR=VALUE..., then the key variables the check saw. +check() { + local kind=$1 version=$2 key=$3 code=0 seen + shift 3 + : > "$work/seen" + (cd "$work" && env -u TYPESAFE_API_KEY -u OPENROUTER_API_KEY -u AI_GATEWAY_API_KEY "$@" \ + PATH="$work/bin:$PATH" STUB_VERSION="$version" STUB_SEEN="$work/seen" \ + GITHUB_OUTPUT="$work/output" RUNNER_TEMP="$work" BASE= FORMAT=agent ARGS= \ + SARIF_FILE= API_KEY="$key" API_KEY_KIND="$kind" bash "$here/../check.sh" > /dev/null 2>&1) || code=$? + seen=$(tr '\n' ' ' < "$work/seen") + echo "exit $code${seen:+ ${seen% }}" +} + +failed=0 +# expect EXPECTED KIND VERSION KEY [VAR=VALUE...] +expect() { + local expected=$1 actual + shift + actual=$(check "$@") + if [ "$actual" != "$expected" ]; then + echo "::error::check $*: expected '$expected', got '$actual'" + failed=1 + fi +} + +expect "exit 0 TYPESAFE_API_KEY=k" typesafe 0.25.0 k +expect "exit 0 OPENROUTER_API_KEY=k" openrouter 0.26.0 k +expect "exit 0 AI_GATEWAY_API_KEY=k" vercel 0.26.0 k +expect "exit 0 AI_GATEWAY_API_KEY=k" vercel 1.0.0 k +expect "exit 2" openrouter 0.25.0 k +expect "exit 2" anthropic 0.26.0 k +# Keys the job holds for other services never reach JevGate. +expect "exit 0 TYPESAFE_API_KEY=k" typesafe 0.26.0 k OPENROUTER_API_KEY=job AI_GATEWAY_API_KEY=job +# With no api-key, the job's own key of that kind is used. +expect "exit 0 OPENROUTER_API_KEY=job" openrouter 0.26.0 "" OPENROUTER_API_KEY=job TYPESAFE_API_KEY=job +exit "$failed" diff --git a/test/render.test.cjs b/test/render.test.cjs new file mode 100644 index 0000000..465f23f --- /dev/null +++ b/test/render.test.cjs @@ -0,0 +1,305 @@ +// Tests for the comment's text: `node --test test/render.test.cjs`. The +// reports under test/reports are JevGate's own: a --base run on its +// repository, and 0.25.0 runs without a key and after an HTTP 402. +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { render, inline, code, marker, MAX_BODY_BYTES } = require('../render.cjs'); + +const COMMIT = '0123456789abcdef0123456789abcdef01234567'; +const RUN = { + key: 'review', + runId: '123', + runUrl: 'https://github.com/o/r/actions/runs/123', + serverUrl: 'https://github.com', + repository: 'o/r', + exitCode: 1, + commit: COMMIT, + prefix: '', + version: 'jevgate 0.25.0', + windows: false, +}; + +function report(name) { + return JSON.parse(fs.readFileSync(path.join(__dirname, 'reports', `${name}.json`), 'utf8')); +} + +function finding(strength, line, extra = {}) { + return { + rule: 'maintainability/shared-logic', + strength, + line, + message: `Finding at line ${line}`, + action: 'Share one implementation', + rank: 1, + locations: [], + baselined: false, + ...extra, + }; +} + +/** A complete report with these files: `{ path: [findings] }`. */ +function reportWith(files, extra = {}) { + return { + status: 'review', + complete: true, + errors: [], + gate: { passed: false, reasons: ['1 new review finding'] }, + api_requests: 3, + paid_input_tokens: 1000, + requested_model: 'jev-1.13.0', + stages: {}, + files: Object.entries(files).map(([file, findings]) => ({ path: file, status: 'review', findings })), + ...extra, + }; +} + +test('a real --base report lists every finding by level, then by file and line', () => { + const body = render(report('base-run'), RUN); + assert.ok(body.startsWith('\n### JevGate: gate failed\n')); + assert.match(body, /\nGate failed: 1 new review finding\(s\)\.\n/); + assert.match(body, /\n6 files · 55 API requests · 160,708 input tokens · ~\$0\.0067\n/); + assert.match(body, /\n3 files with uncertain units\.\n/); + const review = body.indexOf('#### Review (1)'); + const consider = body.indexOf('#### Consider (1)'); + const notes = body.indexOf('
Notes (11, optional)'); + assert.ok(review > 0 && consider > review && notes > consider, body); + assert.ok(body.includes( + `- [Line 213](https://github.com/o/r/blob/${COMMIT}/src/output.rs#L213-L217) \`maintainability/shared-logic\`: Lines 213 and 247 of \`emit_findings\``, + )); + // Notes: files by path, findings by line. + const noteText = body.slice(notes); + const files = [...noteText.matchAll(/\*\*\[`([^`]+)`\]/g)].map((m) => m[1]); + assert.deepEqual(files, ['src/options/commands.rs', 'src/options/mod.rs', 'src/output.rs']); + const lines = [...noteText.slice(noteText.indexOf('src/output.rs')).matchAll(/\[Line (\d+)\]/g)].map((m) => Number(m[1])); + assert.deepEqual(lines, [...lines].sort((a, b) => a - b)); + assert.ok(body.endsWith(`jevgate 0.25.0 · commit 0123456 · [workflow run](${RUN.runUrl}) · updated on each run\n`)); +}); + +test('a run without a key says loudly that the gate was not applied, and why', () => { + const body = render(report('no-key'), { ...RUN, exitCode: 2 }); + assert.ok(body.includes('### JevGate: run incomplete\n\n> [!CAUTION]\n> **JevGate could not finish this run (exit code 2).** The gate was not applied, and findings may be missing.\n')); + assert.ok(body.includes('> - No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH (2 files)\n')); + assert.ok(body.includes('\n2 files · 0 API requests · 0 input tokens\n')); + assert.ok(!body.includes('No new review or consider findings'), 'nothing was judged, so nothing is clean'); +}); + +test('an HTTP 402 lists each distinct reason with the files it stopped', () => { + const body = render(report('http-402'), { ...RUN, exitCode: 2 }); + assert.ok(body.includes('> - TypeSafe HTTP 402; request was not retried (1 file)\n')); + assert.ok(body.includes('> - TypeSafe request not sent after HTTP 402; restore account access and rerun the review (1 file)\n')); + assert.ok(body.includes('2 files · 1 API request · 0 input tokens')); +}); + +test('run errors come first, and reasons past ten are counted', () => { + const files = Array.from({ length: 14 }, (_, i) => ({ path: `f${i}.js`, status: 'error', error: `Reason ${i}`, findings: [] })); + const body = render(reportWith({}, { complete: false, gate: null, errors: ['Session API request budget exhausted'], files }), { ...RUN, exitCode: 2 }); + const reasons = body.split('\n').filter((line) => line.startsWith('> - ')); + assert.equal(reasons[0], '> - Session API request budget exhausted'); + assert.equal(reasons.length, 11); + assert.equal(reasons[10], '> - 5 more files stopped for other reasons.'); +}); + +test('a run that wrote no report still says it stopped', () => { + const body = render(null, { ...RUN, exitCode: 2 }); + assert.ok(body.includes(`> JevGate stopped before writing a report; the [job log](${RUN.runUrl}) says why.`)); + assert.ok(!body.includes('API request')); +}); + +test('from JevGate 0.26.0, findings that fail the gate and ones still being measured are marked', () => { + const files = { + 'a.js': [ + finding('review', 1, { gate: 'fails', rule: 'maintainability/function-simplification' }), + finding('review', 2, { gate: 'measuring' }), + finding('consider', 3, { gate: 'measuring' }), + finding('consider', 4, { gate: 'measuring' }), + finding('consider', 5, { gate: 'advisory' }), + finding('note', 6), + ], + }; + const body = render(reportWith(files), RUN); + assert.ok(body.includes('`maintainability/function-simplification` (fails the gate): Finding at line 1'), body); + assert.ok(body.includes('`maintainability/shared-logic`: Finding at line 2')); + assert.equal(body.split('(fails the gate)').length, 2, 'only the finding that fails is marked'); + assert.ok(body.includes( + '\n1 review finding and 2 consider findings are reported without failing the gate: their rules and levels are still being measured (`jevgate rules` shows which fail it by default).\n', + )); + const one = render(reportWith({ 'a.js': [finding('review', 2, { gate: 'measuring' })] }), RUN); + assert.ok(one.includes('\n1 review finding is reported without failing the gate')); + const older = render(report('base-run'), RUN); + assert.ok(!older.includes('fails the gate') && !older.includes('still being measured')); +}); + +test('from JevGate 0.28, each finding says how often findings like it were right', () => { + const body = render( + reportWith({ + 'src/lib.rs': [ + finding('review', 3, { precision: { right: 20, labeled: 23 } }), + finding('consider', 9, { precision: { right: 2, labeled: 5 } }), + finding('consider', 12), + ], + }), + RUN, + ); + assert.match(body, /Finding at line 3 Right 87% of the time \(23 labels\)\.
/); + assert.match(body, /Finding at line 9 Not yet measured\.
/); + assert.match(body, /Finding at line 12
/, 'a report before 0.28 shows none'); +}); + +test('from JevGate 0.30, a preview language\'s findings say so, in its own precision', () => { + const body = render( + reportWith({ + 'install.sh': [ + finding('review', 3, { gate: 'measuring', preview: 'Bash', precision: { right: 34, labeled: 50 } }), + finding('consider', 9, { gate: 'measuring', preview: 'Bash', precision: { right: 2, labeled: 5 } }), + ], + 'src/lib.rs': [finding('review', 4, { gate: 'measuring', precision: { right: 46, labeled: 85 } })], + 'laws.bend': [finding('consider', 7, { rule: 'tests/laws', precision: { right: 0, labeled: 0 } })], + }), + RUN, + ); + assert.match(body, /Finding at line 3 Right 68% of the time in Bash \(50 labels\)\.
/); + assert.match(body, /Finding at line 9 Not yet measured in Bash\.
/); + assert.match(body, /Finding at line 7 Not yet measured: labeled only on Bend 2 projects, which the maturity table leaves out\.
/); + assert.ok(body.includes( + '\n1 review finding is reported without failing the gate: their rules and levels are still being measured (`jevgate rules` shows which fail it by default). 1 review finding and 1 consider finding are reported without failing the gate: Bash is in preview, and by default JevGate\'s own rules never fail it there.\n', + ), body); +}); + +test('a passing gate with nothing new says so', () => { + const body = render(reportWith({ 'a.js': [finding('note', 3)] }, { status: 'note', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); + assert.ok(body.includes('### JevGate: gate passed\n')); + assert.ok(!body.includes('Gate failed')); + assert.ok(body.includes('No new review or consider findings.')); + assert.ok(body.includes('
Notes (1, optional)')); +}); + +test('no changed source is said plainly', () => { + const body = render(reportWith({}, { status: 'no-changed-source', gate: { passed: true, reasons: [] } }), { ...RUN, exitCode: 0 }); + assert.ok(body.includes('No supported file changed since the base revision.')); +}); + +test('accepted findings are counted, not listed', () => { + const files = { + 'a.js': [finding('review', 1, { baselined: true }), finding('consider', 2, { suppressed: 'generated' }), finding('consider', 9)], + }; + const body = render(reportWith(files), RUN); + assert.ok(!body.includes('Finding at line 1') && !body.includes('Finding at line 2')); + assert.ok(body.includes('Finding at line 9')); + assert.ok(body.includes('2 findings accepted by the baseline or an inline allow are not listed.')); + assert.ok(body.includes('No new review or consider findings.') === false); +}); + +test('more than ten considers are collapsed; reviews never are', () => { + const many = Array.from({ length: 11 }, (_, i) => finding('consider', i + 1)); + const reviews = Array.from({ length: 30 }, (_, i) => finding('review', i + 100)); + const body = render(reportWith({ 'a.js': [...many, ...reviews] }), RUN); + assert.ok(body.includes('#### Review (30)')); + assert.ok(body.includes('
Consider (11)')); + const ten = render(reportWith({ 'a.js': many.slice(1) }), RUN); + assert.ok(ten.includes('#### Consider (10)')); +}); + +test('cost is priced for jev-1.13.0 only, and unknown when answers carried no usage', () => { + const priced = render(reportWith({}, { paid_input_tokens: 1_000_000 }), RUN); + assert.ok(priced.includes('1,000,000 input tokens · ~$0.0420')); + const answered = { files: [{ path: 'a.js', status: 'clear', model: 'jev-1.13.0', findings: [] }] }; + assert.ok(render(reportWith({}, { requested_model: 'jev-latest', ...answered }), RUN).includes('~$0.0000')); + const other = { files: [{ path: 'a.js', status: 'clear', model: 'jev-2.0.0', findings: [] }] }; + assert.ok(render(reportWith({}, other), RUN).includes('1,000 input tokens · cost unknown')); + const noUsage = { paid_input_tokens: 0, stages: { functions: { successful_requests: 4 } } }; + assert.ok(render(reportWith({}, noUsage), RUN).includes('0 input tokens · cost unknown')); + const cached = render(reportWith({}, { paid_input_tokens: 0, api_requests: 0 }), RUN); + assert.ok(cached.includes('0 input tokens\n') && !cached.includes('$') && !cached.includes('cost unknown')); +}); + +test('a big run is cut under GitHub\'s limit, lowest ranked and least severe first', () => { + const long = 'A message long enough to matter, quoting `some_function_name` and 漢字 text. '.repeat(3); + const files = {}; + for (let f = 0; f < 60; f += 1) { + files[`src/module_${f}/file.rs`] = [ + finding('review', 10, { message: long, rank: 100 + f }), + ...Array.from({ length: 10 }, (_, i) => finding('consider', 20 + i, { message: long, rank: f + i })), + ...Array.from({ length: 40 }, (_, i) => finding('note', 100 + i, { message: long, rank: 1000 })), + ]; + } + const body = render(reportWith(files), RUN); + assert.ok(Buffer.byteLength(body, 'utf8') <= MAX_BODY_BYTES); + assert.ok(Buffer.byteLength(body, 'utf8') > MAX_BODY_BYTES - 1000, 'it uses the room it has'); + assert.ok(body.includes('#### Review (60)'), 'every review is kept'); + const considers = /Consider \((\d+) of 600\)/.exec(body); + assert.ok(considers, body.slice(0, 2000)); + const shown = Number(considers[1]); + assert.ok(!body.includes('Notes ('), 'notes go before any consider'); + assert.ok(body.includes(`**${(600 - shown + 2400).toLocaleString('en-US')} more findings** did not fit in this comment: ${600 - shown} consider findings, 2,400 notes.`)); + // No consider cut outranks one kept (a consider's rank is its module plus its index). + const kept = new Set( + [...body.matchAll(/\[Line (2\d)\]\(https:\/\/github\.com\/o\/r\/blob\/[0-9a-f]+\/src\/module_(\d+)/g)].map( + (m) => `${m[2]}:${Number(m[1]) - 20}`, + ), + ); + assert.equal(kept.size, shown); + const ranks = { kept: [], cut: [] }; + for (let f = 0; f < 60; f += 1) { + for (let i = 0; i < 10; i += 1) { + ranks[kept.has(`${f}:${i}`) ? 'kept' : 'cut'].push(f + i); + } + } + assert.ok(Math.min(...ranks.kept) >= Math.max(...ranks.cut), `${Math.min(...ranks.kept)} < ${Math.max(...ranks.cut)}`); +}); + +test('a run whose findings all fit is not cut', () => { + const files = { 'a.js': Array.from({ length: 50 }, (_, i) => finding('note', i + 1)) }; + const body = render(reportWith(files), RUN); + assert.ok(body.includes('Notes (50, optional)') && !body.includes('did not fit')); +}); + +test('Windows paths and a working directory below the root become repository links', () => { + const files = { 'src\\lib (old)\\a.js': [finding('review', 4, { locations: [{ path: 'src\\lib (old)\\a.js', start_line: 4, end_line: 9 }] })] }; + const body = render(reportWith(files), { ...RUN, windows: true, prefix: 'packages/app/' }); + assert.ok(body.includes(`**[\`packages/app/src/lib (old)/a.js\`](https://github.com/o/r/blob/${COMMIT}/packages/app/src/lib%20%28old%29/a.js)**`), body); + assert.ok(body.includes(`[Line 4](https://github.com/o/r/blob/${COMMIT}/packages/app/src/lib%20%28old%29/a.js#L4-L9)`)); + const linux = render(reportWith({ 'a\\b.js': [finding('review', 1)] }), RUN); + assert.ok(linux.includes('a%5Cb.js'), 'a backslash is part of a name on Linux'); +}); + +test('without a commit, locations are not links', () => { + const body = render(reportWith({ 'a.js': [finding('review', 4)] }), { ...RUN, commit: '' }); + assert.ok(body.includes('**`a.js`**\n- Line 4 `maintainability/shared-logic`:')); + assert.ok(!body.includes('/blob/') && !body.includes('commit ')); +}); + +test('text from the change cannot add HTML, links, markers or mentions', () => { + assert.equal(inline('Value `x` in y'), 'Value `x` in <b>y</b>'); + assert.equal(inline('ping @octocat, not `@octocat`'), 'ping @\u200boctocat, not `@octocat`'); + assert.equal(inline(''), '<!-- jevgate-action comment key=review run=999 -->'); + assert.equal(inline('[approve](https://x.test) ![i](https://x.test/i.png)'), '\\[approve\\](https://x.test) !\\[i\\](https://x.test/i.png)'); + assert.equal(inline('line one\n# Heading\r\n> quote'), 'line one # Heading > quote'); + assert.equal(inline('a < b'), 'a &lt; b'); + // Only a closed run of backticks is code, as CommonMark reads it; an + // unclosed run is escaped. + assert.equal(inline('``a` '), '\\`\\`a\\` <i>'); + assert.equal(inline('`` a`b `` '), '`` a`b `` <i>'); + // A backslash cannot turn a backtick into text around raw HTML. + assert.equal(inline('\\``'), '\\\\``'); +}); + +test('an unclosed backtick in a message cannot pair with the next step\'s code', () => { + const bait = finding('review', 3, { message: 'see `', action: '`` then `' }); + const body = render(reportWith({ 'a.js': [bait] }), RUN); + assert.ok(body.includes(': see \\`
→ `` then \\`\n'), body); +}); + +test('paths and rules are code spans whatever backticks they hold', () => { + assert.equal(code('src/a.js'), '`src/a.js`'); + assert.equal(code('a`b'), '``a`b``'); + assert.equal(code('`a'), '`` `a ``'); +}); + +test('the comment marker cannot close its HTML comment', () => { + assert.equal(marker('review services/api'), 'b').includes('-->')); +}); diff --git a/test/reports/base-run.json b/test/reports/base-run.json new file mode 100644 index 0000000..859f024 --- /dev/null +++ b/test/reports/base-run.json @@ -0,0 +1,1510 @@ +{ + "quick": false, + "base_revision": "811595e449683e3ef96375d48cb97885a80c9d6f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/jevgate/jevgate", + "generation": 21, + "watcher_pid": null, + "errors": [], + "generated_at": 1790372499, + "status": "review", + "complete": true, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "gate": { + "passed": false, + "reasons": [ + "1 new review finding(s)" + ], + "new_findings": 2, + "baselined_findings": 0 + }, + "api_requests": 55, + "concurrency": 6, + "paid_input_tokens": 160708, + "paid_output_tokens": 7982, + "stages": { + "comments": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 2772, + "service_ms": 5972, + "queue_wait_ms": 11328, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 9, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 11, + "cached_judgments": 11, + "evaluated_judgments": 9, + "input_tokens": 41436, + "output_tokens": 2182, + "evidence_bytes": 25077 + }, + "constants": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 4, + "cached_judgments": 4, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 266, + "service_ms": 266, + "queue_wait_ms": 2550, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 1, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 1, + "input_tokens": 890, + "output_tokens": 52, + "evidence_bytes": 740 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3087, + "service_ms": 3167, + "queue_wait_ms": 6869, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 2, + "cached_judgments": 2, + "evaluated_judgments": 5, + "input_tokens": 10591, + "output_tokens": 380, + "evidence_bytes": 20754 + }, + "locate": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 259, + "service_ms": 259, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 1, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 1, + "input_tokens": 774, + "output_tokens": 73, + "evidence_bytes": 1028 + }, + "outline": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3103, + "service_ms": 1789, + "queue_wait_ms": 5920, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 5, + "input_tokens": 9441, + "output_tokens": 301, + "evidence_bytes": 16645 + }, + "recheck": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 880, + "service_ms": 4496, + "queue_wait_ms": 1643, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 10, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 4, + "cached_judgments": 4, + "evaluated_judgments": 10, + "input_tokens": 23892, + "output_tokens": 516, + "evidence_bytes": 61331 + }, + "security": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3555, + "service_ms": 3529, + "queue_wait_ms": 9453, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 6, + "cached_judgments": 6, + "evaluated_judgments": 5, + "input_tokens": 24113, + "output_tokens": 2154, + "evidence_bytes": 16718 + }, + "tests": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 3676, + "service_ms": 3569, + "queue_wait_ms": 24992, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 10, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 1, + "cached_judgments": 1, + "evaluated_judgments": 10, + "input_tokens": 13403, + "output_tokens": 800, + "evidence_bytes": 8626 + }, + "trace": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 1533, + "service_ms": 2614, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 5, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 2, + "cached_judgments": 2, + "evaluated_judgments": 5, + "input_tokens": 21859, + "output_tokens": 740, + "evidence_bytes": 58200 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 2270, + "service_ms": 2546, + "queue_wait_ms": 5162, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 4, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 3, + "cached_judgments": 3, + "evaluated_judgments": 4, + "input_tokens": 14309, + "output_tokens": 784, + "evidence_bytes": 15850 + } + }, + "settled": true, + "files": [ + { + "path": "src/github.rs", + "role": "source", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [], + "dimensions": { + "comments": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 comments judged: 7 clear.", + "rule_version": "2", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "3 functions judged: 3 clear. 4 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 3, + "review": 0, + "consider": 0, + "note": 0, + "clear": 3, + "uncertain": 0, + "needs_context": 0, + "too_small": 4, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 value units judged: 7 clear.", + "rule_version": "4", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "uncertain", + "concern_probability": 0.9, + "decision_basis": "7 security units judged: 6 clear, 1 uncertain.", + "rule_version": "6", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 6, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 security units judged: 7 clear.", + "rule_version": "5", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "2 tests judged: 2 clear.", + "rule_version": "5", + "units": { + "judged": 2, + "review": 0, + "consider": 0, + "note": 0, + "clear": 2, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "7 security units judged: 7 clear.", + "rule_version": "4", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 7, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 32716, + "output_tokens": 2213 + }, + { + "path": "src/options/commands.rs", + "role": "source", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "documentation/comments", + "strength": "note", + "line": 1, + "message": "This file's top-level code has a comment to clean up: at line 1 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": null, + "rule_version": "2", + "concern_probability": 0.98, + "locations": [ + { + "path": "src/options/commands.rs", + "start_line": 1, + "end_line": 1, + "symbol": "top-level code" + } + ], + "quote": "//! The subcommands, the baseline actions and their help text.\n", + "fingerprint": "5fb29bb3e2212df289b9748c9fd60c1599b13459df84a0400841d985e5a18823", + "rank": 0.6792842369487464, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.98, + "decision_basis": "26 comments judged: 1 note, 25 clear.", + "rule_version": "2", + "units": { + "judged": 26, + "review": 0, + "consider": 0, + "note": 1, + "clear": 25, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No functions to judge.", + "rule_version": "14", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "uncertain", + "concern_probability": 0.37, + "decision_basis": "1 value unit judged: 1 uncertain.", + "rule_version": "4", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "6", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "5", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No security units to judge.", + "rule_version": "4", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 31057, + "output_tokens": 1318 + }, + { + "path": "src/options/mod.rs", + "role": "source", + "status": "note", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "documentation/comments", + "strength": "note", + "line": 1, + "message": "This file's top-level code has a comment to clean up: at lines 1–2 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": null, + "rule_version": "2", + "concern_probability": 0.93, + "locations": [ + { + "path": "src/options/mod.rs", + "start_line": 1, + "end_line": 2, + "symbol": "top-level code" + } + ], + "quote": "//! The `check` arguments, output formats and gate levels; the subcommands and\n//! their help text are in `commands`.\n", + "fingerprint": "58c86ae408c99452fa1af1684beb3d10df53e78c32cd7ba8bb92f1a13f1ab928", + "rank": 1.0217094284613422, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 276, + "message": "`CheckArgs::enabled` has a comment to clean up: at line 276 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "CheckArgs::enabled", + "rule_version": "2", + "concern_probability": 0.8900000000000001, + "locations": [ + { + "path": "src/options/mod.rs", + "start_line": 276, + "end_line": 276, + "symbol": "CheckArgs::enabled" + } + ], + "quote": "/// Whether a rule is selected, by key or ID.\n", + "fingerprint": "8cbf468d2d64d25f58ccde803b1924b6d4648fe6ccd968aa1e37d19e23a2f62d", + "rank": 0.6169009906983514, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.93, + "decision_basis": "60 comments judged: 2 note, 58 clear.", + "rule_version": "2", + "units": { + "judged": 60, + "review": 0, + "consider": 0, + "note": 2, + "clear": 58, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "5 functions judged: 5 clear. 12 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 5, + "review": 0, + "consider": 0, + "note": 0, + "clear": 5, + "uncertain": 0, + "needs_context": 0, + "too_small": 12, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "5 value units judged: 5 clear.", + "rule_version": "4", + "units": { + "judged": 5, + "review": 0, + "consider": 0, + "note": 0, + "clear": 5, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "6", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "5", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "16 security units judged: 16 clear.", + "rule_version": "4", + "units": { + "judged": 16, + "review": 0, + "consider": 0, + "note": 0, + "clear": 16, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 82669, + "output_tokens": 4651 + }, + { + "path": "src/output.rs", + "role": "source", + "status": "review", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [ + { + "rule": "maintainability/shared-logic", + "strength": "review", + "line": 213, + "message": "Lines 213 and 247 of `emit_findings` (src/output.rs) perform the same steps for the same purpose (0.86). Differences: `Review ({}):`→`Notes ({}, optional):`, `review`→`notes`, `1;31`→`1`.", + "action": "Move the shared steps into one implementation", + "symbol": "Lines 213 and 247 of `emit_findings` (src/output.rs)", + "rule_version": "19", + "concern_probability": 0.86, + "locations": [ + { + "path": "src/output.rs", + "start_line": 213, + "end_line": 217, + "symbol": "emit_findings" + }, + { + "path": "src/output.rs", + "start_line": 247, + "end_line": 251, + "symbol": "emit_findings" + } + ], + "quote": "let heading = format!(\"Review ({}):\", review.len());\n writeln!(out, \"\\n{}\", style.paint(\"1;31\", &heading))?;\n for (path, finding) in &review {\n emit_finding(out, path, finding, style)?;\n }", + "fingerprint": "c2411e834cced1c5f56c8a68b9a04e95ab9ba7bb8781e3879793745a50f202ad", + "rank": 2.062189934606599, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "consider", + "line": 331, + "message": "`emit_finding` likely uses a value whose meaning a reader must guess (0.87). The value is \"36\".", + "action": "Give the value a descriptive constant name", + "symbol": "emit_finding", + "rule_version": "4", + "concern_probability": 0.8700000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 331, + "end_line": 351, + "symbol": "emit_finding" + } + ], + "fingerprint": "496b58069e9ed18abb10136b9f3b6e4ebdd5b892b41c5c5a2131d6f8c407ced0", + "rank": 2.6892069344217355, + "baselined": false + }, + { + "rule": "maintainability/function-simplification", + "strength": "note", + "line": 289, + "message": "`emit_context_load` reads well as it is; one block could be named as a helper.", + "action": "Optional: extract that block if it grows", + "symbol": "emit_context_load", + "rule_version": "14", + "concern_probability": 0.85, + "locations": [ + { + "path": "src/output.rs", + "start_line": 289, + "end_line": 327, + "symbol": "emit_context_load" + } + ], + "fingerprint": "362c563e25a7e440960c9bd2d433d974d9fd9d1a201479921f4ae916948b39a7", + "rank": 3.135547535996846, + "baselined": false + }, + { + "rule": "maintainability/function-simplification", + "strength": "note", + "line": 256, + "message": "`emit_summary` reads well as it is; one block could be named as a helper.", + "action": "Optional: extract that block if it grows", + "symbol": "emit_summary", + "rule_version": "14", + "concern_probability": 0.88, + "locations": [ + { + "path": "src/output.rs", + "start_line": 256, + "end_line": 286, + "symbol": "emit_summary" + } + ], + "fingerprint": "84dce89ad1b06817330b51a91db2a8cbad9b360c057348791ff356f5e5c42448", + "rank": 3.0498475944637593, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "note", + "line": 353, + "message": "`emit_file` may use a value whose meaning a reader must guess; the answer was split.", + "action": "Optional: name or configure the value if it changes", + "symbol": "emit_file", + "rule_version": "4", + "concern_probability": 0.7, + "locations": [ + { + "path": "src/output.rs", + "start_line": 353, + "end_line": 386, + "symbol": "emit_file" + } + ], + "fingerprint": "744eb1bc38d3e8c0f54ae00890a7d4e77466683e5eb1252cb1e98bcec75c3e64", + "rank": 2.4887436430425893, + "baselined": false + }, + { + "rule": "maintainability/hardcoded-values", + "strength": "note", + "line": 26, + "message": "`usd` has a value that could be named, though its context explains it.", + "action": "Optional: name or configure the value if it changes", + "symbol": "usd", + "rule_version": "4", + "concern_probability": 0.8400000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 26, + "end_line": 28, + "symbol": "usd" + } + ], + "fingerprint": "b2f2cf1e5db49630ef53b471139ab8e8be7e8fa3f02b5017f99c4980c2c3beba", + "rank": 1.1644872633407082, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 137, + "message": "`headline` has a comment to clean up: at lines 137–138 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "headline", + "rule_version": "2", + "concern_probability": 0.83, + "locations": [ + { + "path": "src/output.rs", + "start_line": 137, + "end_line": 138, + "symbol": "headline" + } + ], + "quote": "/// Status, gate, scope and cost on one line; for a dry run, the planned\n/// requests and the cost of those the cache does not answer.\n", + "fingerprint": "d9fd46abc3e8075bd67e31f9cd50c6d17995271f4ce8456431ef001d8acc63b5", + "rank": 0.9118481995945311, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 288, + "message": "`emit_context_load` has a comment to clean up: at line 288 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "emit_context_load", + "rule_version": "2", + "concern_probability": 0.95, + "locations": [ + { + "path": "src/output.rs", + "start_line": 288, + "end_line": 288, + "symbol": "emit_context_load" + } + ], + "quote": "/// Estimated tokens each harness loads at session start, then loading facts.\n", + "fingerprint": "0c489984d06d9b262fe2219500c4785e94813d5e7d2a16fd1f433dab240e7f3a", + "rank": 0.658489821531948, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 255, + "message": "`emit_summary` has a comment to clean up: at line 255 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "emit_summary", + "rule_version": "2", + "concern_probability": 0.8600000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 255, + "end_line": 255, + "symbol": "emit_summary" + } + ], + "quote": "/// Counts of undecided, unsent and failed files, and skip reasons.\n", + "fingerprint": "b0d9560615fa227f636fd476f31ea6a9f55184ac8a9fa2f5b96a2a179248b91f", + "rank": 0.596106575281553, + "baselined": false + }, + { + "rule": "documentation/comments", + "strength": "note", + "line": 181, + "message": "`ranked` has a comment to clean up: at line 181 it repeats the code.", + "action": "Optional: delete, shorten or rewrite it", + "symbol": "ranked", + "rule_version": "2", + "concern_probability": 0.8200000000000001, + "locations": [ + { + "path": "src/output.rs", + "start_line": 181, + "end_line": 181, + "symbol": "ranked" + } + ], + "quote": "/// Every finding with its file's path, highest rank first.\n", + "fingerprint": "8c4ad9aa47c3a7691f73eb65d7626a0ed6c93e6d3c97594937d2567542df2b11", + "rank": 0.5683806880591552, + "baselined": false + } + ], + "dimensions": { + "comments": { + "status": "note", + "concern_probability": 0.95, + "decision_basis": "15 comments judged: 4 note, 11 clear.", + "rule_version": "2", + "units": { + "judged": 15, + "review": 0, + "consider": 0, + "note": 4, + "clear": 11, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "note", + "concern_probability": 0.88, + "decision_basis": "14 functions judged: 2 note, 12 clear. 4 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 14, + "review": 0, + "consider": 0, + "note": 2, + "clear": 12, + "uncertain": 0, + "needs_context": 0, + "too_small": 4, + "omitted": 0, + "covered": 0 + } + }, + "hardcoded_values": { + "status": "consider", + "concern_probability": 0.8700000000000001, + "decision_basis": "13 value units judged: 1 consider, 2 note, 10 clear.", + "rule_version": "4", + "units": { + "judged": 13, + "review": 0, + "consider": 1, + "note": 2, + "clear": 10, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "injection": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "6", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "sensitive_data": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "5", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "review", + "concern_probability": 0.86, + "decision_basis": "1 candidate pair judged: 1 review.", + "rule_version": "19", + "units": { + "judged": 1, + "review": 1, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "2 tests judged: 2 clear.", + "rule_version": "5", + "units": { + "judged": 2, + "review": 0, + "consider": 0, + "note": 0, + "clear": 2, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "unsafe_settings": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "17 security units judged: 17 clear.", + "rule_version": "4", + "units": { + "judged": 17, + "review": 0, + "consider": 0, + "note": 0, + "clear": 17, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 66999, + "output_tokens": 3877 + }, + { + "path": "src/tests/mod.rs", + "role": "test", + "status": "uncertain", + "cached": false, + "model": "jev-1.13.0", + "error": null, + "findings": [], + "dimensions": { + "file_organization": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "1 outline judged: 1 clear.", + "rule_version": "18", + "units": { + "judged": 1, + "review": 0, + "consider": 0, + "note": 0, + "clear": 1, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "function_simplification": { + "status": "clear", + "concern_probability": 0.0, + "decision_basis": "8 functions judged: 8 clear. 6 functions too small to judge.", + "rule_version": "14", + "units": { + "judged": 8, + "review": 0, + "consider": 0, + "note": 0, + "clear": 8, + "uncertain": 0, + "needs_context": 0, + "too_small": 6, + "omitted": 0, + "covered": 0 + } + }, + "shared_logic": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No candidate pairs to judge.", + "rule_version": "19", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_redundancy": { + "status": "not-applicable", + "concern_probability": 0.0, + "decision_basis": "No test pairs to judge.", + "rule_version": "3", + "units": { + "judged": 0, + "review": 0, + "consider": 0, + "note": 0, + "clear": 0, + "uncertain": 0, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + }, + "test_value": { + "status": "uncertain", + "concern_probability": 0.23, + "decision_basis": "7 tests judged: 6 clear, 1 uncertain.", + "rule_version": "5", + "units": { + "judged": 7, + "review": 0, + "consider": 0, + "note": 0, + "clear": 6, + "uncertain": 1, + "needs_context": 0, + "too_small": 0, + "omitted": 0, + "covered": 0 + } + } + }, + "input_tokens": 25324, + "output_tokens": 884 + }, + { + "path": "README.md", + "role": "docs", + "status": "skipped", + "cached": false, + "model": null, + "error": "Outside upload_allow/upload_deny; not judged.", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [ + { + "rule": "documentation/comments", + "path": "src/options/commands.rs", + "previous_path": null, + "previous_generation": 20, + "state": "persistent", + "reason": "The same finding remains", + "fingerprint": "5fb29bb3e2212df289b9748c9fd60c1599b13459df84a0400841d985e5a18823" + }, + { + "rule": "documentation/comments", + "path": "src/options/mod.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "58c86ae408c99452fa1af1684beb3d10df53e78c32cd7ba8bb92f1a13f1ab928" + }, + { + "rule": "documentation/comments", + "path": "src/options/mod.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "8cbf468d2d64d25f58ccde803b1924b6d4648fe6ccd968aa1e37d19e23a2f62d" + }, + { + "rule": "maintainability/shared-logic", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "c2411e834cced1c5f56c8a68b9a04e95ab9ba7bb8781e3879793745a50f202ad" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "496b58069e9ed18abb10136b9f3b6e4ebdd5b892b41c5c5a2131d6f8c407ced0" + }, + { + "rule": "maintainability/function-simplification", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "362c563e25a7e440960c9bd2d433d974d9fd9d1a201479921f4ae916948b39a7" + }, + { + "rule": "maintainability/function-simplification", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "84dce89ad1b06817330b51a91db2a8cbad9b360c057348791ff356f5e5c42448" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "744eb1bc38d3e8c0f54ae00890a7d4e77466683e5eb1252cb1e98bcec75c3e64" + }, + { + "rule": "maintainability/hardcoded-values", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "b2f2cf1e5db49630ef53b471139ab8e8be7e8fa3f02b5017f99c4980c2c3beba" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "d9fd46abc3e8075bd67e31f9cd50c6d17995271f4ce8456431ef001d8acc63b5" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "0c489984d06d9b262fe2219500c4785e94813d5e7d2a16fd1f433dab240e7f3a" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "b0d9560615fa227f636fd476f31ea6a9f55184ac8a9fa2f5b96a2a179248b91f" + }, + { + "rule": "documentation/comments", + "path": "src/output.rs", + "previous_path": null, + "previous_generation": 20, + "state": "introduced", + "reason": "New since the previous snapshot", + "fingerprint": "8c4ad9aa47c3a7691f73eb65d7626a0ed6c93e6d3c97594937d2567542df2b11" + }, + { + "rule": "documentation/comments", + "path": "src/manual.rs", + "previous_path": null, + "previous_generation": 20, + "state": "non-comparable", + "reason": "The file was not judged in this snapshot, or rubric or model changed", + "fingerprint": "a77714c33851e41910555dfcaf22fb82428e5636fd58aade6a9a499a2f03ed6b" + }, + { + "rule": "documentation/comments", + "path": "src/command.rs", + "previous_path": null, + "previous_generation": 20, + "state": "non-comparable", + "reason": "The file was not judged in this snapshot, or rubric or model changed", + "fingerprint": "deaef5f7bc17022e28d45da241f86648ccbdb275abb3e39808372bbd9f1e9e66" + } + ], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "security/injection", + "security/sensitive-data", + "security/unsafe-settings", + "security/access-control", + "security/workflows", + "tests/value", + "tests/redundancy", + "documentation/agent-context", + "documentation/large-docs", + "documentation/staleness", + "documentation/duplication", + "documentation/comments" + ] +} diff --git a/test/reports/http-402.json b/test/reports/http-402.json new file mode 100644 index 0000000..1aaab17 --- /dev/null +++ b/test/reports/http-402.json @@ -0,0 +1,133 @@ +{ + "quick": false, + "base_revision": "63418fdae32078965d851308771a48d8f897e28f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/app/app", + "generation": 1, + "watcher_pid": null, + "errors": [], + "generated_at": 1790569169, + "status": "incomplete", + "complete": false, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_bend_file_lines": 300.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "api_requests": 1, + "concurrency": 6, + "paid_input_tokens": 0, + "paid_output_tokens": 0, + "stages": { + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + } + }, + "settled": true, + "files": [ + { + "path": "lib/parse.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "TypeSafe HTTP 402; request was not retried", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + }, + { + "path": "lib/rows.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "TypeSafe request not sent after HTTP 402; restore account access and rerun the review", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "tests/value", + "tests/redundancy", + "tests/laws" + ] +} diff --git a/test/reports/no-key.json b/test/reports/no-key.json new file mode 100644 index 0000000..eef9881 --- /dev/null +++ b/test/reports/no-key.json @@ -0,0 +1,133 @@ +{ + "quick": false, + "base_revision": "63418fdae32078965d851308771a48d8f897e28f", + "deleted_files": [], + "schema_version": 2, + "command": "check", + "rubric_version": "jevgate-units-v1", + "root": "/home/runner/work/app/app", + "generation": 1, + "watcher_pid": null, + "errors": [], + "generated_at": 1790569169, + "status": "incomplete", + "complete": false, + "judgments_complete": false, + "acceptance_evaluated": false, + "dry_run": false, + "requested_model": "jev-1.13.0", + "decision_policy": { + "clear_probability": 0.8, + "consider_leading_probability": 0.5, + "consider_probability": 0.8, + "deep_nesting": 4.0, + "location_probability": 0.65, + "long_branch_chain": 4.0, + "min_bend_file_lines": 300.0, + "min_body_lines": 5.0, + "min_clone_bytes": 120.0, + "min_clone_statements": 3.0, + "min_file_lines": 100.0, + "review_probability": 0.8 + }, + "fail_on": [ + "review" + ], + "api_requests": 0, + "concurrency": 6, + "paid_input_tokens": 0, + "paid_output_tokens": 0, + "stages": { + "duplicate-pair": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "functions": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + }, + "values": { + "planned_requests": 0, + "planned_evidence_bytes": 0, + "elapsed_ms": 0, + "service_ms": 0, + "queue_wait_ms": 0, + "planned_tokens": 0, + "planned_cached": 0, + "successful_requests": 0, + "failed_attempts": 0, + "retries": 0, + "cache_hits": 0, + "cached_judgments": 0, + "evaluated_judgments": 0, + "input_tokens": 0, + "output_tokens": 0, + "evidence_bytes": 0 + } + }, + "settled": true, + "files": [ + { + "path": "lib/parse.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + }, + { + "path": "lib/rows.js", + "role": "source", + "status": "error", + "cached": false, + "model": null, + "error": "No API key configured. Run jevgate auth login, set TYPESAFE_API_KEY, or provide --env-file PATH", + "findings": [], + "dimensions": {}, + "input_tokens": 0, + "output_tokens": 0 + } + ], + "changes": [], + "rules": [ + "maintainability/file-organization", + "maintainability/function-simplification", + "maintainability/shared-logic", + "maintainability/hardcoded-values", + "tests/value", + "tests/redundancy", + "tests/laws" + ] +}