From 0871ba2beae7532d8410be49b9081c551077c673 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 20:55:08 -0300 Subject: [PATCH 01/12] Don't slice inside a multi-byte character in docs roles or test line ranges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A colon after non-ASCII text ("已移除:`x`") panicked in the docs role parser, and a Python test ending in a multi-byte character panicked when counting its last line. Both stopped JevGate on herdr, hermes-agent and OmniRoute. --- src/analysis/test_map.rs | 6 ++++++ src/docs/references.rs | 14 ++++++++++++-- src/test_locations/mod.rs | 6 +++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/analysis/test_map.rs b/src/analysis/test_map.rs index 1ab031e..60f2c8a 100644 --- a/src/analysis/test_map.rs +++ b/src/analysis/test_map.rs @@ -788,6 +788,12 @@ mod tests { .collect() } + #[test] + fn a_test_ending_in_a_wide_character_is_located() { + let source = "import pytest\n\n\ndef test_price():\n café = 1\n assert café\n"; + assert_eq!(located("tests/test_price.py", source), [(4, 6)]); + } + #[test] fn phpunit_methods_and_pest_calls_are_test_cases() { let phpunit = "assertSame(3, total([1, 2]));\n }\n\n /** @test */\n public function it_is_empty(): void\n {\n $this->assertSame(0, (new Summer())->total([]));\n }\n\n #[Test]\n public function keeps_order(): void {}\n}\n\nclass Helper { public function testLike() {} }\n"; diff --git a/src/docs/references.rs b/src/docs/references.rs index 3789468..39752cb 100644 --- a/src/docs/references.rs +++ b/src/docs/references.rs @@ -380,8 +380,9 @@ fn role(before: &str) -> Option<&str> { } let inner = before.strip_suffix(':')?; let start = inner - .rfind(|c: char| !(c.is_ascii_alphanumeric() || c == ':' || c == '-' || c == '_')) - .map_or(0, |i| i + 1); + .char_indices() + .rfind(|&(_, c)| !(c.is_ascii_alphanumeric() || c == ':' || c == '-' || c == '_')) + .map_or(0, |(i, c)| i + c.len_utf8()); let name = inner[start..].strip_prefix(':')?; (!name.is_empty()).then(|| name.rsplit(':').next().unwrap_or(name)) } @@ -628,6 +629,15 @@ mod tests { assert_eq!(names, ["conf/app.json", "data/seed.json"]); } + #[test] + fn a_colon_after_wide_text_is_not_a_role() { + let names: Vec = found("外部 API 已移除:`conf/app.json`") + .into_iter() + .map(|(n, _)| n) + .collect(); + assert_eq!(names, ["conf/app.json"]); + } + #[test] fn a_file_the_section_writes_out_is_the_readers() { let text = "Create a route handler, `app/api/chat.ts`:\n\n```ts filename=\"app/api/chat.ts\"\nexport {}\n```\n\nHere is the example :file:`database.py` module::\n```\n engine = None\n```\n.. code-block:: python\n\nThen update both `kasada-server.ts` and `kasada-client.ts`, like this:\n\n```\nhttps://example.com/api\n```\n"; diff --git a/src/test_locations/mod.rs b/src/test_locations/mod.rs index 64d7e6e..a721d8f 100644 --- a/src/test_locations/mod.rs +++ b/src/test_locations/mod.rs @@ -149,9 +149,9 @@ fn line_range(source: &str, start: usize, end: usize) -> SourceRange { .count() + 1; let end_at = end.saturating_sub(1).max(start); - let end_line = source[..end_at.min(source.len())] - .bytes() - .filter(|byte| *byte == b'\n') + let end_line = source.as_bytes()[..end_at.min(source.len())] + .iter() + .filter(|byte| **byte == b'\n') .count() + 1; SourceRange { From 6e42bdbe34135e7f1d5cee698e5322c12b1e481f Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:07:59 -0300 Subject: [PATCH 02/12] Read a path with an anchor in a code span as its file `docs/en/env/01-variables.md#idempotency` in freellmapi's docs was reported as a missing path although the file and its heading exist. --- src/docs/references.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/docs/references.rs b/src/docs/references.rs index 39752cb..91e0d9d 100644 --- a/src/docs/references.rs +++ b/src/docs/references.rs @@ -259,6 +259,12 @@ fn path_like(token: &str, top: &BTreeSet<&str>) -> Option { { t = path.to_string(); } + // `guide.md#setup` names `guide.md`. + if let Some((path, _)) = t.split_once('#') + && !path.is_empty() + { + t = path.to_string(); + } let excluded = ["http", "mailto:", "#", "$", "-", "@", "~", "/"] .iter() .any(|p| t.starts_with(p)) @@ -629,6 +635,15 @@ mod tests { assert_eq!(names, ["conf/app.json", "data/seed.json"]); } + #[test] + fn a_code_span_with_an_anchor_names_its_file() { + let names: Vec = found("See `docs/guide.md#setup` and `docs/gone.md#usage`.") + .into_iter() + .map(|(n, _)| n) + .collect(); + assert_eq!(names, ["docs/gone.md"]); + } + #[test] fn a_colon_after_wide_text_is_not_a_role() { let names: Vec = found("外部 API 已移除:`conf/app.json`") From 9af3f537ce7eda67976cb79ba295641306d19857 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:12:53 -0300 Subject: [PATCH 03/12] Leave out the docs of one release, such as docs/versions/0.7.5 herdr keeps a copy of its website docs per release under docs/versions/; 137 of its 147 documentation considers named a section of such a copy. --- src/docs/discover.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/docs/discover.rs b/src/docs/discover.rs index 30b5a6c..e5ef292 100644 --- a/src/docs/discover.rs +++ b/src/docs/discover.rs @@ -110,8 +110,15 @@ fn project_doc(path: &Path) -> bool { RECORD_STEMS.contains(&d.as_str()) || matches!( d.as_str(), - "fixtures" | "__fixtures__" | "testdata" | "__snapshots__" | "archive" | "_build" + "fixtures" + | "__fixtures__" + | "testdata" + | "__snapshots__" + | "archive" + | "_build" + | "versioned_docs" ) + || release_dir(d) }); let documentation = dirs.is_empty() || stem.starts_with("readme") @@ -125,6 +132,16 @@ fn project_doc(path: &Path) -> bool { && !RECORD_STEMS.contains(&stem.as_str()) } +/// A directory holding the docs of one release, such as `docs/versions/0.7.5` +/// or `v1.2`: a frozen copy of the current docs, not a second source. +fn release_dir(dir: &str) -> bool { + let number = dir.strip_prefix('v').unwrap_or(dir); + number.contains('.') + && number + .split('.') + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_digit())) +} + /// Claude Code skills, commands and subagent definitions: Markdown under /// `.claude/skills`, `.claude/commands` or `.claude/agents`. A session loads /// only their descriptions and reads the rest when one is used, so they are @@ -331,6 +348,11 @@ mod tests { ("tests/fixtures/readme.md", false), ("docs/changelog/v1.md", false), ("docs/archive/plan.md", false), + ("docs/versions/0.7.5/website/agents.mdx", false), + ("docs/v1.2/guide.md", false), + ("website/versioned_docs/version-2/intro.md", false), + ("docs/v2/guide.md", true), + ("docs/next/README.md", true), ] { assert_eq!(project_doc(Path::new(path)), expected, "{path}"); } From 3e8e058dee1aadf3823972c3e49559dc84838a9f Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:18:15 -0300 Subject: [PATCH 04/12] Changelog: fixes from running JevGate on widely used projects --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0d63b..e3f302c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Notable changes to JevGate. Versions follow [Semantic Versioning](https://semver ## [Unreleased] +Fixes from running JevGate on nine widely used projects under daily development (rtk, headroom, paperclip, hermes-agent, cc-switch, freellmapi, herdr, multica, OmniRoute). + +- Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. +- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. +- Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. No corpus finding changes. + ## [0.24.1] - 2026-09-27 - File organization: in Rust, a function another file passes by path, such as `compose::unconfirmed_units` in `follow_ups(plan, files, compose::unconfirmed_units)`, counts as used by that file, so a file's outline names the files that use each member (`used_by`). JevGate's own `compose.rs`, before it was split, listed `follow_ups.rs` as the user of 3 of its 8 follow-up selectors, and the file stayed clear; with all 8 listed, its outline is a consider (0.80). Only outlines whose members are passed by path change: on the 26 Rust projects of the corpus, 37 outline requests were asked again (under $0.01), one wrong review (a SpacetimeDB conversion module, one job laid out in sections) is a consider, one right consider (zoxide's `util.rs`, a grab bag of helpers) is a review, and two units are undecided; nothing else changed. Other languages are unchanged. From f2ab926b936caa05e1e4b8216b1f1f51ac6645c7 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 21:35:23 -0300 Subject: [PATCH 05/12] Don't report a translation as repeating its original Two documents whose paths name different locales (docs/en and docs/zh-cn, README.md and README_zh.md) or whose prose is in different scripts are translations: their sections are asked only whether they disagree, as when the translation question itself says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition; no corpus finding pairs two languages. --- src/docs/overlap.rs | 119 ++++++++++++++++++++++++++++++++++++- src/units/compose/mod.rs | 11 +++- src/units/drift.rs | 4 ++ src/units/mod.rs | 3 + src/units/outcome/mod.rs | 11 +++- src/units/outcome/pairs.rs | 17 ++++-- 6 files changed, 158 insertions(+), 7 deletions(-) diff --git a/src/docs/overlap.rs b/src/docs/overlap.rs index e339b6c..973898a 100644 --- a/src/docs/overlap.rs +++ b/src/docs/overlap.rs @@ -4,7 +4,10 @@ //! A section that pairs with two or more others heads a family: its members //! are asked against it alone, not against each other, so a section repeated //! in seven quickstarts is six questions, not twenty-one. -use std::collections::{BTreeMap, BTreeSet}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, +}; /// A pair is a candidate when this share of the smaller section recurs. pub const MIN_SHARE: f64 = 0.3; @@ -117,6 +120,80 @@ fn sequences(text: &str) -> BTreeSet { words.windows(3).map(|w| w.join(" ")).collect() } +/// Locale codes a documentation tree or file name carries, such as `zh-cn` +/// in `docs/zh-cn/` or `zh` in `README_zh.md`. +const LOCALES: &[&str] = &[ + "ar", "bg", "bn", "cn", "cs", "da", "de", "el", "en", "en-gb", "en-us", "es", "fa", "fi", "fr", + "he", "hi", "hu", "id", "it", "ja", "jp", "ko", "kr", "ms", "nb", "nl", "no", "pl", "pt", + "pt-br", "pt_br", "ro", "ru", "sv", "th", "tr", "tw", "uk", "vi", "zh", "zh-cn", "zh-hans", + "zh-hant", "zh-tw", "zh_cn", "zh_tw", +]; + +/// Codes that are also common words at the end of a file name, as in +/// `user_id.md`; they name a locale only as a directory. +const WORD_LOCALES: &[&str] = &["id", "it", "no"]; + +/// The locale a document's path names, in lower case: a directory such as +/// `docs/ja/`, or the last part of its file name after a dot or underscore, +/// as in `README.zh-CN.md` or `README_zh.md`. +fn locale(path: &Path) -> Option { + let lower = path + .to_string_lossy() + .replace('\\', "/") + .to_ascii_lowercase(); + let mut parts: Vec<&str> = lower.split('/').collect(); + let name = parts.pop()?; + if let Some(dir) = parts.iter().find(|d| LOCALES.contains(d)) { + return Some((*dir).to_string()); + } + let stem = name.rsplit_once('.').map_or(name, |(stem, _)| stem); + ['.', '_'].into_iter().find_map(|separator| { + let (_, suffix) = stem.rsplit_once(separator)?; + (LOCALES.contains(&suffix) && !WORD_LOCALES.contains(&suffix)).then(|| suffix.to_string()) + }) +} + +/// Whether two documents are written for readers of different languages: +/// their paths name different locales, not both English (`docs/en/` and +/// `docs/zh-cn/`, `README.md` and `README_zh.md`, `ja/` and `zh/`), or +/// their prose is written in different scripts. A translation repeats its +/// original on purpose: freellmapi, cc-switch and rtk had 12 translated +/// pairs reported as repetition, the translation question, which reads the +/// two texts alone, answering from 0.04 to 0.71. No corpus finding pairs +/// documents of two languages. +pub fn other_language(a: (&Path, &str), b: (&Path, &str)) -> bool { + let english = |l: &Option| { + l.as_deref() + .is_none_or(|l| l == "en" || l.starts_with("en-")) + }; + let (x, y) = (locale(a.0), locale(b.0)); + (x != y && !(english(&x) && english(&y))) || other_script(a.1, b.1) +} + +/// Whether one text's prose is mostly in a script other than Latin, such as +/// Han, Kana, Hangul or Cyrillic, and the other's almost never. +fn other_script(a: &str, b: &str) -> bool { + let (x, y) = (non_latin_share(a), non_latin_share(b)); + let (high, low) = if x > y { (x, y) } else { (y, x) }; + high >= 0.3 && low < 0.05 +} + +/// The share of the letters outside program code that are not Latin. +fn non_latin_share(text: &str) -> f64 { + let letters: Vec = outside_code(text) + .chars() + .filter(|c| c.is_alphabetic()) + .collect(); + if letters.is_empty() { + return 0.0; + } + let other = letters + .iter() + .filter(|c| !c.is_ascii() && !matches!(**c, '\u{00C0}'..='\u{024F}')) + .count(); + other as f64 / letters.len() as f64 +} + /// A candidate pair: two indexes into the texts and their share. type Pair = (usize, usize, f64); @@ -220,6 +297,46 @@ fn capped(texts: &[Text<'_>], found: Vec) -> (Vec, usize) { mod tests { use super::*; + #[test] + fn documents_for_readers_of_other_languages_are_translations() { + let pair = |a: &str, b: &str| { + other_language( + (Path::new(a), "Install the tool and run it."), + (Path::new(b), "Install the tool and run it."), + ) + }; + assert!(pair( + "docs/en/api/OVERVIEW.md", + "docs/zh-cn/api/OVERVIEW.md" + )); + assert!(pair("README.md", "README_zh.md")); + assert!(pair("README.md", "README.zh-CN.md")); + assert!(pair( + "docs/user-manual/ja/intro.md", + "docs/user-manual/zh/setup.md" + )); + assert!( + !pair("docs/en/guide.md", "docs/setup.md"), + "English either way" + ); + assert!(!pair("README.zh-CN.md", "docs/next/README.zh-CN.md")); + assert!( + !pair("docs/user_id.md", "docs/guide.md"), + "`id` ends a file name as a word" + ); + assert!(!pair("docs/guide.md", "docs/setup.md")); + assert!(other_language( + ( + Path::new("docs/proxy.md"), + "## Scheme support\n\nHTTP and SOCKS5 proxies work." + ), + ( + Path::new("docs/proxy-notes.md"), + "## 协议支持\n\n支持 HTTP 和 SOCKS5 代理。" + ), + )); + } + #[test] fn repeated_sections_across_files_pair_up() { let stack = "The frontend uses React with Vite and Tailwind while the backend runs Hono on Node with Supabase for storage and auth"; diff --git a/src/units/compose/mod.rs b/src/units/compose/mod.rs index 14cf863..edc0bf7 100644 --- a/src/units/compose/mod.rs +++ b/src/units/compose/mod.rs @@ -309,7 +309,16 @@ fn undecided_unit(unit: &UnitPlan, answers: &Answers<'_>) -> Undecided { // Instruction sections and section pairs settle some signals by others. let settled_sections = match unit.rule { catalog::AGENT_CONTEXT => super::outcome::section_signals(&get), - catalog::DOC_DUPLICATION => super::outcome::pair_signals(&get), + catalog::DOC_DUPLICATION => super::outcome::pair_signals( + &get, + matches!( + unit.detail, + Detail::DocPair { + translated: true, + .. + } + ), + ), _ => None, }; let mut questions: Vec = match (settled_values, settled_sections) { diff --git a/src/units/drift.rs b/src/units/drift.rs index 95129e1..8e1a309 100644 --- a/src/units/drift.rs +++ b/src/units/drift.rs @@ -263,6 +263,10 @@ impl<'a> Shared<'a> { other, check: fits.then(|| (request, asked).into()), settle: fits.then(|| settle.into()), + translated: crate::docs::overlap::other_language( + (file.path, §ion.text), + (other_path, &other_section.text), + ), }, recheck: None, } diff --git a/src/units/mod.rs b/src/units/mod.rs index 11b74f2..47af5e4 100644 --- a/src/units/mod.rs +++ b/src/units/mod.rs @@ -250,6 +250,9 @@ pub enum Detail { check: Option, /// How the two sections relate, asked when the check stays undecided. settle: Option, + /// The two documents are written for readers of different languages, + /// by their paths' locales or their scripts. + translated: bool, }, /// A heading section of an agent instruction file. Section { diff --git a/src/units/outcome/mod.rs b/src/units/outcome/mod.rs index 917b3a1..0496c24 100644 --- a/src/units/outcome/mod.rs +++ b/src/units/outcome/mod.rs @@ -295,7 +295,16 @@ fn rule_outcome(unit: &UnitPlan, answers: &Answers<'_>) -> Option { catalog::WORKFLOWS => workflows_outcome(&get), catalog::LARGE_DOCS => document_outcome(&get), catalog::DOC_STALENESS => staleness_outcome(&get, &unit.detail), - catalog::DOC_DUPLICATION => doc_pair_outcome(&get), + catalog::DOC_DUPLICATION => doc_pair_outcome( + &get, + matches!( + unit.detail, + Detail::DocPair { + translated: true, + .. + } + ), + ), catalog::AGENT_CONTEXT => { section_signals(&get).map(|s| strongest(&s.iter().map(|(_, o)| *o).collect::>())) } diff --git a/src/units/outcome/pairs.rs b/src/units/outcome/pairs.rs index 8888598..de59e24 100644 --- a/src/units/outcome/pairs.rs +++ b/src/units/outcome/pairs.rs @@ -10,16 +10,25 @@ use super::{documentation::kind_share, *}; /// is a note. Sections about different subjects settle what stays /// undecided, and so does the pair's relation, asked apart: a repetition /// or a contradiction it rules out clears that check. -pub(super) fn doc_pair_outcome<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> Option { - let signals: Vec = pair_signals(get)?.into_iter().map(|(_, o)| o).collect(); +pub(super) fn doc_pair_outcome<'a>( + get: &impl Fn(&str) -> Option<&'a Answer>, + translated: bool, +) -> Option { + let signals: Vec = pair_signals(get, translated)? + .into_iter() + .map(|(_, o)| o) + .collect(); Some(strongest(&signals)) } -/// Each check of a section pair with its settled outcome. +/// Each check of a section pair with its settled outcome; `translated` when +/// the documents' paths or scripts show different languages. pub(in crate::units) fn pair_signals<'a>( get: &impl Fn(&str) -> Option<&'a Answer>, + translated: bool, ) -> Option> { - let translated = get("translation").is_some_and(|a| matches!(noul(a), Outcome::Review(_))); + let translated = + translated || get("translation").is_some_and(|a| matches!(noul(a), Outcome::Review(_))); let covers = if translated { &[][..] } else { From 7bb31b6df52a0e40eb608b05d9308789032cb5e6 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:09:18 -0300 Subject: [PATCH 06/12] Ask each kind of security finding the one thing that decided it wrongly The hot projects' security findings were mostly wrong in a few ways, each missing one fact. Each is now asked, only after the finding: - SQL, command or code findings: what their values can hold where they enter the text (fixed clauses a key selects, parsed ids, the program's own names, or a query the sender may run anyway make a note). - Markup considers on parameters: what their values hold where they enter the markup, as markup reviews already were. - Weak-settings findings of the escaping check: what the unescaped HTML holds (a library's escaped output or markup the program ships). - Logging findings: whether the value is the output its user asked for. - Error-detail findings: who reads the error text, with the README's opening; only the operator, the project's own services or the person running it locally, at 0.80, make a note. Corpus (55 projects with labeled security findings): 15 wrong and 1 debatable become notes for 1 right; held-out projects unchanged; about $0.02 of new questions. Hot projects: freellmapi -5, cc-switch -2, headroom -26, rtk -1, multica -17 reviews and considers. The post-finding follow-ups of a security unit move into one boxed Confirms struct. --- CHANGELOG.md | 6 + site/src/privacy-and-cost.md | 1 + src/catalog.rs | 10 +- src/config.rs | 4 + src/docs/mod.rs | 45 ++++++ src/options/mod.rs | 5 + src/units/access.rs | 1 + src/units/compose/due.rs | 66 +++++--- src/units/compose/mod.rs | 10 +- src/units/evidence.rs | 3 + src/units/follow_ups.rs | 28 +++- src/units/handlers/mod.rs | 1 + src/units/mod.rs | 39 +++-- src/units/outcome/exposure.rs | 56 ++++++- src/units/outcome/injection.rs | 17 +- src/units/outcome/mod.rs | 6 +- src/units/plan/file.rs | 1 + src/units/plan/mod.rs | 2 + src/units/questions/security/confirm.rs | 117 +++++++++++++- src/units/security/mod.rs | 109 +++++++++++-- src/units/tests/mod.rs | 46 +++++- src/units/tests/security/confirms.rs | 197 +++++++++++++++++++++++- src/units/tests/security/mod.rs | 15 +- src/units/tests/security/php.rs | 4 +- src/units/tests/security/settings.rs | 5 +- src/units/wording/security.rs | 51 +++++- 26 files changed, 762 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3f302c..df3bfb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ Fixes from running JevGate on nine widely used projects under daily development - Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. - Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. - Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. No corpus finding changes. +- Duplication: two documents whose paths name different locales (`docs/en` and `docs/zh-cn`, `README.md` and `README_zh.md`) or whose prose is in different scripts are translations; their sections are asked only whether they disagree, as when the translation question says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition, the translation question answering 0.04 to 0.71 on them. No corpus finding pairs two languages. +- Security findings on the hot projects were mostly wrong in the same few ways, and each is now asked the one thing that decided it, only after the finding. On the corpus's 55 projects with labeled security findings, 15 findings labeled wrong and 1 debatable are notes, for 1 labeled right; nothing changed on the held-out projects. The run asked about $0.02 of new questions. + - Injection: an SQL, command or code finding is asked what its values can hold where they enter the text; fixed clauses a key selects (freellmapi's `ORDER BY` from a map of presets), parsed ids (multica's option UUIDs), the program's own names or a query the sender may run anyway make it a note. The 23 such corpus findings labeled right put at most 0.12 on those, one labeled wrong 0.82. + - Injection: a markup consider on the function's parameters is also asked what its values hold where they enter the markup, as markup reviews already were; text escaped before (a syntax highlighter's output), typed values or the program's own markup make it a note. 11 labeled wrong and 1 labeled right became notes; the other 44 labeled right put at most 0.47 there. + - Unsafe settings: a finding the escaping check raised is asked what the unescaped HTML holds; markup a library built from escaped text, or that ships with the program (freellmapi's highlighted code, cc-switch's bundled provider icons, multica's KaTeX output), makes it a note. The three corpus findings labeled right put at most 0.18 there. + - Sensitive data: a logging finding whose value is the output the person asked for, such as rtk's `env` command or a CLI printing a new token, is a note (two labeled wrong at 0.96 and 0.98, twelve labeled right at most 0.43). An error-detail finding is asked who reads the error text, with the opening of the root README: when, at 0.80, only the operator, the project's own services or the person running it on their own machine read it, it is a note. That cleared headroom's 24 local-proxy reviews and 6 of multica's daemon endpoints, while multica's handlers for its hosted service's users stay reviews; on the corpus it cleared 2 labeled wrong or debatable and none of the 51 labeled right, which put at most 0.46 there. The README opening is sent only with this question, bounded by the upload patterns. ## [0.24.1] - 2026-09-27 diff --git a/site/src/privacy-and-cost.md b/site/src/privacy-and-cost.md index 25992e4..8dcbcb9 100644 --- a/site/src/privacy-and-cost.md +++ b/site/src/privacy-and-cost.md @@ -2,6 +2,7 @@ - **What is uploaded:** only the selected units of source, bounded by `upload_allow` and `upload_deny`. `--dry-run --show-requests` prints every initial request body without credentials or network access. - **Instruction files:** uploaded only when a documentation rule is selected, and still bounded by the upload patterns. +- **README opening:** a sensitive-data finding about error details is asked who reads the error text, with the first 1,200 characters of the root README's prose (images, badges and HTML left out), unless `upload_deny` covers the README or `upload_allow` leaves it out. - **Credentials:** a check reads `TYPESAFE_API_KEY` from the environment, then `--env-file` or the repository's `.env`, then the key saved by `jevgate auth login` (OS credential store, or an owner-only file). The key is never printed or written to reports. - **Cost:** every run prints its input tokens and an estimated cost. Cached answers cost nothing. - **Secrets:** out of scope on purpose, because judging secrets would mean uploading them. Use a local secret scanner. diff --git a/src/catalog.rs b/src/catalog.rs index 0207197..3432a6c 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -292,15 +292,15 @@ pub fn rule_version(key: &str) -> &'static str { SHARED_LOGIC => "22", TEST_VALUE => "7", TEST_REDUNDANCY => "4", - INJECTION => "12", - SENSITIVE_DATA => "8", + INJECTION => "13", + SENSITIVE_DATA => "9", HARDCODED_VALUES => "8", - UNSAFE_SETTINGS => "6", + UNSAFE_SETTINGS => "7", AGENT_CONTEXT => "3", COMMENTS => "3", - LARGE_DOCS => "3", + LARGE_DOCS => "4", ACCESS_CONTROL => "4", - DOC_STALENESS | DOC_DUPLICATION => "3", + DOC_STALENESS | DOC_DUPLICATION => "4", WORKFLOWS => "2", _ => "1", } diff --git a/src/config.rs b/src/config.rs index 151e072..2e64abc 100644 --- a/src/config.rs +++ b/src/config.rs @@ -158,6 +158,10 @@ impl ConfigContext { args.include_tests |= self.config.include_tests; args.model = args.model.take().or_else(|| self.config.model.clone()); args.cache_ttl_secs = args.cache_ttl_secs.or(self.config.cache_ttl_secs); + args.project = crate::docs::project_opening( + &self.root, + &crate::boundary::Boundary::new(&self.config)?, + ); self.configure_rules(args)?; self.configure_gate(args)?; self.configure_budgets(args) diff --git a/src/docs/mod.rs b/src/docs/mod.rs index efe3d4e..56bfd5e 100644 --- a/src/docs/mod.rs +++ b/src/docs/mod.rs @@ -126,6 +126,51 @@ const TEACHING: [&str; 8] = [ "all comments in this project", ]; +/// Characters of a README's opening sent as what the project is. +const OPENING_CHARS: usize = 1_200; + +/// The opening of the README at the repository root, when the upload +/// boundary permits it: its prose and headings without images, badges or +/// HTML, up to `OPENING_CHARS` characters. Only questions about who reads a +/// program's responses send it. +pub fn project_opening(root: &Path, boundary: &crate::boundary::Boundary) -> Option { + let mut names: Vec = std::fs::read_dir(root) + .ok()? + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.to_lowercase().starts_with("readme")) + .collect(); + // README.md before README.rst or a translated README_zh.md. + names.sort_by_key(|n| (n.len(), !n.to_lowercase().ends_with(".md"))); + let name = names.first()?; + if !boundary.permits(Path::new(name)) { + return None; + } + let text = crate::inventory::read_source(&root.join(name), TEACHING_READ_BYTES).ok()?; + let mut opening = String::new(); + for line in text.lines() { + let line = line.trim(); + let decoration = line.starts_with('<') + || line.starts_with("![") + || line.starts_with("[![") + || line.starts_with("[!") + || line.starts_with("---"); + if line.is_empty() || decoration { + continue; + } + if !opening.is_empty() { + opening.push('\n'); + } + opening.push_str(line); + if opening.chars().count() >= OPENING_CHARS { + break; + } + } + let opening: String = opening.chars().take(OPENING_CHARS).collect(); + (!opening.is_empty()).then_some(opening) +} + /// Bytes of a README or CONTRIBUTING file read for the teaching phrases. const TEACHING_READ_BYTES: u64 = 262_144; diff --git a/src/options/mod.rs b/src/options/mod.rs index f2836bd..447ba96 100644 --- a/src/options/mod.rs +++ b/src/options/mod.rs @@ -166,6 +166,11 @@ pub struct CheckArgs { /// Levels for the files `[[scope]]` entries match, in configuration order. #[arg(skip)] pub path_fail_on: Vec, + /// The opening of the repository's README when the upload boundary + /// permits it: what the program is and who runs it, for the question of + /// who reads an error-detail finding's responses. + #[arg(skip)] + pub project: Option, /// Output format [default: agent; jsonl with --watch; json with --show-requests] #[arg(long, value_enum, help_heading = OUTPUT)] pub format: Option, diff --git a/src/units/access.rs b/src/units/access.rs index 9c8e73c..25d2964 100644 --- a/src/units/access.rs +++ b/src/units/access.rs @@ -84,6 +84,7 @@ pub(super) fn plan( source_hash: &input.result.source_hash, model: args.model(), budget, + project: args.project.as_deref(), framework: None, } }; diff --git a/src/units/compose/due.rs b/src/units/compose/due.rs index d735bec..9fba8c1 100644 --- a/src/units/compose/due.rs +++ b/src/units/compose/due.rs @@ -48,29 +48,61 @@ pub fn unsettled(unit: &UnitPlan, judgments: &[Judgment]) -> BTreeSet<&'static s } /// Security units whose finding a confirm Choice of their own follows, not -/// yet asked: an injection finding whose one concern is a path (what its -/// paths can hold), or markup or a redirect unless its values are asked -/// already (what they hold, where they lead), and a sensitive-data finding -/// its log checks raised (when the log line runs). +/// yet asked: an injection finding whose one concern is a path or markup +/// (what its paths or values can hold), or a redirect unless its values are +/// asked already (where it leads), and a sensitive-data finding its log +/// checks raised (when the log line runs). A markup consider on the +/// function's parameters is asked both what its values can hold and what +/// they hold where they enter the markup: escaped text another party wrote +/// is harmless there. pub fn unconfirmed_units(plan: &FilePlan, judgments: &[Judgment]) -> BTreeSet { + confirm_due(plan, judgments, |u, outcome, resolved| { + let Detail::Security { confirms, .. } = &u.detail else { + return false; + }; + let get = |q: &str| resolved.get(q).copied(); + let kind = confirmable(&get).filter(|k| !QUERIED.contains(k)); + confirms.checked.is_some() + && kind.is_some_and(|k| k == "path" || k == "markup" || !values_due(outcome, resolved)) + || confirms.logging.is_some() && logs_found(&get) + }) +} + +/// Injection findings whose one concern is SQL, a command or evaluated code, +/// not yet asked what their values hold where they enter it, unless what +/// their values can hold is asked already; weak-settings findings their +/// escaping check raised, not yet asked what the unescaped HTML holds; and +/// sensitive-data findings their error-detail checks raised, not yet asked +/// who reads the error text. +pub fn unqueried_units(plan: &FilePlan, judgments: &[Judgment]) -> BTreeSet { + confirm_due(plan, judgments, |u, outcome, resolved| { + let Detail::Security { confirms, .. } = &u.detail else { + return false; + }; + let get = |q: &str| resolved.get(q).copied(); + confirms.queried.is_some() + && confirmable(&get).is_some_and(|k| QUERIED.contains(&k)) + && !values_due(outcome, resolved) + || confirms.rendered.is_some() && escape_found(&get) + || confirms.readers.is_some() && errors_found(&get) + }) +} + +/// Judged units with no locate answer yet whose outcome is a review or +/// consider and that `due` selects. +fn confirm_due( + plan: &FilePlan, + judgments: &[Judgment], + due: impl Fn(&UnitPlan, Outcome, &Answers<'_>) -> bool, +) -> BTreeSet { plan.units .iter() .filter(|u| u.presence == Presence::Judged) .filter(|u| answers(judgments, &u.id, Pass::Locate).is_empty()) .filter(|u| { - let Detail::Security { - checked, logging, .. - } = &u.detail - else { - return false; - }; let (outcome, resolved) = resolved(u, judgments); - let get = |q: &str| resolved.get(q).copied(); - let kind = confirmable(&get); matches!(outcome, Outcome::Review(_) | Outcome::Consider(_)) - && (checked.is_some() - && (kind == Some("path") || kind.is_some() && !values_due(outcome, &resolved)) - || logging.is_some() && logs_found(&get)) + && due(u, outcome, &resolved) }) .map(|u| u.id.clone()) .collect() @@ -119,9 +151,7 @@ pub(super) fn locate_due(unit: &UnitPlan, judgments: &[Judgment]) -> bool { // An injection consider rests on the function's parameters unless // its origin was another party; one that rests on a path is asked // what its paths can hold instead. - Detail::Security { - confirm: Some(_), .. - } => { + Detail::Security { confirms, .. } if confirms.values.is_some() => { values_due(outcome, &resolved) && confirmable(&|q| resolved.get(q).copied()) != Some("path") } diff --git a/src/units/compose/mod.rs b/src/units/compose/mod.rs index edc0bf7..5e603cd 100644 --- a/src/units/compose/mod.rs +++ b/src/units/compose/mod.rs @@ -7,10 +7,10 @@ use super::{ Access, Block, Detail, FilePlan, Presence, UnitPlan, outcome::{ - Answers, Outcome, at_most_note, benefit, checks, choice, choice_mass, confirmable, - document_split, logs_found, lowered, noul, open, organization_outcome, origin_outcome, - part_answers, score, separable_part, settled_checks, several_kind, unit_outcome, - value_signals, + Answers, Outcome, QUERIED, at_most_note, benefit, checks, choice, choice_mass, confirmable, + document_split, errors_found, escape_found, logs_found, lowered, noul, open, + organization_outcome, origin_outcome, part_answers, score, separable_part, settled_checks, + several_kind, unit_outcome, value_signals, }, wording::{Wording, comment_reason, comment_wording}, wording::{ @@ -40,7 +40,7 @@ use caps::*; use comments::*; pub use due::{ finished_plans, uncertain_units, unconfirmed_units, unkinded_units, unkinded_values, - unlocated_units, unparted_units, unsettled, untraced_units, + unlocated_units, unparted_units, unqueried_units, unsettled, untraced_units, }; use located::*; use redundant::*; diff --git a/src/units/evidence.rs b/src/units/evidence.rs index 2e9c36e..24071bc 100644 --- a/src/units/evidence.rs +++ b/src/units/evidence.rs @@ -23,6 +23,9 @@ pub(super) struct FileContext<'a> { /// What a web framework makes of the file, such as a Next.js route /// handler or Server Actions module, sent beside its path. pub framework: Option, + /// The opening of the repository's README, sent only with the question + /// who reads a program's error text. + pub project: Option<&'a str>, } impl FileContext<'_> { diff --git a/src/units/follow_ups.rs b/src/units/follow_ups.rs index 567adcf..bed7909 100644 --- a/src/units/follow_ups.rs +++ b/src/units/follow_ups.rs @@ -9,19 +9,34 @@ use std::collections::BTreeSet; /// per hardcoded-value function raised to a review or consider, per redundant /// test pair raised to a review, per test that asserts internal details, per /// injection consider that rests on its parameters, per injection finding -/// that rests on a path and per logging finding. +/// that rests on a path, markup, a redirect, SQL, a command or code, per +/// weak-settings finding that rests on unescaped HTML and per logging +/// finding. pub fn locates(plan: &Plan, files: &[FileResult]) -> Vec { let mut planned = follow_ups( plan, files, compose::unconfirmed_units, |unit| match &unit.detail { - Detail::Security { - checked, logging, .. - } => checked.as_ref().or(logging.as_ref()), + Detail::Security { confirms, .. } => { + confirms.checked.as_ref().or(confirms.logging.as_ref()) + } _ => None, }, ); + planned.extend(follow_ups( + plan, + files, + compose::unqueried_units, + |unit| match &unit.detail { + Detail::Security { confirms, .. } => confirms + .queried + .as_ref() + .or(confirms.rendered.as_ref()) + .or(confirms.readers.as_ref()), + _ => None, + }, + )); planned.extend(follow_ups( plan, files, @@ -31,9 +46,8 @@ pub fn locates(plan: &Plan, files: &[FileResult]) -> Vec { | Detail::Document { locate, .. } | Detail::Values { locate, .. } | Detail::Constants { locate, .. } => locate.as_ref(), - Detail::TestPair { confirm, .. } - | Detail::Test { confirm } - | Detail::Security { confirm, .. } => confirm.as_ref(), + Detail::TestPair { confirm, .. } | Detail::Test { confirm } => confirm.as_ref(), + Detail::Security { confirms, .. } => confirms.values.as_ref(), _ => None, }, )); diff --git a/src/units/handlers/mod.rs b/src/units/handlers/mod.rs index a607e3a..b8cff91 100644 --- a/src/units/handlers/mod.rs +++ b/src/units/handlers/mod.rs @@ -54,6 +54,7 @@ pub(super) fn plan( source_hash: &input.result.source_hash, model: args.model(), budget, + project: args.project.as_deref(), framework: super::nextjs::describe( &input.result.path, input.source.as_deref().unwrap_or(""), diff --git a/src/units/mod.rs b/src/units/mod.rs index 47af5e4..b1dbd64 100644 --- a/src/units/mod.rs +++ b/src/units/mod.rs @@ -133,6 +133,32 @@ impl From<(Value, Asked)> for FollowUp { } } +/// The Choices a security unit's finding is asked after it is raised, each +/// only for the findings it serves. +#[derive(Clone, Debug, Default)] +pub struct Confirms { + /// For injection, what the values it places can hold, asked only after a + /// consider that rests on its parameters. + pub values: Option, + /// For injection, what the values of a path, markup or redirect finding + /// can hold or where they lead, asked only after a finding whose one + /// concern is one of those. + pub checked: Option, + /// For injection, what the values of an SQL, command or code finding hold + /// where they enter it, asked only after a finding whose one concern is + /// one of those. + pub queried: Option, + /// For weak settings, what the HTML written without escaping holds, asked + /// only after a finding its escaping check raised. + pub rendered: Option, + /// For sensitive data, who reads its error text, asked only after a + /// finding its error-detail checks raised. + pub readers: Option, + /// For sensitive data, when its log line runs, asked only after a finding + /// its log checks raised. + pub logging: Option, +} + #[derive(Clone, Debug)] pub enum Detail { Function { @@ -207,16 +233,9 @@ pub enum Detail { /// trace and recheck, such as where its URLs come from or its output /// goes; each is asked only while its checks are undecided. settles: Vec, - /// For injection, what the values it places can hold, asked only - /// after a consider that rests on its parameters. - confirm: Option, - /// For injection, what the values of a path, markup or redirect - /// finding can hold or where they lead, asked only after a finding - /// whose one concern is one of those. - checked: Option, - /// For sensitive data, when its log line runs, asked only after a - /// finding its log checks raised. - logging: Option, + /// The Choices asked after a finding, each only for the findings it + /// serves. + confirms: Box, /// Django code, asked the Django checks: a weak setting must be /// named by one of them. django: bool, diff --git a/src/units/outcome/exposure.rs b/src/units/outcome/exposure.rs index bfd568b..7305e7d 100644 --- a/src/units/outcome/exposure.rs +++ b/src/units/outcome/exposure.rs @@ -72,6 +72,50 @@ pub(in crate::units) fn opted_in<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) choice_mass(get("logged_when"), &[questions::OPT_IN_LOGGING]).is_some_and(at_least) } +/// Whether the log line of a logging finding leans toward no log at all: the +/// value is the output a person asked for, such as a command printing their +/// own environment. Its log signals are then at most a note. +pub(in crate::units) fn not_logged<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> bool { + choice_mass(get("logged_when"), &[questions::SHOWN_NOT_LOGGED]) + .is_some_and(|p| probability_at_least(p, LEADING_PROBABILITY)) +} + +/// Whether the escaping check found markup written unescaped: its HTML is +/// then asked what it holds. +pub(in crate::units) fn escape_found<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> bool { + matches!(get("escape").map(noul), Some(Outcome::Review(_))) +} + +/// Whether the HTML written unescaped leans toward markup that cannot carry +/// another party's tags: escaped or sanitized by the library that built it, +/// shipped with the program, or typed values. Its escaping signal is then a +/// note that no longer names a setting to change. +pub(in crate::units) fn inert_html<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> bool { + choice_mass(get("raw_html"), &questions::INERT_HTML) + .is_some_and(|p| probability_at_least(p, LEADING_PROBABILITY)) +} + +/// Whether an error-detail check found error text sent to a client: who +/// reads it is then asked. +pub(in crate::units) fn errors_found<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> bool { + ERROR_SIGNALS + .iter() + .any(|q| matches!(get(q).map(noul), Some(Outcome::Review(_)))) +} + +/// Whether the readers of a function's error text are, at the threshold, +/// people who can read the program's logs anyway: its operator, the +/// project's own services, or the person running it on their own machine. +/// Its error-detail signals are then at most a note. Leaning was not enough: +/// multica's handlers that the users of its hosted service call, for skills, +/// issues and source context, put 0.50 to 0.59 on those readers. At the threshold, 2 of the 44 +/// corpus findings labeled wrong or debatable are notes and none of the 51 +/// labeled right, which put at most 0.46 there; so are headroom's 24 +/// local-proxy reviews and 6 of multica's daemon and runtime endpoints. +pub(in crate::units) fn private_readers<'a>(get: &impl Fn(&str) -> Option<&'a Answer>) -> bool { + choice_mass(get("error_readers"), &questions::PRIVATE_READERS).is_some_and(at_least) +} + /// A judged exposure answer and how far it leans toward its concern. type Signal = (Outcome, f64); @@ -112,7 +156,9 @@ fn exposure_signals<'a>( .then(|| messages(get)) .flatten(); let away = rule == catalog::SENSITIVE_DATA && away_from_clients(get); - let opted_in = rule == catalog::SENSITIVE_DATA && opted_in(get); + let opted_in = rule == catalog::SENSITIVE_DATA && (opted_in(get) || not_logged(get)); + let inert = rule == catalog::UNSAFE_SETTINGS && inert_html(get); + let private = rule == catalog::SENSITIVE_DATA && private_readers(get); let judge = |question: &str, answer: &Answer| { let signal = exposure_signal(question, answer, own, away); match signal { @@ -121,6 +167,14 @@ fn exposure_signals<'a>( { (Outcome::Note(p), lean) } + (Outcome::Review(p) | Outcome::Consider(p), _) if inert && question == "escape" => { + (Outcome::Note(p), 0.0) + } + (Outcome::Review(p) | Outcome::Consider(p), lean) + if private && ERROR_SIGNALS.contains(&question) => + { + (Outcome::Note(p), lean) + } signal => signal, } }; diff --git a/src/units/outcome/injection.rs b/src/units/outcome/injection.rs index 6588527..c30ed51 100644 --- a/src/units/outcome/injection.rs +++ b/src/units/outcome/injection.rs @@ -69,12 +69,19 @@ pub(in crate::units) fn injection_outcome<'a>( /// The kinds of injection whose values a confirm Choice asks about after /// the finding, with its question and the options that make it a note. -const CONFIRMED: [(&str, &str, &[&str]); 3] = [ +const CONFIRMED: [(&str, &str, &[&str]); 6] = [ ("path", "paths", &questions::CONFINED_PATHS), ("markup", "markup_values", &questions::HARMLESS_MARKUP), ("redirect", "redirect_reach", &questions::OWN_SITE), + ("sql", "query_values", &questions::HARMLESS_QUERY), + ("shell", "query_values", &questions::HARMLESS_QUERY), + ("code", "query_values", &questions::HARMLESS_QUERY), ]; +/// The kinds whose confirm Choice is `query_values`, sent in a request of +/// its own; the others share the `checked` request. +pub(in crate::units) const QUERIED: [&str; 3] = ["sql", "shell", "code"]; + /// The one kind of injection a finding rests on, when every check that /// found a variable placed unhandled is that kind and a confirm Choice asks /// about it. @@ -101,7 +108,13 @@ pub(in crate::units) fn confirmable<'a>( /// put at most 0.22 on values that cannot open a tag, and vaultwarden's /// percent-encoded username 0.56; the 6 redirect findings labeled right put /// at most 0.44 on staying on the site, and vaultwarden's admin path and -/// shiori's login page 0.68 and 0.58. +/// shiori's login page 0.68 and 0.58. Asked too of markup considers on the +/// function's parameters, it made notes of 11 labeled wrong (escaped +/// before, typed, or the program's own markup) and 1 labeled right, a JSP +/// header writing a session value, at 0.69; the other 44 labeled right put +/// at most 0.47 there. SQL, command and code findings are asked what their +/// text can hold: fixed clauses a key selects, parsed ids, or a query the +/// sender may run anyway. pub(in crate::units) fn harmless<'a>( get: &impl Fn(&str) -> Option<&'a Answer>, ) -> Option<&'static str> { diff --git a/src/units/outcome/mod.rs b/src/units/outcome/mod.rs index 0496c24..88d2c82 100644 --- a/src/units/outcome/mod.rs +++ b/src/units/outcome/mod.rs @@ -27,11 +27,11 @@ pub(super) use comments::{comment_concern_kind, comment_outcome, comment_signals use documentation::staleness_outcome; pub(super) use documentation::{document_outcome, document_split, section_signals}; pub(super) use exposure::{ - Messages, django_settings_outcome, exposure_outcome, logs_found, messages, opted_in, - settings_module_outcome, + Messages, django_settings_outcome, errors_found, escape_found, exposure_outcome, inert_html, + logs_found, messages, not_logged, opted_in, private_readers, settings_module_outcome, }; pub(super) use injection::{ - RESOURCE_CHECKS, confirmable, harmless, injection_outcome, origin_outcome, + QUERIED, RESOURCE_CHECKS, confirmable, harmless, injection_outcome, origin_outcome, }; pub(super) use maintainability::{ PartAnswers, benign_key, function_outcome, organization_outcome, separable_part, several_kind, diff --git a/src/units/plan/file.rs b/src/units/plan/file.rs index 3c03745..7c2da45 100644 --- a/src/units/plan/file.rs +++ b/src/units/plan/file.rs @@ -144,6 +144,7 @@ fn file_context<'a>( source_hash: &input.result.source_hash, model: args.model(), budget, + project: args.project.as_deref(), framework: crate::components::server_template(&input.result.path) .then(|| crate::components::TEMPLATE_SCRIPT.to_string()) .or_else(|| { diff --git a/src/units/plan/mod.rs b/src/units/plan/mod.rs index 2986009..ec3a64d 100644 --- a/src/units/plan/mod.rs +++ b/src/units/plan/mod.rs @@ -155,6 +155,7 @@ fn plan_workflows(scope: &Scope<'_>, args: &CheckArgs, budget: Limits<'_>, resul source_hash: &input.result.source_hash, model: args.model(), budget, + project: args.project.as_deref(), framework: None, }; workflows::plan(&context, &mut file, &mut result.requests); @@ -186,6 +187,7 @@ fn plan_document( source_hash: &input.result.source_hash, model: args.model(), budget, + project: args.project.as_deref(), framework: None, }; if input.result.role == crate::inventory::DOCS { diff --git a/src/units/questions/security/confirm.rs b/src/units/questions/security/confirm.rs index 03fb336..dd78b12 100644 --- a/src/units/questions/security/confirm.rs +++ b/src/units/questions/security/confirm.rs @@ -119,6 +119,75 @@ pub fn markup_values(code: &str, callers: bool) -> Value { /// The options of `markup_values` that cannot open a tag or attribute. pub const HARMLESS_MARKUP: [&str; 3] = ["encoded", "typed", "own"]; +/// What the values of a finding whose one concern is SQL, a shell command or +/// evaluated code hold where they enter that text, asked only after such a +/// finding. freellmapi builds an `ORDER BY` from a map of fixed clauses keyed +/// by a route parameter, rejecting other keys, and a `WHERE` from one of two +/// literals; multica embeds option ids only after `uuid.Parse` accepts them. +/// The query check reads a variable joined into the text, whatever it can +/// hold. Those four answered 0.53 to 0.66 on the harmless options; on the +/// corpus, the 23 such findings labeled right put at most 0.12 there, and +/// one labeled wrong, a JSP page parsing its parameter as a number, 0.82. +pub fn query_values(code: &str, callers: bool, types: bool) -> Value { + let types_note = if types { + " `types_named_in_parameters` holds the definitions of the project's types that its parameters name, with their attributes." + } else { + "" + }; + let note = if callers { + format!("{CALLERS}{types_note} {EVIDENCE}") + } else { + format!("{} {EVIDENCE}", types_note.trim_start()) + .trim_start() + .to_string() + }; + json!({ + "type": "choice", + "instructions": { + "question": format!("What can the values that `{code}` joins into the text of a query, command or evaluated code hold where they enter it?"), + "note": note, + }, + "criteria": { + "fixed": "Only text written in the code: literals or constants, or one of a fixed set of strings chosen by a key, such as a map, object or switch of fixed clauses, where any other key gets no text or is rejected, even when the key comes from a request.", + "typed": "Numbers, booleans, dates, or ids such as UUIDs, parsed or validated as that type before they are joined, so they cannot hold quotes, spaces or syntax.", + "own": "Names the program keeps for itself, such as its own table and column names, or values from its configuration.", + "allowed": "A query, command or script that the person sending it may run anyway, with their own rights, such as the query box of a database client or the console of an admin tool.", + "raw": "Text another party or a caller wrote, which can hold quotes, spaces or the syntax of the query, command or code.", + "unknown": "Values whose origin or handling is not shown.", + }, + }) +} + +/// The options of `query_values` that cannot change the text's syntax. +pub const HARMLESS_QUERY: [&str; 4] = ["fixed", "typed", "own", "allowed"]; + +/// What the HTML a weak-settings finding writes without escaping holds, +/// asked only after a finding its escaping check raised. freellmapi's code +/// block renders highlight.js output, escaped by the highlighter, cc-switch's +/// provider icon renders SVG bundled with the app, and multica's math view +/// renders KaTeX output: the check reads markup written unescaped, whatever +/// it holds. They answered 0.93, 0.81 and 0.56 on the inert options; the +/// three such corpus findings labeled right put at most 0.18 there. +pub fn raw_html(code: &str) -> Value { + json!({ + "type": "choice", + "instructions": { + "question": format!("What does the HTML or SVG that `{code}` writes without escaping hold?"), + "note": EVIDENCE, + }, + "criteria": { + "encoded": "Markup a library or function built from text it escaped or sanitized first, such as a syntax highlighter's or math renderer's output, HTML passed through a sanitizer such as DOMPurify, or Markdown rendered with raw HTML turned off.", + "own": "Markup the program ships itself: its own templates or strings, or icons and images bundled with it.", + "typed": "Numbers, dates, booleans or ids, or names chosen from a fixed list.", + "raw": "HTML or text as another party or a caller wrote it, which can hold tags, attributes or scripts.", + "unknown": "Markup whose origin or handling is not shown.", + }, + }) +} + +/// The options of `raw_html` whose markup cannot carry another party's tags. +pub const INERT_HTML: [&str; 3] = ["encoded", "own", "typed"]; + /// Where a redirect finding's targets can lead, asked only after a finding /// whose one concern is a redirect. vaultwarden's admin login redirects to /// its admin path followed by the form's value, and shiori's to its login @@ -155,7 +224,11 @@ pub const OWN_SITE: [&str; 3] = ["own_site", "checked", "none"]; /// finding raised by its log checks. vaultwarden logs SSO tokens inside /// `if CONFIG.sso_debug_tokens()`, a setting off by default and documented /// for logging them while troubleshooting: logging an identifier instead, -/// as the finding says, would remove the feature. +/// as the finding says, would remove the feature. rtk's `env` command and +/// freellmapi's setup notes print the user's own values as the output they +/// asked for, which the log checks read as a log. On the corpus, the two +/// such findings labeled wrong answered 0.96 and 0.98 on `output`, and the +/// twelve labeled right at most 0.43. pub fn logged_when(code: &str) -> Value { json!({ "type": "choice", @@ -167,6 +240,7 @@ pub fn logged_when(code: &str) -> Value { "always": "Whenever that code runs, at a level the program logs at in normal operation, such as info, warning or error.", "debug": "Only at debug or trace level, which an operator may turn on to troubleshoot.", "opt_in": "Only when an operator turns on a setting, off by default, whose purpose is to log these values for troubleshooting, such as an option named for logging tokens or request bodies.", + "output": "Never to a log: it shows the value to the person who asked for it, as the output of a command they ran on their own machine or a page they requested.", "none": "It writes no secret or personal value to a log.", }, }) @@ -174,3 +248,44 @@ pub fn logged_when(code: &str) -> Value { /// The option of `logged_when` for a setting whose purpose is the logging. pub const OPT_IN_LOGGING: &str = "opt_in"; + +/// Who reads the error text of an error-detail finding, asked only after +/// such a finding, with the opening of the project's README. The corpus's +/// error-detail findings labeled wrong or debatable were most often read only +/// by the project's own services (14 of 46), and headroom, a proxy its users +/// run on their own machine for their coding agents, had 28 reviews returning +/// an upstream error to that user's own tools. Error text is a leak when +/// people who could not read the program's logs see it. Offered as "the +/// person running it on their own machine" alone, the option took govwa, a +/// training web app, at 0.72: web applications meant for others are named in +/// both options. +pub fn error_readers(code: &str, project: bool) -> Value { + let note = if project { + format!( + "`project.readme_opening` is the start of the repository's README, which says what the program is and who runs it. {EVIDENCE}" + ) + } else { + EVIDENCE.to_string() + }; + json!({ + "type": "choice", + "instructions": { + "question": format!("Who reads the error text that `{code}` sends in its responses?"), + "note": note, + }, + "criteria": { + "public": "People outside the team that runs the program: visitors and users of a website or web application, including one made for training, customers or users of a hosted service, members of other accounts, organizations or tenants, or third-party clients of a public API.", + "operator": "Only the person or team that runs this install, such as the admin console of a self-hosted tool one owner uses, who can read the server's logs anyway.", + "own_services": "Only other parts of the same project in one deployment, such as a back-end service that only the project's own front end or services call.", + "local": "Only the person running the program on their own machine, through a server, proxy or back end it starts for their own tools, such as a local proxy for their coding agent or a desktop app's back end; not a web application whose pages are meant for other people, even when someone runs it on their own machine.", + "unknown": "The readers are not shown.", + }, + }) +} + +/// The options of `error_readers` whose readers can read the logs anyway. +pub const PRIVATE_READERS: [&str; 3] = ["operator", "own_services", "local"]; + +/// The option of `logged_when` for a value shown as the output a person +/// asked for. +pub const SHOWN_NOT_LOGGED: &str = "output"; diff --git a/src/units/security/mod.rs b/src/units/security/mod.rs index 33befba..b3d0cc3 100644 --- a/src/units/security/mod.rs +++ b/src/units/security/mod.rs @@ -8,8 +8,8 @@ //! unit is judged on and `settle` holds the Choices that settle an undecided //! check. use super::{ - Asked, Block, Detail, FileContext, FilePlan, Planned, Presence, Questions, Settle, UnitPlan, - compact, identity, pack_runs, questions, unique_ids, + Asked, Block, Confirms, Detail, FileContext, FilePlan, Planned, Presence, Questions, Settle, + UnitPlan, compact, identity, pack_runs, questions, unique_ids, }; use crate::{ analysis::{errors::CreatedError, sites::Site, units::Unit}, @@ -126,6 +126,15 @@ fn push_unit( let checked = (rule == INJECTION) .then(|| confirm_checks(file, subject, id)) .flatten(); + let queried = (rule == INJECTION) + .then(|| confirm_query(file, subject, id)) + .flatten(); + let rendered = (rule == UNSAFE_SETTINGS) + .then(|| confirm_html(file, subject, id)) + .flatten(); + let readers = (rule == SENSITIVE_DATA) + .then(|| confirm_readers(file, subject, id)) + .flatten(); let logging = (rule == SENSITIVE_DATA) .then(|| confirm_logging(file, subject, id)) .flatten(); @@ -148,9 +157,14 @@ fn push_unit( }, trace: trace.map(Into::into), settles, - confirm: confirm.map(Into::into), - checked: checked.map(Into::into), - logging: logging.map(Into::into), + confirms: Box::new(Confirms { + values: confirm.map(Into::into), + checked: checked.map(Into::into), + queried: queried.map(Into::into), + rendered: rendered.map(Into::into), + readers: readers.map(Into::into), + logging: logging.map(Into::into), + }), django: subject.django, test_path: subject.test_path, }, @@ -196,17 +210,13 @@ fn send( if let Detail::Security { trace, settles, - confirm, - checked, - logging, + confirms, .. } = &mut unit.detail { *trace = None; settles.clear(); - *confirm = None; - *checked = None; - *logging = None; + **confirms = Confirms::default(); } } } @@ -610,6 +620,83 @@ fn confirm_checks( file.budget.fits(&request).then_some((request, asked)) } +/// What the values of an SQL, command or code finding hold where they enter +/// it, asked only after a finding whose one concern is one of those: the +/// function, the functions that call it and the project's types its +/// parameters name. +fn confirm_query( + file: &FileContext<'_>, + subject: &Subject<'_>, + id: &str, +) -> Option<(Value, Asked)> { + let code = subject.code(); + let types = !subject.types.is_empty(); + let mut questions = Questions::default(); + questions.ask( + "query_values".into(), + questions::query_values(&code, !subject.callers.is_empty(), types), + id, + INJECTION, + "query_values", + Pass::Locate, + ); + let mut state = with_callers(file, subject); + if types { + state["types_named_in_parameters"] = json!(subject.types); + } + let (request, asked) = file.request("locate", state, questions); + file.budget.fits(&request).then_some((request, asked)) +} + +/// What the HTML a weak-settings finding writes without escaping holds, +/// asked only after a finding its escaping check raised: the function alone. +fn confirm_html(file: &FileContext<'_>, subject: &Subject<'_>, id: &str) -> Option<(Value, Asked)> { + let code = subject.code(); + let mut questions = Questions::default(); + questions.ask( + "raw_html".into(), + questions::raw_html(&code), + id, + UNSAFE_SETTINGS, + "raw_html", + Pass::Locate, + ); + let state = json!({ + "file": file.file_state(), + subject.kind: subject.state(), + }); + let (request, asked) = file.request("locate", state, questions); + file.budget.fits(&request).then_some((request, asked)) +} + +/// Who reads the error text of an error-detail finding, asked only after +/// such a finding: the function and the opening of the project's README. +fn confirm_readers( + file: &FileContext<'_>, + subject: &Subject<'_>, + id: &str, +) -> Option<(Value, Asked)> { + let code = subject.code(); + let mut questions = Questions::default(); + questions.ask( + "error_readers".into(), + questions::error_readers(&code, file.project.is_some()), + id, + SENSITIVE_DATA, + "error_readers", + Pass::Locate, + ); + let mut state = json!({ + "file": file.file_state(), + subject.kind: subject.state(), + }); + if let Some(opening) = file.project { + state["project"] = json!({"readme_opening": opening}); + } + let (request, asked) = file.request("locate", state, questions); + file.budget.fits(&request).then_some((request, asked)) +} + /// When the log line of a logging finding runs, asked only after such a /// finding: the function alone. fn confirm_logging( diff --git a/src/units/tests/mod.rs b/src/units/tests/mod.rs index 9755cca..4c4fdfc 100644 --- a/src/units/tests/mod.rs +++ b/src/units/tests/mod.rs @@ -92,17 +92,47 @@ impl crate::transport::Evaluator for Scripted { } } -/// Replace every answer whose key ends with an override's suffix. +/// Replace every answer whose key ends with an override's suffix: the +/// longest suffix wins, so `markup_values` is not answered as `values`, and +/// a later override wins over an earlier one of the same length. fn apply(overrides: &[(&'static str, Value)], body: &mut Value) { - for (suffix, value) in overrides { - for (key, slot) in body["answers"].as_object_mut().unwrap() { - if key.ends_with(suffix) { - *slot = value.clone(); - } + for (key, slot) in body["answers"].as_object_mut().unwrap() { + if let Some((_, value)) = overrides + .iter() + .filter(|(suffix, _)| key.ends_with(suffix)) + .max_by_key(|(suffix, _)| suffix.len()) + { + *slot = value.clone(); } } } +/// Answers to the Choices asked after a security finding that keep it +/// standing: raw query values or markup, unescaped HTML another party wrote +/// and error text public readers see. Scripted answers otherwise pick `none` +/// or the first option, which clears these findings. +fn standing() -> Vec<(&'static str, Value)> { + let markup = ["encoded", "own", "raw", "typed", "unknown"]; + vec![ + ( + "query_values", + choice_of( + "raw", + &["allowed", "fixed", "own", "raw", "typed", "unknown"], + ), + ), + ("markup_values", choice_of("raw", &markup)), + ("raw_html", choice_of("raw", &markup)), + ( + "error_readers", + choice_of( + "public", + &["local", "operator", "own_services", "public", "unknown"], + ), + ), + ] +} + /// Rechecks carry more evidence: callees, enclosing functions or file source. fn is_recheck(request: &Value) -> bool { request["jevgate"]["stage"] == "recheck" @@ -314,7 +344,9 @@ fn hardcoded_file(path: &str, strength: &str, values: &[&str]) -> crate::schema: /// unit produces goes to a remote client, so the settle Choice clears nothing. fn run_with_nouls(project: &Project, options: &CheckArgs, nouls: &[(&'static str, f64)]) -> Report { let mut eval = scripted(0); - eval.overrides = nouls.iter().map(|&(q, p)| (q, noul_at(p))).collect(); + eval.overrides = standing(); + eval.overrides + .extend(nouls.iter().map(|&(q, p)| (q, noul_at(p)))); eval.overrides.push(to_client()); eval.overrides.push(logs_a_secret()); run(project, options, &mut eval) diff --git a/src/units/tests/security/confirms.rs b/src/units/tests/security/confirms.rs index 3fe6cad..94d57a2 100644 --- a/src/units/tests/security/confirms.rs +++ b/src/units/tests/security/confirms.rs @@ -16,10 +16,7 @@ fn a_path_finding_is_a_note_when_its_paths_stay_in_their_directory() { .units .iter() .find_map(|u| match &u.detail { - Detail::Security { - checked: Some(checked), - .. - } => Some(checked.request()), + Detail::Security { confirms, .. } => confirms.checked.as_ref().map(|c| c.request()), _ => None, }) .expect("an injection unit with a confirm of its checks"); @@ -74,7 +71,7 @@ fn a_log_line_an_operator_turns_on_to_log_tokens_is_a_note() { let logs = [ "plain", "identity", "operator", "secret", "personal", "none", ]; - let when = ["always", "debug", "none", "opt_in"]; + let when = ["always", "debug", "none", "opt_in", "output"]; let mut judged = |chosen: &str| { let mut eval = scripted(0); eval.overrides = vec![ @@ -179,3 +176,193 @@ pub(super) const MARKUP_VALUES: [&str; 5] = ["encoded", "own", "raw", "typed", " /// The options of the Choice on where a redirect finding's targets lead. pub(super) const REACH: [&str; 4] = ["anywhere", "checked", "none", "own_site"]; + +/// The options of the Choice on what an SQL, command or code finding's +/// values hold where they enter it. +pub(super) const QUERY_VALUES: [&str; 6] = ["allowed", "fixed", "own", "raw", "typed", "unknown"]; + +/// A route building `ORDER BY` from one of two clauses its preset selects. +pub(super) const PRESET: &str = "fn sorted(conn: &Connection, preset: &str) -> Result> {\n let order = match preset {\n \"speed\" => \"speed ASC\",\n \"rank\" => \"rank ASC\",\n _ => return Err(unknown()),\n };\n conn.query(&format!(\"SELECT id FROM models ORDER BY {order}\"))\n}\n"; + +#[test] +fn a_query_finding_is_a_note_when_its_values_are_fixed_text() { + let judged = |choice: Value| { + let (project, options) = security_project(PRESET); + let mut eval = scripted(0); + eval.overrides = vec![ + ("interpreted", noul_at(0.95)), + ("sql", noul_at(0.95)), + ("origin", spread(0.0, 0.0, 1.0)), + ("query_values", choice), + ]; + let report = run(&project, &options, &mut eval); + report.files[0] + .findings + .iter() + .find(|f| f.rule == "security/injection") + .map(|f| (f.strength, f.message.clone())) + .unwrap() + }; + assert_eq!(judged(choice_of("raw", &QUERY_VALUES)).0, Strength::Review); + let (strength, message) = judged(choice_of("fixed", &QUERY_VALUES)); + assert_eq!( + strength, + Strength::Note, + "one of two clauses written in the code" + ); + assert!(message.contains("cannot change the syntax"), "{message}"); +} + +#[test] +fn a_markup_consider_on_parameters_is_a_note_when_its_values_arrive_escaped() { + let judged = |choice: Value| { + let (project, options) = security_project(ENCODED); + let mut eval = scripted(0); + eval.overrides = vec![ + ("interpreted", noul_at(0.95)), + ("markup", noul_at(0.95)), + ("origin", spread(0.0, 1.0, 0.0)), + ("values", choice_of("outside", &VALUES)), + ("markup_values", choice), + ]; + let report = run(&project, &options, &mut eval); + report.files[0] + .findings + .iter() + .find(|f| f.rule == "security/injection") + .map(|f| f.strength) + .unwrap() + }; + assert_eq!(judged(choice_of("raw", &MARKUP_VALUES)), Strength::Consider); + assert_eq!( + judged(choice_of("encoded", &MARKUP_VALUES)), + Strength::Note, + "text another party wrote, escaped before it enters the markup" + ); +} + +/// A React component writing a syntax highlighter's output as raw HTML. +pub(super) const HIGHLIGHTED: &str = "export function CodeBlock({ code }) {\n const html = highlight(code)\n return \n}\n"; + +#[test] +fn unescaped_html_is_a_note_when_the_library_that_built_it_escaped_it() { + let markup = MARKUP_VALUES; + let judged = |choice: Value| { + let (report, _) = settings_run( + "code-block.jsx", + HIGHLIGHTED, + &[("weakened", 0.95), ("escape", 0.95)], + Some(("raw_html", choice)), + ); + report.files[0] + .findings + .iter() + .find(|f| f.rule == "security/unsafe-settings") + .map(|f| (f.strength, f.message.clone())) + .unwrap() + }; + assert_eq!(judged(choice_of("raw", &markup)).0, Strength::Review); + let (strength, message) = judged(choice_of("encoded", &markup)); + assert_eq!(strength, Strength::Note, "the highlighter escapes the code"); + assert!(message.contains("escaped or sanitized"), "{message}"); +} + +/// The options of the Choice on who reads a function's error text. +pub(super) const READERS: [&str; 5] = ["local", "operator", "own_services", "public", "unknown"]; + +#[test] +fn error_details_only_their_own_user_reads_are_a_note_asked_with_the_readme() { + let (project, mut options) = security_project(QUERY); + project.write( + "README.md", + "# Proxy\n\n[![CI](https://example.org/badge.svg)](https://example.org)\n\n\nA proxy you run on your own machine for your coding agent.\n", + ); + let boundary = |deny: &[&str]| { + let config = crate::config::Config { + upload_deny: deny.iter().map(|d| d.to_string()).collect(), + ..Default::default() + }; + crate::boundary::Boundary::new(&config).unwrap() + }; + assert_eq!( + crate::docs::project_opening(&project.0, &boundary(&["README.md"])), + None, + "a README the upload boundary denies is not sent" + ); + options.project = crate::docs::project_opening(&project.0, &boundary(&[])); + assert_eq!( + options.project.as_deref(), + Some("# Proxy\nA proxy you run on your own machine for your coding agent."), + "badges and HTML are left out" + ); + let (_, plan) = planned(&project, &options); + let readers = plan.files[&0] + .units + .iter() + .find_map(|u| match &u.detail { + Detail::Security { confirms, .. } => confirms.readers.as_ref().map(|c| c.request()), + _ => None, + }) + .expect("a sensitive-data unit asked who reads its errors"); + assert_eq!( + readers["state"]["project"]["readme_opening"], + json!(options.project) + ); + let mut judged = |chosen: &str| { + let mut eval = scripted(0); + eval.overrides = vec![ + ("error_details", noul_at(0.95)), + ("destination", to_client().1), + ("error_readers", choice_of(chosen, &READERS)), + ]; + let report = run(&project, &options, &mut eval); + options.refresh = true; + report.files[0] + .findings + .iter() + .find(|f| f.rule == "security/sensitive-data") + .map(|f| (f.strength, f.message.clone())) + .unwrap() + }; + assert_eq!(judged("public").0, Strength::Review); + let (strength, message) = judged("local"); + assert_eq!( + strength, + Strength::Note, + "the person running it reads its logs anyway" + ); + assert!( + message.contains("see the program's logs anyway"), + "{message}" + ); +} + +#[test] +fn a_value_shown_as_the_output_its_user_asked_for_is_no_logged_secret() { + let (project, options) = security_project(TOKENS); + let logs = [ + "plain", "identity", "operator", "secret", "personal", "none", + ]; + let when = ["always", "debug", "none", "opt_in", "output"]; + let mut eval = scripted(0); + eval.overrides = vec![ + ("logs_secret", noul_at(0.95)), + ("logs_object_secret", noul_at(0.95)), + ("logged", choice_of("secret", &logs)), + ("logged_when", choice_of("output", &when)), + ]; + let report = run(&project, &options, &mut eval); + let finding = report.files[0] + .findings + .iter() + .find(|f| f.rule == "security/sensitive-data") + .unwrap(); + assert_eq!(finding.strength, Strength::Note); + assert!( + finding + .message + .contains("shows the value only to the person who asked"), + "{}", + finding.message + ); +} diff --git a/src/units/tests/security/mod.rs b/src/units/tests/security/mod.rs index 2d5bd4f..cf6413c 100644 --- a/src/units/tests/security/mod.rs +++ b/src/units/tests/security/mod.rs @@ -65,14 +65,19 @@ fn clear_presence_needs_no_trace_and_clears_every_security_rule() { fn an_unhandled_value_from_another_party_is_a_located_injection_review() { let (project, options) = security_project(QUERY); let mut eval = scripted(0); - eval.overrides = vec![ + eval.overrides = standing(); + eval.overrides.extend([ ("interpreted", noul_at(0.95)), ("sql", noul_at(0.95)), ("origin", spread(0.0, 0.1, 0.9)), ("site", site("S1")), - ]; + ]); let report = run(&project, &options, &mut eval); - assert_eq!(eval.stages, ["first", "first"], "one trace, no recheck"); + assert_eq!( + eval.stages, + ["first", "first", "first"], + "one trace and what the query's values hold, no recheck" + ); let finding = &report.files[0].findings[0]; assert_eq!(finding.rule, "security/injection"); assert_eq!(finding.strength, Strength::Review); @@ -403,7 +408,9 @@ fn settled_status( settle: (&'static str, Value), ) -> (Status, u64) { let mut eval = scripted(0); - eval.overrides = nouls.iter().map(|&(q, p)| (q, noul_at(p))).collect(); + eval.overrides = standing(); + eval.overrides + .extend(nouls.iter().map(|&(q, p)| (q, noul_at(p)))); eval.overrides.push(("origin", spread(0.0, 0.9, 0.1))); eval.overrides .push(("values", choice_of("unknown", &VALUES))); diff --git a/src/units/tests/security/php.rs b/src/units/tests/security/php.rs index b321831..ea42ded 100644 --- a/src/units/tests/security/php.rs +++ b/src/units/tests/security/php.rs @@ -156,7 +156,9 @@ pub(super) fn php_page( let mut options = args(); options.rules = vec![catalog::INJECTION.into()]; let mut eval = scripted(0); - eval.overrides = nouls.iter().map(|(q, p)| (*q, noul_at(*p))).collect(); + eval.overrides = standing(); + eval.overrides + .extend(nouls.iter().map(|(q, p)| (*q, noul_at(*p)))); eval.overrides.push(("origin", origin)); eval.overrides.extend(settles); let report = run(&project, &options, &mut eval); diff --git a/src/units/tests/security/settings.rs b/src/units/tests/security/settings.rs index 83001f8..791c582 100644 --- a/src/units/tests/security/settings.rs +++ b/src/units/tests/security/settings.rs @@ -83,7 +83,10 @@ impl crate::transport::Evaluator for Recording { pub(super) fn recording(nouls: &[(&'static str, f64)]) -> Recording { let mut inner = scripted(0); - inner.overrides = nouls.iter().map(|&(q, p)| (q, noul_at(p))).collect(); + inner.overrides = standing(); + inner + .overrides + .extend(nouls.iter().map(|&(q, p)| (q, noul_at(p)))); Recording { inner, requests: Vec::new(), diff --git a/src/units/wording/security.rs b/src/units/wording/security.rs index a9f1480..db3171c 100644 --- a/src/units/wording/security.rs +++ b/src/units/wording/security.rs @@ -437,7 +437,8 @@ fn script_wording( /// A note whose confirm Choice found values that can do no harm where they /// go: a path that stays in its directory, markup values already escaped or -/// encoded, a redirect that stays on the site. +/// encoded, a redirect that stays on the site, query or command text that is +/// fixed, typed or the program's own. fn confirmed_wording(kind: &str, subject: &str, noun: &str) -> Option { Some(match kind { "path" => ( @@ -448,10 +449,16 @@ fn confirmed_wording(kind: &str, subject: &str, noun: &str) -> Option { ), "markup" => ( format!( - "{subject} places a variable into {noun}, but it was likely escaped or encoded before, so it cannot open a tag or attribute." + "{subject} places a variable into {noun}, but it likely holds text escaped or encoded before, typed values or the program's own markup, so it cannot open a tag or attribute." ), "Optional: confirm the value is escaped on every path that reaches the markup", ), + "sql" | "shell" | "code" => ( + format!( + "{subject} joins a variable into {noun}, but it likely holds only fixed text, a typed value or the program's own names, so it cannot change the syntax." + ), + "Optional: pass the value as a bound parameter or argument anyway", + ), "redirect" => ( format!( "{subject} redirects clients to a target built from a variable, but a fixed path or check likely keeps it on the site." @@ -524,6 +531,46 @@ fn exposure_wording( ); let decided = crate::policy::probability_at_least(p, crate::policy::REVIEW_PROBABILITY); let opted_in = category.starts_with("CWE-532") && crate::units::outcome::opted_in(&get); + if strength == Strength::Note + && category.starts_with("CWE-532") + && !opted_in + && crate::units::outcome::not_logged(&get) + { + return ( + ( + format!( + "{subject} likely shows the value only to the person who asked for it, as the output of their command or page, rather than writing it to a log." + ), + "Optional: confirm the value never reaches a log file", + ), + category.to_string(), + ); + } + if strength == Strength::Note + && category.starts_with("CWE-209") + && crate::units::outcome::private_readers(&get) + { + return ( + ( + format!( + "{subject} sends internal error details, but likely only to readers who can see the program's logs anyway: the person running it, its operator or the project's own services." + ), + "Optional: return a generic message if other people can reach this code", + ), + category.to_string(), + ); + } + if strength == Strength::Note && crate::units::outcome::inert_html(&get) { + return ( + ( + format!( + "{subject} writes HTML without escaping it, but the markup likely comes escaped or sanitized from the library that built it, or ships with the program." + ), + "Optional: confirm no path writes text another party wrote without escaping it", + ), + category.to_string(), + ); + } if strength == Strength::Note && opted_in { return ( ( From 96fcd5aad3045236f3cd411fbb094d4c424ebca9 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:18:58 -0300 Subject: [PATCH 07/12] Ask where a URL finding's URLs come from whenever the URL check is not clear The settle Choice on a URL's parts was asked only while the check stayed undecided. paperclip's cloud route, a fixed path on its configured origin with the user's id in a header, was a review at 0.81 and puts 0.88 on its own host. On the corpus no review or consider changed (every labeled right finding asked it put at most 0.56 there) and 47 notes on URLs of fixed or configured hosts cleared, for about $0.02. --- CHANGELOG.md | 1 + src/units/questions/settle.rs | 10 +++++++--- src/units/security/settle.rs | 2 +- src/units/tests/nextjs.rs | 4 ++++ src/units/tests/security/mod.rs | 26 ++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df3bfb3..c320bd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Fixes from running JevGate on nine widely used projects under daily development - Injection: an SQL, command or code finding is asked what its values can hold where they enter the text; fixed clauses a key selects (freellmapi's `ORDER BY` from a map of presets), parsed ids (multica's option UUIDs), the program's own names or a query the sender may run anyway make it a note. The 23 such corpus findings labeled right put at most 0.12 on those, one labeled wrong 0.82. - Injection: a markup consider on the function's parameters is also asked what its values hold where they enter the markup, as markup reviews already were; text escaped before (a syntax highlighter's output), typed values or the program's own markup make it a note. 11 labeled wrong and 1 labeled right became notes; the other 44 labeled right put at most 0.47 there. - Unsafe settings: a finding the escaping check raised is asked what the unescaped HTML holds; markup a library built from escaped text, or that ships with the program (freellmapi's highlighted code, cc-switch's bundled provider icons, multica's KaTeX output), makes it a note. The three corpus findings labeled right put at most 0.18 there. + - Injection: where a finding's URLs come from, asked before only while the URL check was undecided, is asked whenever it is not clear; a host written in the code or configuration, with only ids in the path or query, clears it at 0.80. paperclip's cloud route, a fixed path on its configured origin, is no longer a review; on the corpus no review or consider changed and 47 notes on URLs of fixed or configured hosts cleared. - Sensitive data: a logging finding whose value is the output the person asked for, such as rtk's `env` command or a CLI printing a new token, is a note (two labeled wrong at 0.96 and 0.98, twelve labeled right at most 0.43). An error-detail finding is asked who reads the error text, with the opening of the root README: when, at 0.80, only the operator, the project's own services or the person running it on their own machine read it, it is a note. That cleared headroom's 24 local-proxy reviews and 6 of multica's daemon endpoints, while multica's handlers for its hosted service's users stay reviews; on the corpus it cleared 2 labeled wrong or debatable and none of the 51 labeled right, which put at most 0.46 there. The README opening is sent only with this question, bounded by the upload patterns. ## [0.24.1] - 2026-09-27 diff --git a/src/units/questions/settle.rs b/src/units/questions/settle.rs index db0c6c9..c2d9655 100644 --- a/src/units/questions/settle.rs +++ b/src/units/questions/settle.rs @@ -50,11 +50,15 @@ pub fn security_path_source(code: &str, callers: bool) -> Value { /// program's own, or no request. pub const OWN_PARTS: [&str; 2] = ["own", "none"]; -/// Where the URLs a function requests come from, asked when the URL check -/// stays undecided: on clients of a fixed or configured service the check +/// Where the URLs a function requests come from, asked whenever the URL +/// check is not clear: on clients of a fixed or configured service the check /// split on a variable path or query, while naming the host decided them. A /// host that is sent another URL to fetch is its own option, since internal -/// proxies fetched what users sent. The same question about paths once +/// proxies fetched what users sent. paperclip's cloud route requests a fixed +/// path on its configured origin, with the user's id in a header, and was a +/// review at 0.81; it put 0.88 on its own host. Every labeled corpus finding +/// right that was asked it put at most 0.56 there, and no review or consider +/// changed. The same question about paths once /// cleared real traversals, reading names stored in an index as the /// program's own; `security_path_source` names such records as another /// party's input. diff --git a/src/units/security/settle.rs b/src/units/security/settle.rs index 73a5bfd..85322fe 100644 --- a/src/units/security/settle.rs +++ b/src/units/security/settle.rs @@ -92,7 +92,7 @@ pub(in crate::units) const SETTLES: [SettleKind; 14] = [ checks: &["url"], clears: &questions::OWN_PARTS, callers: true, - when: SettleWhen::Undecided, + when: SettleWhen::NotClear, files: SettleFiles::All, }, SettleKind { diff --git a/src/units/tests/nextjs.rs b/src/units/tests/nextjs.rs index 9497a69..4433b25 100644 --- a/src/units/tests/nextjs.rs +++ b/src/units/tests/nextjs.rs @@ -246,6 +246,10 @@ fn a_url_note_clears_when_the_request_leaves_from_the_browser() { "runs_in", choice_of(runs_in, &["browser", "server", "either"]), ), + ( + "url_parts", + choice_of("given", &["own", "forwards", "given", "outside", "none"]), + ), ]; let report = run(&project, options, &mut eval); let file = report diff --git a/src/units/tests/security/mod.rs b/src/units/tests/security/mod.rs index cf6413c..74c67d3 100644 --- a/src/units/tests/security/mod.rs +++ b/src/units/tests/security/mod.rs @@ -211,6 +211,7 @@ fn a_parameter_in_a_path_or_url_is_a_note_until_callers_show_another_party() { ("resource", noul_at(0.95)), ("url", noul_at(0.95)), ("origin", spread(0.0, 0.9, 0.1)), + ("url_parts", choice_of("given", &URL_PARTS)), ]; let finding = &first_finding(&project, &options, &mut eval); assert_eq!(finding.strength, Strength::Note); @@ -220,6 +221,31 @@ fn a_parameter_in_a_path_or_url_is_a_note_until_callers_show_another_party() { ); } +#[test] +fn a_decided_url_finding_on_the_programs_own_host_is_clear() { + let (project, mut options) = security_project(FETCH_QUOTE); + let mut status = |parts: &str| { + let mut eval = scripted(0); + eval.overrides = vec![ + ("resource", noul_at(0.95)), + ("url", noul_at(0.95)), + ("origin", spread(0.0, 0.0, 1.0)), + ("url_parts", choice_of(parts, &URL_PARTS)), + ]; + let report = run(&project, &options, &mut eval); + options.refresh = true; + report.files[0].dimensions[catalog::INJECTION] + .status + .clone() + }; + assert_eq!(status("outside"), Status::Review); + assert_eq!( + status("own"), + Status::Clear, + "a configured base URL with an id in its path" + ); +} + const FETCH_QUOTE: &str = "fn quote(client: &Client, base: &Url, symbol: &str) -> String {\n let url = base.join(&format!(\"quotes/{symbol}\")).unwrap();\n client.get(url).send().unwrap().text().unwrap()\n}\n"; const URL_PARTS: [&str; 5] = ["own", "forwards", "given", "outside", "none"]; From 5f22584d3a180fac1b683973a466c8371995f6b8 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:28:26 -0300 Subject: [PATCH 08/12] Leave out one language's copy of the docs under an i18n directory OmniRoute keeps its docs in 30 languages under docs/i18n//; 543 of its 547 staleness considers repeated an original's finding in a translation. Documents under i18n, l10n, locales or translations followed by a language code are left out like per-release copies. No corpus finding changes. --- CHANGELOG.md | 2 +- src/docs/discover.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c320bd6..d7ae695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Fixes from running JevGate on nine widely used projects under daily development - Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. - Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. -- Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. No corpus finding changes. +- Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. So are one language's copy of the docs under `i18n`, `l10n`, `locales` or `translations` (`docs/i18n/ja/`, Docusaurus's `i18n/zh-Hans/`): translations whose stale links are the original's. OmniRoute keeps its docs in 30 languages, and 543 of its 547 staleness considers repeated an original's finding in a translation. No corpus finding changes. - Duplication: two documents whose paths name different locales (`docs/en` and `docs/zh-cn`, `README.md` and `README_zh.md`) or whose prose is in different scripts are translations; their sections are asked only whether they disagree, as when the translation question says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition, the translation question answering 0.04 to 0.71 on them. No corpus finding pairs two languages. - Security findings on the hot projects were mostly wrong in the same few ways, and each is now asked the one thing that decided it, only after the finding. On the corpus's 55 projects with labeled security findings, 15 findings labeled wrong and 1 debatable are notes, for 1 labeled right; nothing changed on the held-out projects. The run asked about $0.02 of new questions. - Injection: an SQL, command or code finding is asked what its values can hold where they enter the text; fixed clauses a key selects (freellmapi's `ORDER BY` from a map of presets), parsed ids (multica's option UUIDs), the program's own names or a query the sender may run anyway make it a note. The 23 such corpus findings labeled right put at most 0.12 on those, one labeled wrong 0.82. diff --git a/src/docs/discover.rs b/src/docs/discover.rs index e5ef292..019002b 100644 --- a/src/docs/discover.rs +++ b/src/docs/discover.rs @@ -119,7 +119,7 @@ fn project_doc(path: &Path) -> bool { | "versioned_docs" ) || release_dir(d) - }); + }) || translation_dir(&dirs); let documentation = dirs.is_empty() || stem.starts_with("readme") || stem.starts_with("contributing") @@ -132,6 +132,27 @@ fn project_doc(path: &Path) -> bool { && !RECORD_STEMS.contains(&stem.as_str()) } +/// Whether a document sits in one language's copy of the docs, such as +/// `docs/i18n/am/` or Docusaurus's `i18n/zh-hans/`: a translation, whose +/// stale links and repetition are the original's. OmniRoute keeps its docs +/// in 30 languages, and 543 of its 547 staleness considers repeated an +/// original's finding in a translation. +fn translation_dir(dirs: &[String]) -> bool { + dirs.windows(2).any(|w| { + matches!(w[0].as_str(), "i18n" | "l10n" | "locales" | "translations") && locale_code(&w[1]) + }) +} + +/// A language code, optionally with a script or region, such as `ja`, +/// `zh-hans` or `pt_br`. +fn locale_code(dir: &str) -> bool { + let (language, region) = dir.split_once(['-', '_']).unwrap_or((dir, "")); + (2..=3).contains(&language.len()) + && language.chars().all(|c| c.is_ascii_lowercase()) + && (region.is_empty() + || (2..=4).contains(®ion.len()) && region.chars().all(|c| c.is_ascii_alphanumeric())) +} + /// A directory holding the docs of one release, such as `docs/versions/0.7.5` /// or `v1.2`: a frozen copy of the current docs, not a second source. fn release_dir(dir: &str) -> bool { @@ -353,6 +374,13 @@ mod tests { ("website/versioned_docs/version-2/intro.md", false), ("docs/v2/guide.md", true), ("docs/next/README.md", true), + ("docs/i18n/am/CONTRIBUTING.md", false), + ("docs/i18n/uk-UA/docs/guide.md", false), + ( + "website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/intro.md", + false, + ), + ("docs/i18n/README.md", true), ] { assert_eq!(project_doc(Path::new(path)), expected, "{path}"); } From 2cb24d83986f2fdd6b39ab605a5c955dac8c4e4a Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:32:35 -0300 Subject: [PATCH 09/12] Read a module path without its extension as the file it names dify's web/docs/test.md tells readers to import from web/test/i18n-mock, which is web/test/i18n-mock.ts; it was reported as missing. No corpus finding changes. --- CHANGELOG.md | 2 +- src/docs/references.rs | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7ae695..844487e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Notable changes to JevGate. Versions follow [Semantic Versioning](https://semver Fixes from running JevGate on nine widely used projects under daily development (rtk, headroom, paperclip, hermes-agent, cc-switch, freellmapi, herdr, multica, OmniRoute). - Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. -- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. +- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. So does a module path without its extension, such as dify's `web/test/i18n-mock` for `i18n-mock.ts`. No corpus finding changes. - Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. So are one language's copy of the docs under `i18n`, `l10n`, `locales` or `translations` (`docs/i18n/ja/`, Docusaurus's `i18n/zh-Hans/`): translations whose stale links are the original's. OmniRoute keeps its docs in 30 languages, and 543 of its 547 staleness considers repeated an original's finding in a translation. No corpus finding changes. - Duplication: two documents whose paths name different locales (`docs/en` and `docs/zh-cn`, `README.md` and `README_zh.md`) or whose prose is in different scripts are translations; their sections are asked only whether they disagree, as when the translation question says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition, the translation question answering 0.04 to 0.71 on them. No corpus finding pairs two languages. - Security findings on the hot projects were mostly wrong in the same few ways, and each is now asked the one thing that decided it, only after the finding. On the corpus's 55 projects with labeled security findings, 15 findings labeled wrong and 1 debatable are notes, for 1 labeled right; nothing changed on the held-out projects. The run asked about $0.02 of new questions. diff --git a/src/docs/references.rs b/src/docs/references.rs index 91e0d9d..ad768aa 100644 --- a/src/docs/references.rs +++ b/src/docs/references.rs @@ -290,11 +290,24 @@ fn present(root: &Path, base: &Path, name: &str, history: &History) -> bool { { return true; } - // A partial path such as `services/quota.ts` names a deeper file. + // A partial path such as `services/quota.ts` names a deeper file, and a + // module path without its extension, such as `web/test/i18n-mock` in an + // import, names `i18n-mock.ts`. let suffix = format!("/{trimmed}"); + let module = Path::new(trimmed).extension().is_none(); + let wanted: Vec = candidates + .iter() + .map(|c| c.to_string_lossy().into_owned()) + .collect(); history.tracked.iter().any(|p| { let p = p.to_string_lossy(); - p.ends_with(&suffix) || p.starts_with(&format!("{trimmed}/")) + let stem = p + .rsplit_once('.') + .filter(|(_, e)| !e.contains('/')) + .map(|(s, _)| s); + p.ends_with(&suffix) + || p.starts_with(&format!("{trimmed}/")) + || module && stem.is_some_and(|s| wanted.iter().any(|w| w == s) || s.ends_with(&suffix)) }) } @@ -635,6 +648,15 @@ mod tests { assert_eq!(names, ["conf/app.json", "data/seed.json"]); } + #[test] + fn a_module_path_without_its_extension_names_its_file() { + let names: Vec = found("Mock it with `src/services/quota` or `src/services/gone`.") + .into_iter() + .map(|(n, _)| n) + .collect(); + assert_eq!(names, ["src/services/gone"]); + } + #[test] fn a_code_span_with_an_anchor_names_its_file() { let names: Vec = found("See `docs/guide.md#setup` and `docs/gone.md#usage`.") From 54e2f88340e566e31286a16761f80579474ab437 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:35:07 -0300 Subject: [PATCH 10/12] Check make and just targets only where a Makefile or justfile is tracked openclaw documents `make routing-isolation` from a separate models repository and has no Makefile; the target was reported as undeclared. No corpus finding changes. --- CHANGELOG.md | 2 +- src/docs/references.rs | 58 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 844487e..14058c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Notable changes to JevGate. Versions follow [Semantic Versioning](https://semver Fixes from running JevGate on nine widely used projects under daily development (rtk, headroom, paperclip, hermes-agent, cc-switch, freellmapi, herdr, multica, OmniRoute). - Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. -- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. So does a module path without its extension, such as dify's `web/test/i18n-mock` for `i18n-mock.ts`. No corpus finding changes. +- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. So does a module path without its extension, such as dify's `web/test/i18n-mock` for `i18n-mock.ts`. A `make` or `just` target is checked only when the repository tracks a Makefile or justfile: openclaw documents `make routing-isolation` from a separate models repository. No corpus finding changes. - Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. So are one language's copy of the docs under `i18n`, `l10n`, `locales` or `translations` (`docs/i18n/ja/`, Docusaurus's `i18n/zh-Hans/`): translations whose stale links are the original's. OmniRoute keeps its docs in 30 languages, and 543 of its 547 staleness considers repeated an original's finding in a translation. No corpus finding changes. - Duplication: two documents whose paths name different locales (`docs/en` and `docs/zh-cn`, `README.md` and `README_zh.md`) or whose prose is in different scripts are translations; their sections are asked only whether they disagree, as when the translation question says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition, the translation question answering 0.04 to 0.71 on them. No corpus finding pairs two languages. - Security findings on the hot projects were mostly wrong in the same few ways, and each is now asked the one thing that decided it, only after the finding. On the corpus's 55 projects with labeled security findings, 15 findings labeled wrong and 1 debatable are notes, for 1 labeled right; nothing changed on the held-out projects. The run asked about $0.02 of new questions. diff --git a/src/docs/references.rs b/src/docs/references.rs index ad768aa..c66d9cd 100644 --- a/src/docs/references.rs +++ b/src/docs/references.rs @@ -118,7 +118,18 @@ pub fn missing( } } if !scripts.is_empty() { - for script in commands(text) { + let tracks = |names: &[&str]| { + history.tracked.iter().any(|p| { + p.file_name().and_then(|n| n.to_str()).is_some_and(|n| { + names.contains(&n) || n.ends_with(".mk") && names.contains(&"*.mk") + }) + }) + }; + let runners = Runners { + make: tracks(&["Makefile", "makefile", "GNUmakefile", "*.mk"]), + just: tracks(&["justfile", "Justfile", ".justfile"]), + }; + for script in commands(text, runners) { if !scripts.contains(&script) && found.insert(format!("script:{script}")) { out.push(Missing { name: script, @@ -504,7 +515,17 @@ fn link_targets(text: &str) -> Vec { /// starts: at the start of a code line or an inline code span, after a /// prompt, or after `&&`, `||`, `;` or `|`. The same word inside a comment /// or a sentence, such as "make sure", is not a command. -fn commands(text: &str) -> Vec { +/// The build tools whose targets the repository can declare: `make` and +/// `just` name a target only when it tracks a Makefile or justfile. +/// openclaw documents `make routing-isolation` from a separate models +/// repository and has no Makefile. +#[derive(Clone, Copy)] +struct Runners { + make: bool, + just: bool, +} + +fn commands(text: &str, runners: Runners) -> Vec { let (prose, code) = split_fences(text); let spans = prose.into_iter().flat_map(line_spans).map(|(_, span)| span); let mut out = Vec::new(); @@ -518,7 +539,9 @@ fn commands(text: &str) -> Vec { let next = |n: usize| words.get(n).copied().unwrap_or(""); let script = match next(0) { "pnpm" | "yarn" | "npm" if next(1) == "run" => next(2), - "pnpm" | "yarn" | "make" | "just" => next(1), + "pnpm" | "yarn" => next(1), + "make" if runners.make => next(1), + "just" if runners.just => next(1), _ => continue, }; let name_like = script @@ -550,13 +573,24 @@ mod tests { /// The missing names of `text`, a section of `docs/intro.md`, in a /// repository tracking a few files and with two removed. fn found(text: &str) -> Vec<(String, Fate)> { + found_in( + &[ + "src/app.ts", + "docs/guide.md", + "src/services/quota.ts", + "Makefile", + "justfile", + ], + text, + ) + } + + /// The missing names of `text` in a repository tracking `tracked`. + fn found_in(tracked: &[&str], text: &str) -> Vec<(String, Fate)> { let project = crate::tests::Project::new(); project.write("src/app.ts", ""); let history = History { - tracked: ["src/app.ts", "docs/guide.md", "src/services/quota.ts"] - .into_iter() - .map(PathBuf::from) - .collect(), + tracked: tracked.iter().map(PathBuf::from).collect(), tags: BTreeSet::new(), removed: [ (PathBuf::from("src/old.ts"), None), @@ -629,6 +663,16 @@ mod tests { assert_eq!(found(text), [("lint".to_string(), Fate::NoScript)]); } + #[test] + fn make_targets_are_checked_only_where_a_makefile_is_tracked() { + let text = "Run `make routing-isolation` from the models repository."; + assert_eq!(found_in(&["src/app.ts"], text), []); + assert_eq!( + found_in(&["src/app.ts", "build/rules.mk"], text), + [("routing-isolation".to_string(), Fate::NoScript)] + ); + } + #[test] fn scripts_are_read_where_a_command_starts() { let text = "```sh\n$ CI=1 pnpm dev && make release\n# make sure the server runs\nprint('just not yet')\n```\nUse `yarn` or `pnpm` with `just check-all`, or the ``python\nyourapp.py`` will not work. Let's just say."; From f2e6e76623dc72ff3d68bc12937440cd99ae4d62 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:49:16 -0300 Subject: [PATCH 11/12] Count a package's own binaries as scripts a document may run n8n's i18n docs run `pnpm n8n-generate-translations`, a bin of its core package; it was reported as a script no manifest declares. No corpus finding changes. --- CHANGELOG.md | 2 +- src/docs/project.rs | 51 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14058c2..0f43b84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Notable changes to JevGate. Versions follow [Semantic Versioning](https://semver Fixes from running JevGate on nine widely used projects under daily development (rtk, headroom, paperclip, hermes-agent, cc-switch, freellmapi, herdr, multica, OmniRoute). - Two crashes on text outside ASCII: a colon right after non-ASCII text in documentation (`已移除:` before a code span) and a Python test ending in a multi-byte character. Both stopped the whole run, on herdr, hermes-agent and OmniRoute. -- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. So does a module path without its extension, such as dify's `web/test/i18n-mock` for `i18n-mock.ts`. A `make` or `just` target is checked only when the repository tracks a Makefile or justfile: openclaw documents `make routing-isolation` from a separate models repository. No corpus finding changes. +- Staleness: a path with an anchor in a code span, such as `docs/en/env/01-variables.md#idempotency`, names its file; it was reported as missing although the file and its heading exist. So does a module path without its extension, such as dify's `web/test/i18n-mock` for `i18n-mock.ts`. A `make` or `just` target is checked only when the repository tracks a Makefile or justfile: openclaw documents `make routing-isolation` from a separate models repository. A package's own binaries (`bin` in `package.json`) count as scripts `pnpm` can run, such as n8n's `n8n-generate-translations`. No corpus finding changes. - Documentation: the docs of one release, in a directory named like a version (`docs/versions/0.7.5`, `v1.2`) or under `versioned_docs`, are left out as frozen copies. herdr keeps its website docs per release, and 137 of its 147 documentation considers named a section of such a copy. So are one language's copy of the docs under `i18n`, `l10n`, `locales` or `translations` (`docs/i18n/ja/`, Docusaurus's `i18n/zh-Hans/`): translations whose stale links are the original's. OmniRoute keeps its docs in 30 languages, and 543 of its 547 staleness considers repeated an original's finding in a translation. No corpus finding changes. - Duplication: two documents whose paths name different locales (`docs/en` and `docs/zh-cn`, `README.md` and `README_zh.md`) or whose prose is in different scripts are translations; their sections are asked only whether they disagree, as when the translation question says so. freellmapi, cc-switch and rtk had 12 translated pairs reported as repetition, the translation question answering 0.04 to 0.71 on them. No corpus finding pairs two languages. - Security findings on the hot projects were mostly wrong in the same few ways, and each is now asked the one thing that decided it, only after the finding. On the corpus's 55 projects with labeled security findings, 15 findings labeled wrong and 1 debatable are notes, for 1 labeled right; nothing changed on the held-out projects. The run asked about $0.02 of new questions. diff --git a/src/docs/project.rs b/src/docs/project.rs index c3a8cf1..72f870e 100644 --- a/src/docs/project.rs +++ b/src/docs/project.rs @@ -209,8 +209,8 @@ const DEPENDENCY_FIELDS: &[&str] = &[ "optionalDependencies", ]; -/// Scripts and dependencies of every tracked `package.json`, and targets of -/// every tracked Makefile or justfile. +/// Scripts, binaries and dependencies of every tracked `package.json`, and +/// targets of every tracked Makefile or justfile. pub fn scripts(root: &Path, history: &super::history::History) -> BTreeSet { let mut scripts = BTreeSet::new(); for path in &history.tracked { @@ -226,6 +226,17 @@ pub fn scripts(root: &Path, history: &super::history::History) -> BTreeSet scripts.extend(bins.keys().cloned()), + Value::String(_) => { + if let Some(name) = package["name"].as_str() { + scripts.insert(name.rsplit('/').next().unwrap_or(name).to_string()); + } + } + _ => {} + } // `pnpm tsx` and `yarn eslint` run a dependency's binary, // usually named after its package. for field in DEPENDENCY_FIELDS { @@ -393,21 +404,45 @@ mod tests { } #[test] - fn scripts_include_the_binaries_dependencies_bring() { + fn scripts_include_the_binaries_packages_declare_or_dependencies_bring() { let project = crate::tests::Project::new(); project.write( "examples/app/package.json", r#"{"scripts":{"dev":"vite"},"devDependencies":{"tsx":"4","@biomejs/biome":"1"}}"#, ); project.write("justfile", "check:\n\tcargo check\n"); + project.write( + "packages/core/package.json", + r#"{"name":"n8n-core","bin":{"n8n-generate-translations":"./bin/generate-translations"}}"#, + ); + project.write( + "packages/cli/package.json", + r#"{"name":"@acme/tool","bin":"./cli.js"}"#, + ); let history = crate::docs::history::History { - tracked: ["examples/app/package.json", "justfile"] - .into_iter() - .map(PathBuf::from) - .collect(), + tracked: [ + "examples/app/package.json", + "justfile", + "packages/core/package.json", + "packages/cli/package.json", + ] + .into_iter() + .map(PathBuf::from) + .collect(), ..Default::default() }; let found: Vec = scripts(&project.0, &history).into_iter().collect(); - assert_eq!(found, ["@biomejs/biome", "biome", "check", "dev", "tsx"]); + assert_eq!( + found, + [ + "@biomejs/biome", + "biome", + "check", + "dev", + "n8n-generate-translations", + "tool", + "tsx" + ] + ); } } From bc108c93b037d171eba7d4135247305c0bd07c13 Mon Sep 17 00:00:00 2001 From: Tauan BF <11513929+tauanbinato@users.noreply.github.com> Date: Sun, 27 Sep 2026 23:07:22 -0300 Subject: [PATCH 12/12] Compare extensionless module paths with forward slashes The candidate paths are built with the platform's separator, so on Windows `src\services\quota` never matched the tracked `src/services/quota.ts`. --- src/docs/references.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/docs/references.rs b/src/docs/references.rs index c66d9cd..6e476ae 100644 --- a/src/docs/references.rs +++ b/src/docs/references.rs @@ -306,13 +306,19 @@ fn present(root: &Path, base: &Path, name: &str, history: &History) -> bool { // import, names `i18n-mock.ts`. let suffix = format!("/{trimmed}"); let module = Path::new(trimmed).extension().is_none(); - let wanted: Vec = candidates - .iter() - .map(|c| c.to_string_lossy().into_owned()) - .collect(); + // Compared with forward slashes: `normal` joins with the platform's + // separator, while tracked paths keep Git's. + let slashed = |p: &Path| { + p.iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/") + }; + let wanted: Vec = candidates.iter().map(|c| slashed(c)).collect(); history.tracked.iter().any(|p| { + let whole = slashed(p); let p = p.to_string_lossy(); - let stem = p + let stem = whole .rsplit_once('.') .filter(|(_, e)| !e.contains('/')) .map(|(s, _)| s);