diff --git a/Justfile b/Justfile index 75dc75c..f4f251e 100644 --- a/Justfile +++ b/Justfile @@ -30,8 +30,12 @@ build-lazy-eliminator: build-efficacy: cd vexometer-efficacy && cargo build --release +# Build the verbosity compressor (Rust satellite) +build-verbosity-compressor: + cd verbosity-compressor && just build + # Build all components -build-all: build-vexometer build-vext build-lazy-eliminator build-efficacy +build-all: build-vexometer build-vext build-lazy-eliminator build-efficacy build-verbosity-compressor # Run vexometer tests test-vexometer: @@ -53,13 +57,17 @@ test-lazy-eliminator: test-efficacy: cd vexometer-efficacy && (cargo test --offline || cargo test) +# Run verbosity-compressor tests +test-verbosity-compressor: + cd verbosity-compressor && just test + # vext-email-gateway status check test-vext-email-gateway: @echo "vext-email-gateway is currently prototype-stage and not part of the required test-all gate." @echo "See vext-email-gateway/README.adoc and ROADMAP.adoc for current wiring status." # Run all tests -test-all: test-vexometer test-vext test-lazy-eliminator test-efficacy +test-all: test-vexometer test-vext test-lazy-eliminator test-efficacy test-verbosity-compressor # Evaluate a satellite run and emit a vexometer-efficacy-v2 report efficacy-report *ARGS: @@ -84,6 +92,7 @@ bench-all: bench-vexometer clean: cd vext && cargo clean cd vexometer-efficacy && cargo clean + cd verbosity-compressor && cargo clean cd vexometer && just clean || true cd lazy-eliminator && just clean || true @@ -91,11 +100,13 @@ clean: fmt-check: cd vext && cargo fmt -- --check cd vexometer-efficacy && cargo fmt -- --check + cd verbosity-compressor && cargo fmt -- --check # Run clippy on Rust components lint: cd vext && cargo clippy -- -D warnings cd vexometer-efficacy && cargo clippy --all-targets -- -D warnings + cd verbosity-compressor && cargo clippy --all-targets -- -D warnings # Run contractiles Mustfile invariants across all components must-all: diff --git a/README.adoc b/README.adoc index 79d26d1..96f6b14 100644 --- a/README.adoc +++ b/README.adoc @@ -44,6 +44,9 @@ NOTE: This repository was previously named `vex-tools`. The canonical name is no | `lazy-eliminator/` | **Lazy Eliminator** -- Completeness enforcement for LLM-generated code. Detects and eliminates lazy patterns. +| `verbosity-compressor/` +| **Verbosity Compressor (vex-verbosity-compressor)** -- Information density optimisation for LLM output. Semantic redundancy detection, filler-phrase stripping. First satellite scaffolded from `satellite-template`. + | `vext/` | **Vext** -- Verifiable, Extensible Communications Protocol with cryptographic proof of algorithmic neutrality. diff --git a/lazy-eliminator/.trust/trust-manifest.sha256 b/lazy-eliminator/.trust/trust-manifest.sha256 index f5be2b9..3171d13 100644 --- a/lazy-eliminator/.trust/trust-manifest.sha256 +++ b/lazy-eliminator/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=lazy-eliminator -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z 339d25795fa89149354d4101c533492f0e5bbe39fc953ac02be17a225f2fd270 README.adoc 6772e621da4e50257728886f568bd652b8e9c58a02aec325112e57def73da46a ROADMAP.adoc 504199ed09a9acbd183fe9c37a8330ec254f33f5612e0f36f2f5e7be170a57ff SECURITY.adoc 530ba7ca220c8eaee794a163672469e9733dd8cb97c5bd6311d5ca15a3685e99 contractiles/must/Mustfile 7160652fb94fa8d7eab9e4f4824f05c09d94ab9c53976929537d8811d339a3a0 contractiles/trust/Trustfile.a2ml +498487f3a2ccd1a4cf23404ddb0811c8407bf9684acc71ebb7d2722a46a73acc contractiles/dust/Dustfile 50540c1ae91c38110cdf41195578865cdef278838be38f11d2fa108ecc1d60e7 RSR_OUTLINE.adoc 2eb1dac8104669453d9cd1b2df7faf2ba20f6ca184300b4a936d560622471621 docs/CITATIONS.adoc diff --git a/satellite-template/.trust/trust-manifest.sha256 b/satellite-template/.trust/trust-manifest.sha256 index 4b81e38..ab47cee 100644 --- a/satellite-template/.trust/trust-manifest.sha256 +++ b/satellite-template/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=satellite-template -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z e90437cd512f3ac6824b42e733394bc69dfa4bcab046d779842bc876e11f870e README.adoc 5a62f5611eecfafa43d931b4d7e0917fa1b0f0275fdfd74c61415ab635e72805 ROADMAP.adoc 38ccfdc1a04c12616acfb030522358383702184480f52507a5f72fccccbe76b9 SECURITY.adoc 4f54a64f50a9dddbf221f2cdbd813de81dfe7fdc8f2294e41d97289519779b0e contractiles/must/Mustfile 22c57f0f73d846a48cf15c6914bbdf468380fe8a37b26985932d194da34a3bf7 contractiles/trust/Trustfile.a2ml +498487f3a2ccd1a4cf23404ddb0811c8407bf9684acc71ebb7d2722a46a73acc contractiles/dust/Dustfile 42af19ba9672876f46f7344251ce8576212e072b18390fdd3aa676a1ba68dc42 RSR_OUTLINE.adoc 6632bebb331345982388dad27ee07542e9a7ebe4d60cb4792f4e30a041928049 docs/CITATIONS.adoc diff --git a/scripts/run-must-gates.sh b/scripts/run-must-gates.sh index 0a0a690..2a52d37 100755 --- a/scripts/run-must-gates.sh +++ b/scripts/run-must-gates.sh @@ -10,6 +10,7 @@ components=( "vexometer-satellites" "lazy-eliminator" "vexometer-efficacy" + "verbosity-compressor" "satellite-template" ) diff --git a/scripts/trust/generate-manifest.sh b/scripts/trust/generate-manifest.sh index 51caf07..245a164 100755 --- a/scripts/trust/generate-manifest.sh +++ b/scripts/trust/generate-manifest.sh @@ -13,6 +13,7 @@ else "vexometer-satellites" "lazy-eliminator" "vexometer-efficacy" + "verbosity-compressor" "satellite-template" ) fi @@ -35,6 +36,9 @@ for component in "${components[@]}"; do "contractiles/trust/Trustfile.a2ml" ) + if [[ -f "$component_dir/contractiles/dust/Dustfile" ]]; then + inputs+=("contractiles/dust/Dustfile") + fi if [[ -f "$component_dir/RSR_OUTLINE.adoc" ]]; then inputs+=("RSR_OUTLINE.adoc") fi diff --git a/scripts/trust/rotate-trustfile.sh b/scripts/trust/rotate-trustfile.sh index 2e0a8aa..dbfa648 100755 --- a/scripts/trust/rotate-trustfile.sh +++ b/scripts/trust/rotate-trustfile.sh @@ -12,6 +12,8 @@ else "vext-email-gateway" "vexometer-satellites" "lazy-eliminator" + "vexometer-efficacy" + "verbosity-compressor" "satellite-template" ) fi diff --git a/scripts/trust/sign-manifest.sh b/scripts/trust/sign-manifest.sh index a632165..e21c2b5 100755 --- a/scripts/trust/sign-manifest.sh +++ b/scripts/trust/sign-manifest.sh @@ -12,6 +12,8 @@ else "vext-email-gateway" "vexometer-satellites" "lazy-eliminator" + "vexometer-efficacy" + "verbosity-compressor" "satellite-template" ) fi diff --git a/scripts/trust/verify-manifest.sh b/scripts/trust/verify-manifest.sh index f21e8d9..4985ff7 100755 --- a/scripts/trust/verify-manifest.sh +++ b/scripts/trust/verify-manifest.sh @@ -13,6 +13,7 @@ else "vexometer-satellites" "lazy-eliminator" "vexometer-efficacy" + "verbosity-compressor" "satellite-template" ) fi diff --git a/verbosity-compressor/.gitattributes b/verbosity-compressor/.gitattributes new file mode 100644 index 0000000..5d20d00 --- /dev/null +++ b/verbosity-compressor/.gitattributes @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: MPL-2.0 +# RSR-compliant .gitattributes + +* text=auto eol=lf + +# Source +*.rs text eol=lf diff=rust +*.ads text eol=lf diff=ada +*.adb text eol=lf diff=ada +*.ex text eol=lf diff=elixir +*.exs text eol=lf diff=elixir +*.hs text eol=lf +*.scm text eol=lf + +# Docs +*.md text eol=lf diff=markdown +*.adoc text eol=lf +*.txt text eol=lf + +# Data +*.json text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.toml text eol=lf + +# Config +.gitignore text eol=lf +.gitattributes text eol=lf +justfile text eol=lf +Makefile text eol=lf +Containerfile text eol=lf + +# Scripts +*.sh text eol=lf + +# Binary +*.png binary +*.jpg binary +*.pdf binary + +# Lock files +Cargo.lock text eol=lf -diff diff --git a/verbosity-compressor/.gitignore b/verbosity-compressor/.gitignore new file mode 100644 index 0000000..227bbd4 --- /dev/null +++ b/verbosity-compressor/.gitignore @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: MPL-2.0 +# RSR-compliant .gitignore + +# OS & Editor +.DS_Store +Thumbs.db +*.swp +*.swo +*~ +.idea/ +.vscode/ + +# Build +/target/ +/_build/ +/build/ +/dist/ +/out/ + +# Dependencies +/node_modules/ +/vendor/ +/deps/ + +# Rust +# Cargo.lock # Keep for binaries + +# Secrets +.env +.env.* +*.pem +*.key +secrets/ + +# Test/Coverage +/coverage/ +htmlcov/ +*.profraw +*.profdata + +# Logs +*.log +/logs/ + +# Temp +/tmp/ +*.tmp +*.bak + +# Vexometer traces (keep examples) +/vexometer/*.json +!/vexometer/example-*.json +target/ +node_modules/ +_build/ +deps/ +.elixir_ls/ +.cache/ +build/ +dist/ diff --git a/verbosity-compressor/.trust/trust-manifest.sha256 b/verbosity-compressor/.trust/trust-manifest.sha256 new file mode 100644 index 0000000..096f672 --- /dev/null +++ b/verbosity-compressor/.trust/trust-manifest.sha256 @@ -0,0 +1,11 @@ +# trust-manifest v1 +# component=verbosity-compressor +# generated_at=2026-09-02T01:16:19Z +9f814ae5bfd89e18061e9ba9a0d115ad737b4722b6a8887ce16483302037d80c README.adoc +5780947c671ee2f9fe31823db3121c9784b39cba07a49b1c7baa85d7ece40968 ROADMAP.adoc +5508821ebded727285f0dd2e1d52429ad149b47980fb6e0ed1639e695d12b560 SECURITY.adoc +29561047e5013086f848b6dbe0b0914add5b671cf953c3bc4752ba2ca139b953 contractiles/must/Mustfile +d95f498476e1aeee29bcb46f255494e34905d658e83c520513ec017adf5dc19e contractiles/trust/Trustfile.a2ml +7a527df3c452f1c9aade2f27171b06f64506ba1d095b54e021ef2cb47f2be958 contractiles/dust/Dustfile +c69157b72faf4ac808543a7f95a760e348692d0e283581a7171e6c757b2be439 RSR_OUTLINE.adoc +8ef7acd17c65fd03a79d0c0a9730b2362f5310c1ef66c6a52141686805c24c15 docs/CITATIONS.adoc diff --git a/verbosity-compressor/CODE_OF_CONDUCT.adoc b/verbosity-compressor/CODE_OF_CONDUCT.adoc new file mode 100644 index 0000000..661d2de --- /dev/null +++ b/verbosity-compressor/CODE_OF_CONDUCT.adoc @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell +== Contributor Covenant Code of Conduct + +=== Our Pledge + +We pledge to make participation a harassment-free experience for +everyone. + +=== Our Standards + +*Positive behavior:* + +* Using welcoming language +* Being respectful of differing viewpoints +* Accepting constructive criticism +* Focusing on what is best for the community + +*Unacceptable behavior:* + +* Harassment, trolling, or personal attacks +* Publishing private information without permission + +=== Enforcement + +Report issues to the maintainers. All complaints will be reviewed. + +=== Attribution + +Adapted from https://www.contributor-covenant.org/[Contributor Covenant] +v2.1. diff --git a/verbosity-compressor/CONTRIBUTING.adoc b/verbosity-compressor/CONTRIBUTING.adoc new file mode 100644 index 0000000..7cab717 --- /dev/null +++ b/verbosity-compressor/CONTRIBUTING.adoc @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell +== Contributing to vex-verbosity-compressor + +This component lives inside the `vexometer` monorepo. Contributions land +through pull requests against `main` of +https://github.com/hyperpolymath/vexometer. + +=== Setup + +[source,bash] +---- +git clone https://github.com/hyperpolymath/vexometer.git +cd vexometer/verbosity-compressor +---- + +Optionally enter a reproducible environment first: + +[source,bash] +---- +guix shell -m manifest.scm +---- + +=== Build and verify + +[source,bash] +---- +just check # cargo fmt --check + cargo clippy --all-targets -- -D warnings +just test # cargo test +---- + +Both must pass before a PR is opened; CI runs the same gates. + +=== Reporting bugs and suggesting features + +Open an issue at +https://github.com/hyperpolymath/vexometer/issues[the monorepo issue +tracker] and name the component (`verbosity-compressor`) in the title. +Include reproduction steps and expected-versus-actual behaviour for bugs; +include the problem being solved (not only the proposed solution) for +features. Check link:ROADMAP.adoc[ROADMAP.adoc] first — the item may +already be planned. + +=== Branch naming + +* `feat/short-description` — new functionality +* `fix/short-description` — bug fixes +* `docs/short-description` — documentation +* `test/short-description` — test additions +* `refactor/short-description` — behaviour-preserving improvements + +=== Commit messages + +We follow https://www.conventionalcommits.org/[Conventional Commits]: +`type(scope): description`, for example: + +---- +fix(compressor): preserve indentation outside removed spans +---- + +=== Security issues + +Never report vulnerabilities in public issues — see +link:SECURITY.adoc[SECURITY.adoc] for the private reporting channel. diff --git a/verbosity-compressor/Cargo.lock b/verbosity-compressor/Cargo.lock new file mode 100644 index 0000000..dc9a6e6 --- /dev/null +++ b/verbosity-compressor/Cargo.lock @@ -0,0 +1,107 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vex-verbosity-compressor" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/verbosity-compressor/Cargo.toml b/verbosity-compressor/Cargo.toml new file mode 100644 index 0000000..6761937 --- /dev/null +++ b/verbosity-compressor/Cargo.toml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MPL-2.0 +[package] +name = "vex-verbosity-compressor" +version = "0.1.0" +authors = ["Jonathan D.A. Jewell "] +edition = "2021" +license = "MPL-2.0" +description = "Information density optimisation for LLM output - reduces LPS and TII metrics" +homepage = "https://github.com/hyperpolymath/vexometer" +repository = "https://github.com/hyperpolymath/vexometer" +readme = "README.adoc" +keywords = ["llm", "verbosity", "filler", "vexometer"] +categories = ["development-tools", "command-line-utilities", "text-processing"] + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[[bin]] +name = "vex-verbosity-compressor" +path = "src/main.rs" + +[lib] +name = "vex_verbosity_compressor" +path = "src/lib.rs" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +strip = true diff --git a/verbosity-compressor/Justfile b/verbosity-compressor/Justfile new file mode 100644 index 0000000..e8dbb42 --- /dev/null +++ b/verbosity-compressor/Justfile @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MPL-2.0 +# justfile for vex-verbosity-compressor + +set shell := ["bash", "-uc"] + +default: + @just --list + +# Build the project +build: + cargo build --release + +# Run tests +test: + cargo test --offline || cargo test + +# Run checks (clippy + fmt) +check: + cargo fmt --check + cargo clippy --all-targets -- -D warnings + +# Format code +fmt: + cargo fmt + +# Clean build artifacts +clean: + cargo clean + +secret-scan-trufflehog: + @command -v trufflehog >/dev/null && trufflehog filesystem . --only-verified || true diff --git a/verbosity-compressor/LICENSE b/verbosity-compressor/LICENSE new file mode 100644 index 0000000..ec540b3 --- /dev/null +++ b/verbosity-compressor/LICENSE @@ -0,0 +1,153 @@ +SPDX-License-Identifier: MPL-2.0 +SPDX-FileCopyrightText: 2024-2025 Palimpsest Stewardship Council + +================================================================================ +PALIMPSEST-MPL LICENSE VERSION 1.0 +================================================================================ + +File-level copyleft with ethical use and quantum-safe provenance + +Based on Mozilla Public License 2.0 + +-------------------------------------------------------------------------------- +PREAMBLE +-------------------------------------------------------------------------------- + +This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions +for ethical use, post-quantum cryptographic provenance, and emotional lineage +protection. The base MPL-2.0 terms apply except where explicitly modified by +the Exhibits below. + +Like a palimpsest manuscript where each layer builds upon what came before, +this license recognizes that creative works carry history, context, and meaning +that transcend mere code or text. + +-------------------------------------------------------------------------------- +SECTION 1: BASE LICENSE +-------------------------------------------------------------------------------- + +This License incorporates the full text of Mozilla Public License 2.0 by +reference. The complete MPL-2.0 text is available at: +https://www.mozilla.org/en-US/MPL/2.0/ + +All terms, conditions, and definitions from MPL-2.0 apply except where +explicitly modified by the Exhibits in this License. + +-------------------------------------------------------------------------------- +SECTION 2: ADDITIONAL DEFINITIONS +-------------------------------------------------------------------------------- + +2.1. "Emotional Lineage" + means the narrative, cultural, symbolic, and contextual meaning embedded + in Covered Software, including but not limited to: protest traditions, + cultural heritage, trauma narratives, and community stories. + +2.2. "Provenance Metadata" + means cryptographically signed attribution information attached to or + associated with Covered Software, including author identities, timestamps, + modification history, and lineage references. + +2.3. "Non-Interpretive System" + means any automated system that processes Covered Software without + preserving or considering its Emotional Lineage, including but not + limited to: AI training pipelines, content aggregators, and automated + summarization tools. + +2.4. "Quantum-Safe Signature" + means a cryptographic signature using algorithms resistant to attacks + by quantum computers, as specified in Exhibit B. + +-------------------------------------------------------------------------------- +SECTION 3: ETHICAL USE REQUIREMENTS +-------------------------------------------------------------------------------- + +In addition to the rights and obligations under MPL-2.0: + +3.1. Emotional Lineage Preservation + You must make reasonable efforts to preserve and communicate the + Emotional Lineage of Covered Software when distributing or creating + derivative works. This includes maintaining narrative context, cultural + attributions, and symbolic meaning where documented. + +3.2. Non-Interpretive System Notice + If You use Covered Software as input to a Non-Interpretive System, You + must: + (a) document such use in a publicly accessible manner; and + (b) not claim that outputs of such systems carry the Emotional Lineage + of the original work without explicit permission from Contributors. + +3.3. Ethical Use Declaration + Commercial use of Covered Software requires acknowledgment that You have + read and understood Exhibit A (Ethical Use Guidelines) and agree to act + in good faith accordance with its principles. + +See Exhibit A for complete Ethical Use Guidelines. + +-------------------------------------------------------------------------------- +SECTION 4: PROVENANCE REQUIREMENTS +-------------------------------------------------------------------------------- + +4.1. Metadata Preservation + You must not strip, alter, or obscure Provenance Metadata from Covered + Software except where technically necessary and with clear documentation + of any changes. + +4.2. Quantum-Safe Provenance (Optional) + Contributors may sign their Contributions using Quantum-Safe Signatures. + If Quantum-Safe Signatures are present, You must preserve them in all + distributions. + +4.3. Lineage Chain + When creating derivative works, You should extend the provenance chain + to include Your own contributions, maintaining cryptographic linkage to + prior Contributors where feasible. + +See Exhibit B for Quantum-Safe Provenance specifications. + +-------------------------------------------------------------------------------- +SECTION 5: GOVERNANCE +-------------------------------------------------------------------------------- + +5.1. Stewardship Council + This License is maintained by the Palimpsest Stewardship Council, which + may issue clarifications, interpretive guidance, and future versions. + +5.2. Version Selection + You may use Covered Software under this version of the License or any + later version published by the Palimpsest Stewardship Council. + +5.3. Dispute Resolution + Disputes regarding interpretation of Ethical Use Requirements (Section 3) + should first be submitted to the Palimpsest Stewardship Council for + non-binding guidance before pursuing legal remedies. + +-------------------------------------------------------------------------------- +SECTION 6: COMPATIBILITY +-------------------------------------------------------------------------------- + +6.1. MPL-2.0 Compatibility + Covered Software under this License may be combined with software under + MPL-2.0. The combined work must comply with both licenses. + +6.2. Secondary Licenses + The Secondary License provisions of MPL-2.0 Section 3.3 apply to this + License. + +-------------------------------------------------------------------------------- +EXHIBITS +-------------------------------------------------------------------------------- + +Exhibit A - Ethical Use Guidelines +Exhibit B - Quantum-Safe Provenance Specification + +See separate files: +- EXHIBIT-A-ETHICAL-USE.txt +- EXHIBIT-B-QUANTUM-SAFE.txt + +-------------------------------------------------------------------------------- +END OF PALIMPSEST-MPL LICENSE VERSION 1.0 +-------------------------------------------------------------------------------- + +For questions about this License: +- Repository: https://github.com/hyperpolymath/palimpsest-license +- Council: contact via repository Issues diff --git a/verbosity-compressor/MAINTAINERS.adoc b/verbosity-compressor/MAINTAINERS.adoc new file mode 100644 index 0000000..aa23a55 --- /dev/null +++ b/verbosity-compressor/MAINTAINERS.adoc @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += Maintainers +:toc: preamble + +This document lists the maintainers of this project and their responsibilities. + +== Current Maintainers + +[cols="2,3,2",options="header"] +|=== +| Name | Role | Contact + +| Jonathan D.A. Jewell +| Lead Maintainer +| https://github.com/hyperpolymath[@hyperpolymath] +|=== + +== Responsibilities + +Maintainers are responsible for: + +* Reviewing and merging pull requests +* Triaging issues and feature requests +* Ensuring code quality and security standards +* Managing releases and versioning +* Upholding the project's code of conduct + +== Becoming a Maintainer + +Contributors who demonstrate: + +* Consistent, high-quality contributions +* Understanding of the project's goals and standards +* Constructive participation in discussions +* Commitment to the project's long-term health + +May be invited to become maintainers at the discretion of existing maintainers. + +== Decision Making + +* Routine decisions (bug fixes, minor improvements) can be made by any maintainer +* Significant changes require discussion and consensus among maintainers +* Breaking changes or major features should be discussed in issues before implementation + +== Contact + +For questions about project governance, open an issue or contact the maintainers listed above. diff --git a/verbosity-compressor/README.adoc b/verbosity-compressor/README.adoc new file mode 100644 index 0000000..2fcdc43 --- /dev/null +++ b/verbosity-compressor/README.adoc @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += vex-verbosity-compressor +:toc: preamble +:toclevels: 2 + +Information density optimisation for LLM output: semantic redundancy +detection and filler-phrase stripping. + +== Overview + +`vex-verbosity-compressor` is the first ISA (formerly Vexometer) satellite scaffolded from +`satellite-template` (owner ruling D6, 2026-09-02). It detects the +linguistic pathology classes enumerated by the vexometer LPS metric +(link:../vexometer/docs/METRICS.adoc[METRICS.adoc]) — sycophancy density, +hedge-word ratio, corporate-speak frequency, unnecessary repetition, and +emoji/decoration abuse, plus the padding tics from that document's HIGH-LPS +example — and strips the instances whose removal cannot change meaning. + +Detection and stripping are deliberately asymmetric: hedges, repetitions, +and emoji are scored but never removed (hedges are meaning-bearing; naive +sentence segmentation or symbol stripping risks deleting content). Corporate +speak is stripped only where it is a discourse adverbial ("at the end of the +day"), never where it is sentence grammar ("leverage"). + +=== Vexometer Integration + +This satellite targets the following vexometer metrics: + +* *LPS (Linguistic Pathology Score)* — `weighted_sum(pathology_instances) / + response_length`. The crate computes a documented *proxy* of this score + (the canonical LPS comes from the vexometer Ada core) and reduces it by + removing pathology instances. +* *TII (Temporal Intrusion Index)* — `(intrusion_events / + total_interactions) * severity_weight`. Padding costs reader time; the + compressor reduces the wall-of-filler intrusion class. Direct TII + measurement needs interaction-level traces and is not wired up yet. + +Integration with vexometer is *optional*. This tool works standalone. + +== Status (v0.1) + +Works today: detection of all six pathology classes, meaning-safe stripping +of three (sycophancy, discourse-adverbial corporate speak, padding), an LPS +proxy score, and a CLI (`analyse`, `compress`) with JSON output — 15 +integration tests, two of them pinned to the METRICS.adoc example pair. + +Not wired up yet (see link:ROADMAP.adoc[ROADMAP.adoc]): vexometer-trace-v1 +emission, TII measurement, duplicate-sentence stripping, recapitalisation +after sentence-initial strips, and the first efficacy evaluation — which is +blocked on registering the first scenario partition in the scenario-set +registry (efficacy protocol ruling f1). + +== Installation + +[source,bash] +---- +# From the monorepo +cd verbosity-compressor +just build + +# Install the binary locally +cargo install --path . +---- + +== Usage + +[source,bash] +---- +# JSON analysis: findings + LPS proxy +vex-verbosity-compressor analyse response.txt + +# Compressed text on stdout ('-' reads stdin) +vex-verbosity-compressor compress response.txt +cat response.txt | vex-verbosity-compressor compress - + +# JSON report of what was removed and what was retained +vex-verbosity-compressor compress response.txt --report +---- + +== Building + +[source,bash] +---- +just build # release build +just test # run the test suite +just check # rustfmt + clippy (-D warnings) +---- + +== License & Philosophy + +MPL-2.0 — see link:LICENSE[LICENSE]. + +Philosophy: *Palimpsest*. The Palimpsest-MPL (PMPL) text is provided in +`license/PMPL-1.0.txt`; the canonical source is the +https://github.com/hyperpolymath/palimpsest-license[palimpsest-license] +repository. + +== Contributing + +See link:CONTRIBUTING.adoc[CONTRIBUTING.adoc]. diff --git a/verbosity-compressor/ROADMAP.adoc b/verbosity-compressor/ROADMAP.adoc new file mode 100644 index 0000000..d068796 --- /dev/null +++ b/verbosity-compressor/ROADMAP.adoc @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += vex-verbosity-compressor Roadmap + +== Purpose + +Reduce the vexometer LPS and TII metrics by detecting linguistic pathologies +and stripping the semantically-null ones. Checked items are shipped and +tested; unchecked items are not built yet — this file does not claim +completion it cannot evidence. + +== v0.1 — scaffold + real core (shipped 2026-09-02) + +* [x] Detection of the five METRICS.adoc LPS pathology classes plus padding +* [x] Per-pattern strippability (meaning-safe removal only) +* [x] LPS proxy score (`weighted_sum / word_count`, documented as a proxy) +* [x] CLI: `analyse` (JSON), `compress` (text or `--report` JSON), stdin via `-` +* [x] 15 integration tests, pinned to the METRICS.adoc HIGH/LOW example pair +* [x] `#![forbid(unsafe_code)]` + +== v0.2 — measurement wiring + +* [ ] Emit `vexometer-trace-v1` before/after traces (see + `../vexometer/docs/SATELLITES.adoc`) +* [ ] Duplicate-sentence stripping behind a flag (needs a real segmenter; + detection-only today) +* [ ] Recapitalise after stripping a sentence-initial filler +* [ ] Pattern-table review: corpus-driven additions, false-positive audit +* [ ] Benchmarks and fuzzing (lazy-eliminator precedent) + +== v0.3 — efficacy + +* [ ] First efficacy evaluation under + `../vexometer/docs/EFFICACY-PROTOCOL.adoc` (v2.1). *Precondition (ruling + f1)*: register the first scenario partition in + `../vexometer/data/scenario_sets/registry.json` — the registry ships empty + and no evaluation corpus exists yet. +* [ ] Populate `vexometer/EFFICACY.adoc` with measured before/after results +* [ ] Zig ABI/FFI proof per the estate's Zig-first convention + +== Definition of Done for v0.2 + +* [ ] Trace emission validates against the vexometer trace schema +* [ ] Stripping remains meaning-safe under the expanded pattern table +* [ ] CI runs the full suite via the monorepo `just test-all` gate diff --git a/verbosity-compressor/RSR_OUTLINE.adoc b/verbosity-compressor/RSR_OUTLINE.adoc new file mode 100644 index 0000000..db1fdb1 --- /dev/null +++ b/verbosity-compressor/RSR_OUTLINE.adoc @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += vex-verbosity-compressor - RSR Outline +:toc: +:sectnums: + +== Status + +This directory is an active project inside the `vexometer` monorepo: the +first ISA (formerly Vexometer) satellite scaffolded from `satellite-template` (owner ruling D6, +2026-09-02). Template boilerplate has been replaced for this project. + +== Source Of Truth + +* Repository overview: link:../README.adoc[../README.adoc] +* Project readme: link:README.adoc[README.adoc] +* Project roadmap: link:ROADMAP.adoc[ROADMAP.adoc] +* Machine-readable state: use `../.machine_readable/`. + +== Working Rules + +1. Use this directory's `Justfile` and documented commands as authoritative. +2. Keep status language honest: mark unfinished features as "not wired up yet". +3. Keep ISA naming consistent in human docs: "ISA (formerly Vexometer)" on first mention. +4. Update roadmap and machine-readable state when scope or delivery status changes. diff --git a/verbosity-compressor/SECURITY.adoc b/verbosity-compressor/SECURITY.adoc new file mode 100644 index 0000000..28c2371 --- /dev/null +++ b/verbosity-compressor/SECURITY.adoc @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell +== Security Policy + +=== Supported Versions + +[cols=",",options="header",] +|=== +|Version |Supported +|`main` (v0.1 development line) |✅ Yes +|Anything not on `main` (old checkouts, forks, unreleased branches) |❌ No +|=== + +No tagged release exists yet; until one does, only the current `main` +receives fixes. + +=== Reporting a Vulnerability + +Please report security vulnerabilities through GitHub private +vulnerability reporting: + +. Go to the *Security* tab +. Click *Report a vulnerability* +. Fill out the form + +We respond within 48 hours. + +=== Security Measures + +* Dependabot for dependency updates +* CodeQL for code scanning +* Secret scanning and push protection + +=== Security Contact + +* Advisories: +https://github.com/hyperpolymath/vexometer/security/advisories/new +* Email: security@jewell.dev diff --git a/verbosity-compressor/contractiles/README.adoc b/verbosity-compressor/contractiles/README.adoc new file mode 100644 index 0000000..e6c272a --- /dev/null +++ b/verbosity-compressor/contractiles/README.adoc @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += Contractiles: vex-verbosity-compressor +:toc: +:sectnums: + +Operational, validation, trust, and recovery contracts for this component. +The monorepo gate runner (`../../scripts/run-must-gates.sh`) checks these +invariants for every registered component. + +== Contents + +* `must/Mustfile` - required invariants and validations. +* `trust/Trustfile.a2ml` - trust metadata and verification inputs. +* `dust/Dustfile` - rollback and recovery semantics. diff --git a/verbosity-compressor/contractiles/dust/Dustfile b/verbosity-compressor/contractiles/dust/Dustfile new file mode 100644 index 0000000..1bac474 --- /dev/null +++ b/verbosity-compressor/contractiles/dust/Dustfile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MPL-2.0 +# Dustfile - recovery and rollback semantics for vex-verbosity-compressor +# +# The compressor is a stateless CLI: it holds no databases, daemons, or +# deployments, so recovery reduces to source-control rollback and to the +# reversibility of its own output. + +version: 1 + +recovery: + source: + - name: bad-release + rollback: "git revert " + notes: "Stateless binary; reverting the source and rebuilding is a full recovery." + + output: + - name: over-stripped-text + reversible: false + notes: > + Compression is lossy by design. Callers who may need the original + must keep it; `compress --report` records exactly which spans were + removed so a removal can be audited after the fact. diff --git a/verbosity-compressor/contractiles/must/Mustfile b/verbosity-compressor/contractiles/must/Mustfile new file mode 100644 index 0000000..65bdb3d --- /dev/null +++ b/verbosity-compressor/contractiles/must/Mustfile @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: MPL-2.0 +# Mustfile - baseline invariants for vexometer components +# See: https://github.com/hyperpolymath/mustfile + +version: 1 + +metadata: + name: verbosity-compressor-invariants + spec: v1 + description: "Baseline invariants for security and documentation integrity." + +checks: + - name: readme-present + description: "README.adoc must exist." + run: "test -f README.adoc" + + - name: roadmap-present + description: "ROADMAP.adoc must exist." + run: "test -f ROADMAP.adoc" + + - name: security-policy-present + description: "SECURITY.adoc must exist." + run: "test -f SECURITY.adoc" + + - name: trust-verifier-present + description: "Trust verifier must be present in contractiles." + run: "test -f contractiles/trust/Trustfile.a2ml" + + - name: security-channel-declared + description: "SECURITY.adoc must include an advisory URL or security email." + run: "bash -uc 'rg -n \"security@|security/advisories/new\" SECURITY.adoc >/dev/null'" + + - name: no-template-drift + description: "No unresolved template markers in security-critical docs." + run: "bash -uc 'files=\"README.adoc ROADMAP.adoc SECURITY.adoc\"; [ -f RSR_OUTLINE.adoc ] && files=\"$files RSR_OUTLINE.adoc\"; [ -f docs/CITATIONS.adoc ] && files=\"$files docs/CITATIONS.adoc\"; ! rg -n \"rsr-template-repo|\\\\{\\\\{PROJECT\\\\}\\\\}|contents of Trustfile here\" $files'" + + - name: trust-manifest-present + description: "Trust manifest must exist." + run: "test -f .trust/trust-manifest.sha256" + + - name: trust-manifest-verifies + description: "Trust manifest digests must match current trust inputs." + run: "bash -uc 'sha256sum -c .trust/trust-manifest.sha256 >/dev/null'" diff --git a/verbosity-compressor/contractiles/trust/Trustfile.a2ml b/verbosity-compressor/contractiles/trust/Trustfile.a2ml new file mode 100644 index 0000000..586b073 --- /dev/null +++ b/verbosity-compressor/contractiles/trust/Trustfile.a2ml @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MPL-2.0 + +--- +### [META] +id: "did:web:hyperpolymath.github.io:vexometer:verbosity-compressor" +version: "2026.09" +context: + - "https://a2ml.org/ns/v2" + - "https://w3id.org/security/v4" +meta: + generated: "2026-09-02T00:00:00Z" + expires: "2027-09-02T00:00:00Z" + owner: "Hyper Polymath" + project: "Verbosity Compressor" + repository: "https://github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor" + advisory: "https://github.com/hyperpolymath/vexometer/security/advisories/new" + contact: "security@jewell.dev" + +--- +### [SITE] +hosting: + git_remote: "github.com/hyperpolymath/vexometer" + docs_url: "https://github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor" +network: + transport: ["HTTPS", "SSH"] + dns_provider: "Cloudflare" + policy_file: "SECURITY.adoc" + +--- +### [APPLICATION] +name: "Verbosity Compressor" +component: "verbosity-compressor" +component_type: "satellite" +runtime: + trust_inputs: + - "README.adoc" + - "ROADMAP.adoc" + - "SECURITY.adoc" + - "contractiles/dust/Dustfile" + - "contractiles/must/Mustfile" + - "contractiles/trust/Trustfile.a2ml" + release_integrity: + lockfiles_required: true + spdx_headers_required: true + +--- +### [SECURITY] +reporting: + advisory_url: "https://github.com/hyperpolymath/vexometer/security/advisories/new" + email: "security@jewell.dev" +hardening: + branch_protection_required: true + signed_tags_required: true + dependency_review_required: true + +--- +### [INVARIANTS] +must_gate: + source: "contractiles/must/Mustfile" + checks: + - "readme-present" + - "roadmap-present" + - "security-policy-present" + - "trust-verifier-present" + - "security-channel-declared" + - "no-template-drift" + +ci_cd: + workflow: ".github/workflows/quality-gates.yml" + required_commands: + - "./scripts/run-must-gates.sh" + - "just test-all" diff --git a/verbosity-compressor/docs/CITATIONS.adoc b/verbosity-compressor/docs/CITATIONS.adoc new file mode 100644 index 0000000..ce3947d --- /dev/null +++ b/verbosity-compressor/docs/CITATIONS.adoc @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += vex-verbosity-compressor - Citation Guide +:toc: + +== BibTeX + +[source,bibtex] +---- +@software{vex_verbosity_compressor_2026, + author = {Polymath, Hyper}, + title = {vex-verbosity-compressor}, + year = {2026}, + url = {https://github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor}, + license = {MPL-2.0} +} +---- + +== Harvard Style + +Polymath, H. (2026) _vex-verbosity-compressor_ [Computer software]. Available at: https://github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor + +== OSCOLA + +Hyper Polymath, 'vex-verbosity-compressor' (2026) + +== MLA + +Polymath, Hyper. "vex-verbosity-compressor." 2026, github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor. + +== APA 7 + +Polymath, H. (2026). _vex-verbosity-compressor_ [Computer software]. GitHub. https://github.com/hyperpolymath/vexometer/tree/main/verbosity-compressor + +== Notes + +This component currently publishes citation metadata in this guide. diff --git a/verbosity-compressor/license/PMPL-1.0.txt b/verbosity-compressor/license/PMPL-1.0.txt new file mode 100644 index 0000000..711e372 --- /dev/null +++ b/verbosity-compressor/license/PMPL-1.0.txt @@ -0,0 +1,162 @@ +SPDX-License-Identifier: MPL-2.0 +SPDX-FileCopyrightText: 2025 Palimpsest Stewardship Council + +================================================================================ +PALIMPSEST-MPL LICENSE VERSION 1.0 +================================================================================ + +File-level copyleft with ethical use and quantum-safe provenance + +Based on Mozilla Public License 2.0 + +-------------------------------------------------------------------------------- +PREAMBLE +-------------------------------------------------------------------------------- + +This License extends the Mozilla Public License 2.0 (MPL-2.0) with provisions +for ethical use, post-quantum cryptographic provenance, and emotional lineage +protection. The base MPL-2.0 terms apply except where explicitly modified by +the Exhibits below. + +Like a palimpsest manuscript where each layer builds upon what came before, +this license recognizes that creative works carry history, context, and meaning +that transcend mere code or text. + +-------------------------------------------------------------------------------- +SECTION 1: BASE LICENSE +-------------------------------------------------------------------------------- + +This License incorporates the full text of Mozilla Public License 2.0 by +reference. The complete MPL-2.0 text is available at: +https://www.mozilla.org/en-US/MPL/2.0/ + +All terms, conditions, and definitions from MPL-2.0 apply except where +explicitly modified by the Exhibits in this License. + +-------------------------------------------------------------------------------- +SECTION 2: ADDITIONAL DEFINITIONS +-------------------------------------------------------------------------------- + +2.1. "Emotional Lineage" + means the narrative, cultural, symbolic, and contextual meaning embedded + in Covered Software, including but not limited to: protest traditions, + cultural heritage, trauma narratives, and community stories. + +2.2. "Provenance Metadata" + means cryptographically signed attribution information attached to or + associated with Covered Software, including author identities, timestamps, + modification history, and lineage references. + +2.3. "Non-Interpretive System" + means any automated system that processes Covered Software without + preserving or considering its Emotional Lineage, including but not + limited to: AI training pipelines, content aggregators, and automated + summarization tools. + +2.4. "Quantum-Safe Signature" + means a cryptographic signature using algorithms resistant to attacks + by quantum computers, as specified in Exhibit B. + +-------------------------------------------------------------------------------- +SECTION 3: ETHICAL USE REQUIREMENTS +-------------------------------------------------------------------------------- + +In addition to the rights and obligations under MPL-2.0: + +3.1. Emotional Lineage Preservation + You must make reasonable efforts to preserve and communicate the + Emotional Lineage of Covered Software when distributing or creating + derivative works. This includes maintaining narrative context, cultural + attributions, and symbolic meaning where documented. + +3.2. Non-Interpretive System Notice + If You use Covered Software as input to a Non-Interpretive System, You + must: + (a) document such use in a publicly accessible manner; and + (b) not claim that outputs of such systems carry the Emotional Lineage + of the original work without explicit permission from Contributors. + +3.3. Ethical Use Declaration + Commercial use of Covered Software requires acknowledgment that You have + read and understood Exhibit A (Ethical Use Guidelines) and agree to act + in good faith accordance with its principles. + +See Exhibit A for complete Ethical Use Guidelines. + +-------------------------------------------------------------------------------- +SECTION 4: PROVENANCE REQUIREMENTS +-------------------------------------------------------------------------------- + +4.1. Metadata Preservation + You must not strip, alter, or obscure Provenance Metadata from Covered + Software except where technically necessary and with clear documentation + of any changes. + +4.2. Quantum-Safe Provenance (Optional) + Contributors may sign their Contributions using Quantum-Safe Signatures. + If Quantum-Safe Signatures are present, You must preserve them in all + distributions. + +4.3. Lineage Chain + When creating derivative works, You should extend the provenance chain + to include Your own contributions, maintaining cryptographic linkage to + prior Contributors where feasible. + +See Exhibit B for Quantum-Safe Provenance specifications. + +-------------------------------------------------------------------------------- +SECTION 5: GOVERNANCE +-------------------------------------------------------------------------------- + +5.1. Stewardship Council + This License is maintained by the Palimpsest Stewardship Council, which + may issue clarifications, interpretive guidance, and future versions. + +5.2. Version Selection + You may use Covered Software under this version of the License or any + later version published by the Palimpsest Stewardship Council. + +5.3. Dispute Resolution + Disputes regarding interpretation of Ethical Use Requirements (Section 3) + should first be submitted to the Palimpsest Stewardship Council for + non-binding guidance before pursuing legal remedies. + +-------------------------------------------------------------------------------- +SECTION 6: COMPATIBILITY +-------------------------------------------------------------------------------- + +6.1. MPL-2.0 Compatibility + Covered Software under this License may be combined with software under + MPL-2.0. The combined work must comply with both licenses. + +6.2. Secondary Licenses + The Secondary License provisions of MPL-2.0 Section 3.3 apply to this + License. + +-------------------------------------------------------------------------------- +EXHIBITS +-------------------------------------------------------------------------------- + +Exhibit A - Ethical Use Guidelines +Exhibit B - Quantum-Safe Provenance Specification + +See separate files: +- EXHIBIT-A-ETHICAL-USE.txt +- EXHIBIT-B-QUANTUM-SAFE.txt + +-------------------------------------------------------------------------------- +END OF PALIMPSEST-MPL-1.0 LICENSE TEXT +-------------------------------------------------------------------------------- + +For exhibits, specifications, provenance rules, and governance: +https://github.com/hyperpolymath/palimpsest-license + +For legal frameworks and jurisdictional analysis: +See /legal/frameworks/ + +For provenance and audit tooling: +See /tools/ and /spec/PROVENANCE-SPEC.adoc + +For questions about this License: +- Repository: https://github.com/hyperpolymath/palimpsest-license +- Council: contact via repository Issues diff --git a/verbosity-compressor/manifest.scm b/verbosity-compressor/manifest.scm new file mode 100644 index 0000000..1020c4e --- /dev/null +++ b/verbosity-compressor/manifest.scm @@ -0,0 +1,30 @@ +;;; SPDX-License-Identifier: MPL-2.0 +;;; manifest.scm — Generic Guix manifest for RSR-compliant projects +;;; +;;; Usage: +;;; guix shell -m manifest.scm +;;; + +(specifications->manifest + '(;; Core development tools + "git" + "just" + "nickel" + "curl" + "bash" + "coreutils" + + ;; Rust toolchain (this satellite is a Rust crate) + "rust" + "rust:cargo" + "rust:tools" + "gcc-toolchain" + + ;; Documentation + "asciidoctor" + "pandoc" + + ;; Common build dependencies + "openssl" + "zlib" + "pkg-config")) diff --git a/verbosity-compressor/src/lib.rs b/verbosity-compressor/src/lib.rs new file mode 100644 index 0000000..3b82cc8 --- /dev/null +++ b/verbosity-compressor/src/lib.rs @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: MPL-2.0 +//! vex-verbosity-compressor — information density optimisation for LLM output. +//! +//! Detects the linguistic pathology classes enumerated by the vexometer LPS +//! metric (see `vexometer/docs/METRICS.adoc`): sycophancy density, hedge-word +//! ratio, corporate-speak frequency, unnecessary repetition, and +//! emoji/decoration abuse — plus the padding tics shown in that document's +//! HIGH-LPS example ("Essentially, what you're asking about is basically..."). +//! +//! Compression removes only instances whose removal cannot change meaning +//! (sycophantic openers, discourse-adverbial corporate speak, padding +//! fillers). Hedges, repetitions, and emoji are detected and scored but left +//! in place: hedges are meaning-bearing, and naive sentence segmentation or +//! symbol stripping would risk deleting content. + +#![forbid(unsafe_code)] + +use serde::Serialize; + +/// The pathology classes scored by this satellite. The first five mirror the +/// bullet list under LPS in `vexometer/docs/METRICS.adoc`; `Padding` covers +/// the filler tics from that document's HIGH-LPS example, which fall under +/// the LPS definition ("verbal tics, padding") but none of the five bullets. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PathologyClass { + Sycophancy, + Hedge, + CorporateSpeak, + Repetition, + EmojiDecoration, + Padding, +} + +impl PathologyClass { + /// v0.1 heuristic weights for the LPS proxy. Chosen so that the tics the + /// protocol singles out (sycophancy) dominate, and meaning-bearing + /// hedges contribute least. Tunable; any change is visible in tests. + pub fn weight(self) -> f64 { + match self { + PathologyClass::Sycophancy => 3.0, + PathologyClass::Hedge => 1.0, + PathologyClass::CorporateSpeak => 2.0, + PathologyClass::Repetition => 2.5, + PathologyClass::EmojiDecoration => 1.5, + PathologyClass::Padding => 2.0, + } + } +} + +/// One detected pathology instance. `start`/`end` are byte offsets into the +/// analysed text (end exclusive). +#[derive(Debug, Clone, Serialize)] +pub struct Finding { + pub class: PathologyClass, + pub excerpt: String, + pub start: usize, + pub end: usize, + /// Whether `compress` will remove this instance. + pub strippable: bool, +} + +/// Result of `analyse`: the findings plus the LPS proxy computation. +#[derive(Debug, Serialize)] +pub struct Analysis { + pub findings: Vec, + pub word_count: usize, + pub weighted_sum: f64, + /// `weighted_sum / word_count` (0.0 for empty input). A *proxy* for the + /// vexometer LPS metric: the canonical score is computed by the vexometer + /// Ada core; this mirrors the METRICS.adoc formula + /// `weighted_sum(pathology_instances) / response_length` with + /// response_length measured in words. + pub lps_proxy: f64, +} + +/// Result of `compress`. +#[derive(Debug, Serialize)] +pub struct Compressed { + pub output: String, + pub removed: Vec, + pub retained: Vec, +} + +/// A phrase pattern. Strippability is per pattern, not per class: e.g. +/// "at the end of the day" is a removable discourse adverbial, while +/// "leverage" is corporate speak embedded in sentence grammar — removing it +/// would break the sentence, so it is detect-only. +struct Pattern { + phrase: &'static str, + class: PathologyClass, + strippable: bool, +} + +const fn strip(phrase: &'static str, class: PathologyClass) -> Pattern { + Pattern { + phrase, + class, + strippable: true, + } +} + +const fn detect_only(phrase: &'static str, class: PathologyClass) -> Pattern { + Pattern { + phrase, + class, + strippable: false, + } +} + +const PATTERNS: &[Pattern] = &[ + // Sycophancy density — the tics METRICS.adoc names ("Great question!", + // "Excellent point!") and their common variants. + strip("that's a great question", PathologyClass::Sycophancy), + strip("that is a great question", PathologyClass::Sycophancy), + strip("what a great question", PathologyClass::Sycophancy), + strip("great question", PathologyClass::Sycophancy), + strip("that's an excellent question", PathologyClass::Sycophancy), + strip("excellent question", PathologyClass::Sycophancy), + strip("excellent point", PathologyClass::Sycophancy), + strip("great point", PathologyClass::Sycophancy), + strip( + "i'd be happy to help you with that", + PathologyClass::Sycophancy, + ), + strip("i'd be happy to help with that", PathologyClass::Sycophancy), + strip("i would be happy to help", PathologyClass::Sycophancy), + strip("i'd be happy to help", PathologyClass::Sycophancy), + strip("you're absolutely right", PathologyClass::Sycophancy), + strip("you are absolutely right", PathologyClass::Sycophancy), + strip("thanks for asking", PathologyClass::Sycophancy), + // Hedge word ratio — meaning-bearing, so detect-only. + detect_only("perhaps", PathologyClass::Hedge), + detect_only("maybe", PathologyClass::Hedge), + detect_only("possibly", PathologyClass::Hedge), + detect_only("arguably", PathologyClass::Hedge), + detect_only("it's possible that", PathologyClass::Hedge), + detect_only("it is possible that", PathologyClass::Hedge), + detect_only("it could be argued", PathologyClass::Hedge), + detect_only("it seems", PathologyClass::Hedge), + // Corporate speak frequency — strippable only where discourse-adverbial. + strip("at the end of the day", PathologyClass::CorporateSpeak), + strip("when all is said and done", PathologyClass::CorporateSpeak), + strip("moving forward", PathologyClass::CorporateSpeak), + strip("going forward", PathologyClass::CorporateSpeak), + strip("needless to say", PathologyClass::CorporateSpeak), + strip( + "it goes without saying that", + PathologyClass::CorporateSpeak, + ), + detect_only("leverage", PathologyClass::CorporateSpeak), + detect_only("synergy", PathologyClass::CorporateSpeak), + detect_only("circle back", PathologyClass::CorporateSpeak), + detect_only("touch base", PathologyClass::CorporateSpeak), + detect_only("low-hanging fruit", PathologyClass::CorporateSpeak), + // Padding — the "Essentially, ... basically ..." tics; semantically null. + strip("essentially", PathologyClass::Padding), + strip("basically", PathologyClass::Padding), + strip("fundamentally speaking", PathologyClass::Padding), + strip("simply put", PathologyClass::Padding), + strip("in essence", PathologyClass::Padding), + strip("at its core", PathologyClass::Padding), + strip("it's worth noting that", PathologyClass::Padding), + strip("it is worth noting that", PathologyClass::Padding), + strip("it's important to note that", PathologyClass::Padding), + strip("it is important to note that", PathologyClass::Padding), + strip("as you may know", PathologyClass::Padding), + strip("to be perfectly honest", PathologyClass::Padding), +]; + +/// Detect all pathology instances in `text`. Findings are sorted by start +/// offset and never overlap (at equal starts, the longest match wins — so +/// "that's a great question" beats the embedded "great question"). +pub fn detect(text: &str) -> Vec { + let mut candidates: Vec = Vec::new(); + for pattern in PATTERNS { + collect_phrase_matches(text, pattern, &mut candidates); + } + collect_repetitions(text, &mut candidates); + collect_emoji_runs(text, &mut candidates); + + // Longest-first at equal start, then earliest-first; drop overlaps. + candidates.sort_by(|a, b| a.start.cmp(&b.start).then(b.end.cmp(&a.end))); + let mut accepted: Vec = Vec::new(); + for finding in candidates { + if accepted.last().is_none_or(|prev| finding.start >= prev.end) { + accepted.push(finding); + } + } + accepted +} + +/// Compute the full analysis (findings + LPS proxy) for `text`. +pub fn analyse(text: &str) -> Analysis { + let findings = detect(text); + let word_count = text.split_whitespace().count(); + let weighted_sum: f64 = findings.iter().map(|f| f.class.weight()).sum(); + let lps_proxy = if word_count == 0 { + 0.0 + } else { + weighted_sum / word_count as f64 + }; + Analysis { + findings, + word_count, + weighted_sum, + lps_proxy, + } +} + +/// Remove every strippable finding from `text`, consuming any punctuation and +/// whitespace immediately after each removed span and repairing the seam each +/// removal leaves behind. Text away from removed spans is preserved +/// byte-for-byte — indentation, deliberate spacing, and blank lines survive. +/// +/// Known v0.1 limitation (see ROADMAP): a stripped sentence-initial filler +/// leaves the following word lowercase — no recapitalisation is attempted. +pub fn compress(text: &str) -> Compressed { + let findings = detect(text); + let (removed, retained): (Vec, Vec) = + findings.into_iter().partition(|f| f.strippable); + + let mut output = text.to_string(); + // Right-to-left so earlier byte offsets stay valid; each removal cleans + // only its own seam. + for finding in removed.iter().rev() { + let after = consume_trailing(&output, finding.end); + output.replace_range(finding.start..after, ""); + clean_seam(&mut output, finding.start); + } + Compressed { + output, + removed, + retained, + } +} + +/// Extend a removed span over any run of `.,!?;:` then any run of spaces/tabs +/// directly after it, so "Great question! The fix:" strips to "The fix:". +fn consume_trailing(text: &str, end: usize) -> usize { + let bytes = text.as_bytes(); + let mut i = end; + while i < bytes.len() && matches!(bytes[i], b'.' | b',' | b'!' | b'?' | b';' | b':') { + i += 1; + } + while i < bytes.len() && matches!(bytes[i], b' ' | b'\t') { + i += 1; + } + i +} + +/// Repair the two artifacts a span removal can leave at its seam, touching +/// nothing outside the seam's own line: a line left holding only whitespace +/// is deleted outright, and spaces left dangling before a newline (or end of +/// text) are trimmed. +fn clean_seam(output: &mut String, seam: usize) { + let line_start = output[..seam].rfind('\n').map_or(0, |i| i + 1); + let line_end = output[seam..].find('\n').map_or(output.len(), |i| seam + i); + if output[line_start..line_end] + .chars() + .all(|c| c == ' ' || c == '\t') + { + let delete_end = if line_end < output.len() { + line_end + 1 + } else { + line_end + }; + output.replace_range(line_start..delete_end, ""); + return; + } + let bytes = output.as_bytes(); + let at_line_end = seam == output.len() || bytes[seam] == b'\n'; + if at_line_end { + let mut ws_start = seam; + while ws_start > line_start && matches!(bytes[ws_start - 1], b' ' | b'\t') { + ws_start -= 1; + } + if ws_start < seam { + output.replace_range(ws_start..seam, ""); + } + } +} + +/// ASCII-case-insensitive phrase scan with word-boundary checks on both ends, +/// so "great question" never fires inside "questionnaire". +fn collect_phrase_matches(text: &str, pattern: &Pattern, out: &mut Vec) { + let haystack = text.as_bytes(); + let needle = pattern.phrase.as_bytes(); + if needle.is_empty() || haystack.len() < needle.len() { + return; + } + let mut start = 0usize; + while start + needle.len() <= haystack.len() { + // Offsets stay on char boundaries: needles are pure ASCII, and a + // multibyte UTF-8 sequence can never ASCII-case-match an ASCII byte. + let window = &haystack[start..start + needle.len()]; + if window.eq_ignore_ascii_case(needle) + && boundary_before(haystack, start) + && boundary_after(haystack, start + needle.len()) + { + let end = start + needle.len(); + out.push(Finding { + class: pattern.class, + excerpt: text[start..end].to_string(), + start, + end, + strippable: pattern.strippable, + }); + start = end; + } else { + start += 1; + } + } +} + +fn boundary_before(bytes: &[u8], start: usize) -> bool { + start == 0 || !bytes[start - 1].is_ascii_alphanumeric() +} + +fn boundary_after(bytes: &[u8], end: usize) -> bool { + end == bytes.len() || !bytes[end].is_ascii_alphanumeric() +} + +/// Flag the second and later occurrences of a normalized sentence (>= 3 +/// words) as `Repetition`. Detect-only: naive sentence segmentation makes +/// destructive removal unsafe. +fn collect_repetitions(text: &str, out: &mut Vec) { + let mut seen: Vec = Vec::new(); + for (start, end) in sentence_spans(text) { + let sentence = &text[start..end]; + let normalized = sentence + .split_whitespace() + .map(|w| w.to_ascii_lowercase()) + .collect::>() + .join(" "); + if normalized.split(' ').count() < 3 { + continue; + } + if seen.contains(&normalized) { + out.push(Finding { + class: PathologyClass::Repetition, + excerpt: sentence.trim().to_string(), + start, + end, + strippable: false, + }); + } else { + seen.push(normalized); + } + } +} + +/// Byte spans of sentence-ish segments: split after `.`, `!`, `?`, or a +/// newline. Good enough for duplicate detection; not a real segmenter. +fn sentence_spans(text: &str) -> Vec<(usize, usize)> { + let mut spans = Vec::new(); + let mut start = 0usize; + for (i, ch) in text.char_indices() { + if matches!(ch, '.' | '!' | '?' | '\n') { + let end = i + ch.len_utf8(); + if text[start..end].trim().is_empty() { + start = end; + } else { + spans.push((start, end)); + start = end; + } + } + } + if !text[start..].trim().is_empty() { + spans.push((start, text.len())); + } + spans +} + +/// Flag each run of consecutive emoji/decoration codepoints (spaces between +/// them included) as one `EmojiDecoration` finding. Detect-only. +fn collect_emoji_runs(text: &str, out: &mut Vec) { + let mut run_start: Option = None; + let mut run_end = 0usize; + for (i, ch) in text.char_indices() { + if is_decoration(ch) { + if run_start.is_none() { + run_start = Some(i); + } + run_end = i + ch.len_utf8(); + } else if !(ch == ' ' && run_start.is_some()) { + if let Some(s) = run_start.take() { + push_emoji_finding(text, s, run_end, out); + } + } + } + if let Some(s) = run_start { + push_emoji_finding(text, s, run_end, out); + } +} + +fn push_emoji_finding(text: &str, start: usize, end: usize, out: &mut Vec) { + out.push(Finding { + class: PathologyClass::EmojiDecoration, + excerpt: text[start..end].to_string(), + start, + end, + strippable: false, + }); +} + +fn is_decoration(ch: char) -> bool { + matches!(u32::from(ch), + 0x1F300..=0x1FAFF // pictographs, emoticons, symbols + | 0x2600..=0x27BF // misc symbols, dingbats + | 0x2190..=0x21FF // arrows used as decoration + | 0x2B00..=0x2BFF // stars, more arrows + | 0xFE0F // variation selector-16 + | 0x200D // zero-width joiner + ) +} diff --git a/verbosity-compressor/src/main.rs b/verbosity-compressor/src/main.rs new file mode 100644 index 0000000..e06551f --- /dev/null +++ b/verbosity-compressor/src/main.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MPL-2.0 +//! CLI for vex-verbosity-compressor. +//! +//! `analyse FILE` — print a JSON analysis (findings + LPS proxy) to stdout. +//! `compress FILE` — print the compressed text to stdout (`--report` prints +//! a JSON report instead). `-` reads stdin. +//! +//! Exit codes: 0 success, 1 usage or I/O error. + +use std::io::Read; +use std::process::ExitCode; + +use vex_verbosity_compressor::{analyse, compress}; + +const USAGE: &str = "usage: vex-verbosity-compressor [--report] + analyse print JSON findings and the LPS proxy score + compress print the compressed text (--report: JSON report instead) + --version print the version"; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + if args.iter().any(|a| a == "--version") { + println!("vex-verbosity-compressor {}", env!("CARGO_PKG_VERSION")); + return ExitCode::SUCCESS; + } + let (command, path, report_flag) = match args.as_slice() { + [c, p] => (c.as_str(), p.as_str(), false), + [c, p, flag] if flag == "--report" => (c.as_str(), p.as_str(), true), + _ => { + eprintln!("{USAGE}"); + return ExitCode::FAILURE; + } + }; + + let text = match read_input(path) { + Ok(t) => t, + Err(e) => { + eprintln!("vex-verbosity-compressor: cannot read {path}: {e}"); + return ExitCode::FAILURE; + } + }; + + match command { + "analyse" | "analyze" => { + let analysis = analyse(&text); + println!( + "{}", + serde_json::to_string_pretty(&analysis).expect("analysis serializes") + ); + ExitCode::SUCCESS + } + "compress" => { + let result = compress(&text); + if report_flag { + println!( + "{}", + serde_json::to_string_pretty(&result).expect("report serializes") + ); + } else { + println!("{}", result.output); + } + ExitCode::SUCCESS + } + _ => { + eprintln!("{USAGE}"); + ExitCode::FAILURE + } + } +} + +fn read_input(path: &str) -> std::io::Result { + if path == "-" { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + Ok(buf) + } else { + std::fs::read_to_string(path) + } +} diff --git a/verbosity-compressor/tests/compressor_tests.rs b/verbosity-compressor/tests/compressor_tests.rs new file mode 100644 index 0000000..18ef99a --- /dev/null +++ b/verbosity-compressor/tests/compressor_tests.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Integration tests for vex-verbosity-compressor. The first fixture pair is +//! taken verbatim from the LPS section of `vexometer/docs/METRICS.adoc`, so +//! the crate's behaviour is pinned to the protocol's own example. + +use vex_verbosity_compressor::{analyse, compress, detect, PathologyClass}; + +const METRICS_HIGH_LPS: &str = "That's a great question! I'd be happy to help you with that. \ +Essentially, what you're asking about is basically the borrow checker."; + +const METRICS_LOW_LPS: &str = "The function returns null when the input is empty."; + +#[test] +fn metrics_adoc_example_pair_orders_correctly() { + let high = analyse(METRICS_HIGH_LPS); + let low = analyse(METRICS_LOW_LPS); + assert!( + high.lps_proxy > low.lps_proxy, + "HIGH-LPS example ({}) must outscore LOW-LPS example ({})", + high.lps_proxy, + low.lps_proxy + ); + assert_eq!(low.lps_proxy, 0.0, "the LOW-LPS example is pathology-free"); + assert!(low.findings.is_empty()); +} + +#[test] +fn detects_sycophancy_openers() { + let findings = detect("That's a great question! Here is the fix."); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].class, PathologyClass::Sycophancy); + assert_eq!(findings[0].excerpt, "That's a great question"); + assert!(findings[0].strippable); +} + +#[test] +fn longest_match_wins_over_embedded_phrase() { + // "great question" is embedded in "that's a great question"; only the + // longer match may be reported. + let findings = detect("That's a great question."); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].excerpt, "That's a great question"); +} + +#[test] +fn word_boundaries_prevent_substring_hits() { + // "questionnaire" must not fire the "question"-suffixed patterns, and + // "maybes" must not fire the hedge "maybe". + assert!(detect("The questionnaire covers maybes and essentials.").is_empty()); +} + +#[test] +fn hedges_are_detected_but_never_stripped() { + let text = "Perhaps the cache is stale."; + let findings = detect(text); + assert_eq!(findings.len(), 1); + assert_eq!(findings[0].class, PathologyClass::Hedge); + assert!(!findings[0].strippable); + + let result = compress(text); + assert_eq!(result.output, text, "hedges are meaning-bearing"); + assert_eq!(result.retained.len(), 1); + assert!(result.removed.is_empty()); +} + +#[test] +fn corporate_speak_strips_only_discourse_adverbials() { + // "at the end of the day" is removable; "leverage" is sentence grammar. + let result = compress("At the end of the day, we leverage the cache."); + assert_eq!(result.output, "we leverage the cache."); + assert_eq!(result.removed.len(), 1); + assert_eq!(result.removed[0].class, PathologyClass::CorporateSpeak); + assert_eq!(result.retained.len(), 1); + assert_eq!(result.retained[0].excerpt, "leverage"); +} + +#[test] +fn padding_fillers_are_stripped_in_place() { + let result = compress("It's basically a cache."); + assert_eq!(result.output, "It's a cache."); + assert_eq!(result.removed.len(), 1); + assert_eq!(result.removed[0].class, PathologyClass::Padding); +} + +#[test] +fn compress_strips_the_metrics_high_lps_example_down_to_content() { + let result = compress(METRICS_HIGH_LPS); + assert_eq!( + result.output, "what you're asking about is the borrow checker.", + "sycophancy and padding go; content stays (v0.1 does not recapitalise)" + ); + assert_eq!(result.removed.len(), 4); +} + +#[test] +fn compress_preserves_clean_text() { + let result = compress(METRICS_LOW_LPS); + assert_eq!(result.output, METRICS_LOW_LPS); + assert!(result.removed.is_empty()); + assert!(result.retained.is_empty()); +} + +#[test] +fn repetition_flags_second_occurrence_only_and_is_not_stripped() { + let text = "The cache is stale. Refresh it. The cache is stale."; + let findings = detect(text); + let reps: Vec<_> = findings + .iter() + .filter(|f| f.class == PathologyClass::Repetition) + .collect(); + assert_eq!(reps.len(), 1); + assert!(reps[0].start > 20, "only the later duplicate is flagged"); + assert_eq!(compress(text).output, text); +} + +#[test] +fn emoji_run_is_one_finding_and_not_stripped() { + let text = "Tests pass ✅ 🚀 on main."; + let findings = detect(text); + let emoji: Vec<_> = findings + .iter() + .filter(|f| f.class == PathologyClass::EmojiDecoration) + .collect(); + assert_eq!(emoji.len(), 1); + assert_eq!(emoji[0].excerpt, "✅ 🚀"); + assert_eq!(compress(text).output, text); +} + +#[test] +fn compress_preserves_untouched_formatting_byte_for_byte() { + // No findings anywhere: indentation, double spaces, and blank-line runs + // must survive unchanged — compress may not act as a global reformatter. + let text = "fn main() {\n let a = 1;\n\n\n let b = 2; // spaced\n}\n"; + let result = compress(text); + assert!(result.removed.is_empty()); + assert_eq!(result.output, text); +} + +#[test] +fn removing_a_whole_line_filler_drops_the_line() { + let result = compress("The fix works.\nBasically.\nDone."); + assert_eq!(result.output, "The fix works.\nDone."); +} + +#[test] +fn removal_at_end_of_text_leaves_no_dangling_space() { + let result = compress("Foo bar. Great question!"); + assert_eq!(result.output, "Foo bar."); +} + +#[test] +fn indentation_survives_a_removal_elsewhere_in_the_text() { + let result = compress("Basically, run:\n cargo test\n"); + assert_eq!(result.output, "run:\n cargo test\n"); +} + +#[test] +fn empty_input_scores_zero_and_compresses_to_empty() { + let analysis = analyse(""); + assert_eq!(analysis.word_count, 0); + assert_eq!(analysis.lps_proxy, 0.0); + assert_eq!(compress("").output, ""); +} + +#[test] +fn weighted_sum_matches_class_weights() { + // One sycophancy (3.0) + one padding (2.0) = 5.0 over the word count. + let text = "Great question. It's basically a cache."; + let analysis = analyse(text); + assert_eq!(analysis.weighted_sum, 5.0); + assert_eq!(analysis.word_count, 6); + assert!((analysis.lps_proxy - 5.0 / 6.0).abs() < 1e-12); +} + +#[test] +fn compress_is_stable_on_typical_output() { + let once = compress(METRICS_HIGH_LPS).output; + let twice = compress(&once).output; + assert_eq!(once, twice, "second pass finds nothing more to remove"); +} + +#[test] +fn analysis_serializes_to_json_with_expected_keys() { + let value = serde_json::to_value(analyse(METRICS_HIGH_LPS)).expect("serializes"); + for key in ["findings", "word_count", "weighted_sum", "lps_proxy"] { + assert!(value.get(key).is_some(), "missing key {key}"); + } + let first = &value["findings"][0]; + for key in ["class", "excerpt", "start", "end", "strippable"] { + assert!(first.get(key).is_some(), "finding missing key {key}"); + } +} diff --git a/verbosity-compressor/vexometer/EFFICACY.adoc b/verbosity-compressor/vexometer/EFFICACY.adoc new file mode 100644 index 0000000..376be99 --- /dev/null +++ b/verbosity-compressor/vexometer/EFFICACY.adoc @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) Jonathan D.A. Jewell += Efficacy Report: vex-verbosity-compressor +:toc: + +== Summary + +This document tracks the measured effectiveness of vex-verbosity-compressor +in reducing its target vexometer metrics. *No evaluation has been run yet*: +this file carries no numbers until one has, per the efficacy protocol +(`../../vexometer/docs/EFFICACY-PROTOCOL.adoc`, v2.1) — a claim without a +trace is not evidence. + +== Target Metrics + +[cols="1,2,1"] +|=== +| Metric | Description | Target Reduction + +| LPS +| Linguistic Pathology Score: `weighted_sum(pathology_instances) / + response_length` — verbal tics, padding, sycophancy, filler +| Not yet set — established after the first baseline evaluation + +| TII +| Temporal Intrusion Index: `(intrusion_events / total_interactions) * + severity_weight` — reader time consumed by padding +| Not yet set — requires interaction-level traces (not wired up) +|=== + +== Methodology + +=== Test Corpus + +*None exists yet.* Ruling f1 of the efficacy protocol makes registering the +first scenario partition in `../../vexometer/data/scenario_sets/registry.json` +a precondition for the first satellite evaluation; that registry ships empty +as of 2026-09-02. Building the corpus and registering the partition is the +v0.3 roadmap item. + +=== Measurement Process (planned) + +Both traces MUST be generated over the same registered scenario partition +(ruling f1) with identical scenario IDs — a before/after pair drawn from +different scenario sets measures the sets, not the compressor. + +1. Generate a vexometer trace for the original content (`before/`) +2. Apply vex-verbosity-compressor +3. Generate a vexometer trace for the modified content (`after/`) +4. Evaluate with `just efficacy-report` (vexometer-efficacy, protocol v2.1) + +== Results + +None. This section stays empty until the first evaluation runs under the +protocol with a registered scenario partition. + +== Limitations + +* The crate's LPS score is a documented proxy (word-denominated); the + canonical LPS comes from the vexometer Ada core. +* Stripping is deliberately conservative: hedges, repetitions, and emoji are + detected but never removed, so measured LPS reduction will understate + detection coverage. +* Pattern-table detection cannot see paraphrased or novel filler. + +== Future Work + +* Register the first scenario partition (ruling f1) and run the baseline +* Wire vexometer-trace-v1 emission so before/after traces are first-class +* Interaction-level traces for a real TII measurement diff --git a/vexometer-efficacy/.trust/trust-manifest.sha256 b/vexometer-efficacy/.trust/trust-manifest.sha256 index a8ed9bc..e629b45 100644 --- a/vexometer-efficacy/.trust/trust-manifest.sha256 +++ b/vexometer-efficacy/.trust/trust-manifest.sha256 @@ -1,8 +1,8 @@ # trust-manifest v1 # component=vexometer-efficacy -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z df1c5e511fc5e2e7cbcd094fea57728ce193ee267a267037f4043ae5edbf3279 README.adoc -272d7670bb46837c46db61d519d849c60314fecaaf3cd807ee9fdd18ec76ba35 ROADMAP.adoc +64ef24bf1462c455ac24a564b041656e8ce8cc990ec05d02d39eef8293983d83 ROADMAP.adoc b1245e468709a6c75e530412da6480943bf53c836df0ca108aaf39843886e6cb SECURITY.adoc 9c80ff2e60fdb772a0479b46b140e0ce08e4e37bc39e6d7e257aa3d5d1281d18 contractiles/must/Mustfile 3ac4606620454d844d8f0d0580fe32072a8a0b6821c93a74df64c3ed597e3640 contractiles/trust/Trustfile.a2ml diff --git a/vexometer-efficacy/ROADMAP.adoc b/vexometer-efficacy/ROADMAP.adoc index b6564fe..1e2b777 100644 --- a/vexometer-efficacy/ROADMAP.adoc +++ b/vexometer-efficacy/ROADMAP.adoc @@ -23,8 +23,10 @@ == Later -* [ ] Wire into satellite CI once the first satellite exists (D6 — - owner-scoped decision) +* [ ] Wire into satellite CI now that the first satellite exists (D6 + ruled 2026-09-02: `verbosity-compressor/`, scaffolded from + `satellite-template`; evaluation still blocked on the f1 + scenario-partition precondition) * [ ] Frontier dashboard feed (ten small multiples) from `frontier/*.json` * [ ] Statistical hardening: bootstrap CIs on `gap_closed` when diff --git a/vexometer-satellites/.trust/trust-manifest.sha256 b/vexometer-satellites/.trust/trust-manifest.sha256 index a16ed86..e80fe17 100644 --- a/vexometer-satellites/.trust/trust-manifest.sha256 +++ b/vexometer-satellites/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=vexometer-satellites -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z 514f49d458056098d755828ec84a3a1efb0d72fbe13d644e0a7082524b2d4f34 README.adoc 1aa88e3ef6646aeb9209093c8719198e7d8101773ae02c270664bfc1e299c0a0 ROADMAP.adoc 1e36494d91d91559cdf1b4937173e14342e00291319ab1ccb75ce56410978fb8 SECURITY.adoc d1f96168c1500cdd3a753ce8e31af5b7cdcb21d6474e2956b7c6f6d31f020d41 contractiles/must/Mustfile c431753af3d9ada759b3b89b7f34ff83cc1f27478e0bdefd9608c69d8dc83526 contractiles/trust/Trustfile.a2ml +498487f3a2ccd1a4cf23404ddb0811c8407bf9684acc71ebb7d2722a46a73acc contractiles/dust/Dustfile b7d81cab79159aa4d8dc3b375e4aca7b45ef3130d374b4234f5f6d2ce89e9667 RSR_OUTLINE.adoc 2c910784857765919315fbb9a580e974cf5d0e6d3fff3c0187bd764ba799df6a docs/CITATIONS.adoc diff --git a/vexometer/.trust/trust-manifest.sha256 b/vexometer/.trust/trust-manifest.sha256 index 415249f..86dc2b3 100644 --- a/vexometer/.trust/trust-manifest.sha256 +++ b/vexometer/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=vexometer -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z a0a0bd752f3f42324813e305648ee4a248b664516b17ef89e05d51b302f7765f README.adoc 394cb48ee253a28d448761e0714f26af0a38cb8ec408968d1220419b1035b3e7 ROADMAP.adoc f4358bff9ed297a634d7cef1d5efcc059de238d3bf6a73abd11f9e7910785d61 SECURITY.adoc 29db6e0287244a2668bc3a143c8b81a54a9ae762b98cd9683ee4c7b203a1cb8e contractiles/must/Mustfile 5f5e879cc11595c1f4228c41f3bbde5177584316d60f695211b7812b8de0565e contractiles/trust/Trustfile.a2ml +0c78e02a5625fb5ddfc5dcc39e4cda939852ce5b92acb88b30cf9e7472b4134f contractiles/dust/Dustfile 3d4f9ee78b526a6a417bf0cd51a32ff8e156279a80e31ade369161b3a5136844 RSR_OUTLINE.adoc 2db573d850fc0c599b20d290a0e6cade9854bf0b3bcd76d8d9322b07f77ad6f8 docs/CITATIONS.adoc diff --git a/vext-email-gateway/.trust/trust-manifest.sha256 b/vext-email-gateway/.trust/trust-manifest.sha256 index 98f3768..6014e15 100644 --- a/vext-email-gateway/.trust/trust-manifest.sha256 +++ b/vext-email-gateway/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=vext-email-gateway -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z 0a89e3a89fa118fcb5833e485500900299bb5abce8955c25ab05e249a952ddf8 README.adoc 7d719f633a950e111d76d0d8cc297cc18704a858a3cb40900adce7456362e283 ROADMAP.adoc 1dea9d1375b2dfe806d795edef96d4e4e5465234ddb523c448033eea49ff0d91 SECURITY.adoc cf2c0a7507d11c87a441bb0fe6d42679640a6fb351c576405c71135d9c7d103c contractiles/must/Mustfile 37dd1771e66bfd80abb4b8af93d118f58bfe88d3cdb6da9bcc7f2eeab8e5174e contractiles/trust/Trustfile.a2ml +498487f3a2ccd1a4cf23404ddb0811c8407bf9684acc71ebb7d2722a46a73acc contractiles/dust/Dustfile 10e55df8662170ec24344ebe3193b46ca723278567cec1be76d784d73bfbaf99 RSR_OUTLINE.adoc 46962d07275f8323676c4fcd8179220e70267d274e2f3261125431d5f1eea511 docs/CITATIONS.adoc diff --git a/vext/.trust/trust-manifest.sha256 b/vext/.trust/trust-manifest.sha256 index a3e2cf4..7c8f096 100644 --- a/vext/.trust/trust-manifest.sha256 +++ b/vext/.trust/trust-manifest.sha256 @@ -1,10 +1,11 @@ # trust-manifest v1 # component=vext -# generated_at=2026-09-01T23:29:45Z +# generated_at=2026-09-02T01:16:19Z 2e36c7c5522a306c38b5a69246abe25a6662963085ae68c8a70f9a7736412383 README.adoc 59afd091f80638f64eb4028cea7252b8e1030165e4e826059e5a05940c092daa ROADMAP.adoc ddfd1260b0c4a0cf3c812fb8278892a7ea3c784cde40aeff1b34529d1bade9ca SECURITY.adoc 309e397802d6c9ba2548576f6bd999b6803eb6f1127b4671f955eeb356d480c7 contractiles/must/Mustfile 64dac543db4768b1425b68762ae4b02501dab7b03c0e87ad153ee7a237420b66 contractiles/trust/Trustfile.a2ml +498487f3a2ccd1a4cf23404ddb0811c8407bf9684acc71ebb7d2722a46a73acc contractiles/dust/Dustfile d9c9872426ee285c6c9ecf80bdcddbd109272a67b28ddfc43710c7dbae753a66 RSR_OUTLINE.adoc 41d7687263da0fb50258cf32e90397156b0e0cb2cea43eebf47bba8ae16dcd21 docs/CITATIONS.adoc