From 840c88d6bf9413d6b66c40cadb6e9678b87e6689 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 10:18:34 -0700 Subject: [PATCH 01/24] bd init: initialize beads issue tracking --- .beads/.gitignore | 77 +++++ .beads/README.md | 81 +++++ .beads/config.yaml | 68 ++++ .beads/hooks/applypatch-msg | 3 + .beads/hooks/commit-msg | 3 + .beads/hooks/post-applypatch | 3 + .beads/hooks/post-checkout | 36 ++ .beads/hooks/post-commit | 3 + .beads/hooks/post-merge | 36 ++ .beads/hooks/post-rewrite | 3 + .beads/hooks/pre-applypatch | 3 + .beads/hooks/pre-auto-gc | 3 + .beads/hooks/pre-commit | 101 ++++++ .beads/hooks/pre-merge-commit | 3 + .beads/hooks/pre-push | 36 ++ .beads/hooks/pre-rebase | 3 + .beads/hooks/prepare-commit-msg | 36 ++ .beads/interactions.jsonl | 0 .beads/metadata.json | 7 + .gitignore | 6 + AGENTS.md | 574 ++++++++++++++++++++++++++++---- CLAUDE.md | 88 +++-- 22 files changed, 1073 insertions(+), 100 deletions(-) create mode 100644 .beads/.gitignore create mode 100644 .beads/README.md create mode 100644 .beads/config.yaml create mode 100755 .beads/hooks/applypatch-msg create mode 100755 .beads/hooks/commit-msg create mode 100755 .beads/hooks/post-applypatch create mode 100755 .beads/hooks/post-checkout create mode 100755 .beads/hooks/post-commit create mode 100755 .beads/hooks/post-merge create mode 100755 .beads/hooks/post-rewrite create mode 100755 .beads/hooks/pre-applypatch create mode 100755 .beads/hooks/pre-auto-gc create mode 100755 .beads/hooks/pre-commit create mode 100755 .beads/hooks/pre-merge-commit create mode 100755 .beads/hooks/pre-push create mode 100755 .beads/hooks/pre-rebase create mode 100755 .beads/hooks/prepare-commit-msg create mode 100644 .beads/interactions.jsonl create mode 100644 .beads/metadata.json diff --git a/.beads/.gitignore b/.beads/.gitignore new file mode 100644 index 0000000..f773858 --- /dev/null +++ b/.beads/.gitignore @@ -0,0 +1,77 @@ +# Dolt database (managed by Dolt, not git) +dolt/ +embeddeddolt/ +proxieddb/ + +# Runtime files +bd.sock +bd.sock.startlock +sync-state.json +last-touched +.exclusive-lock + +# Daemon runtime (lock, log, pid) +daemon.* + +# Push state (runtime, per-machine) +push-state.json + +# Lock files (various runtime locks) +*.lock + +# Credential key (encryption key for federation peer auth โ€” never commit) +.beads-credential-key + +# Local version tracking (prevents upgrade notification spam after git ops) +.local_version + +proxied_server_client_info.json + +# Worktree redirect file (contains relative path to main repo's .beads/) +# Must not be committed as paths would be wrong in other clones +redirect + +# Sync state (local-only, per-machine) +# These files are machine-specific and should not be shared across clones +.sync.lock +export-state/ +export-state.json +last_pull + +# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned) +ephemeral.sqlite3 +ephemeral.sqlite3-journal +ephemeral.sqlite3-wal +ephemeral.sqlite3-shm + +# Dolt server management (auto-started by bd) +dolt-server.pid +dolt-server.log +dolt-server.lock +dolt-server.port +dolt-server.activity + +# Debug-mode pprof artifacts (written when dolt.debug: true in config.yaml) +dolt-pprof/ + +# Corrupt backup directories (created by bd doctor --fix recovery) +*.corrupt.backup/ + +# Backup data (auto-exported JSONL, local-only) +backup/ + +# Per-project environment file (Dolt connection config, GH#2520) +.env + +# Legacy files (from pre-Dolt versions) +*.db +*.db?* +*.db-journal +*.db-wal +*.db-shm +db.sqlite +bd.db +# NOTE: Do NOT add negation patterns here. +# They would override fork protection in .git/info/exclude. +# Config files (metadata.json, config.yaml) are tracked by git by default +# since no pattern above ignores them. diff --git a/.beads/README.md b/.beads/README.md new file mode 100644 index 0000000..63e8f4c --- /dev/null +++ b/.beads/README.md @@ -0,0 +1,81 @@ +# Beads - AI-Native Issue Tracking + +Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code. + +## What is Beads? + +Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git. + +**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads) + +## Quick Start + +### Essential Commands + +```bash +# Create new issues +bd create "Add user authentication" + +# View all issues +bd list + +# View issue details +bd show + +# Update issue status +bd update --claim +bd update --status done + +# Sync with Dolt remote +bd dolt push +``` + +### Working with Issues + +Issues in Beads are: +- **Git-native**: Stored in Dolt database with version control and branching +- **AI-friendly**: CLI-first design works perfectly with AI coding agents +- **Branch-aware**: Issues can follow your branch workflow +- **Sync-ready**: Uses Dolt remotes for backup and team sharing + +## Why Beads? + +โœจ **AI-Native Design** +- Built specifically for AI-assisted development workflows +- CLI-first interface works seamlessly with AI coding agents +- No context switching to web UIs + +๐Ÿš€ **Developer Focused** +- Issues live in your repo, right next to your code +- Works offline, syncs when you push +- Fast, lightweight, and stays out of your way + +๐Ÿ”ง **Git Integration** +- Dolt-native sync via bd dolt push / bd dolt pull +- Branch-aware issue tracking +- Dolt-native three-way merge resolution + +## Get Started with Beads + +Try Beads in your own projects: + +```bash +# Install Beads +curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash + +# Initialize in your repo +bd init + +# Create your first issue +bd create "Try out Beads" +``` + +## Learn More + +- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs) +- **Quick Start Guide**: Run `bd quickstart` +- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples) + +--- + +*Beads: Issue tracking that moves at the speed of thought* โšก diff --git a/.beads/config.yaml b/.beads/config.yaml new file mode 100644 index 0000000..af0fa36 --- /dev/null +++ b/.beads/config.yaml @@ -0,0 +1,68 @@ +# Beads Configuration File +# This file configures default behavior for all bd commands in this repository +# All settings can also be set via environment variables (BD_* prefix) +# or overridden with command-line flags + +# Issue prefix for this repository (used by bd init) +# If not set, bd init will auto-detect from directory name +# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc. +# issue-prefix: "" + +# Use no-db mode: JSONL-only, no Dolt database +# When true, .beads/issues.jsonl is the only local store +# no-db: false + +# Enable JSON output by default +# json: false + +# Feedback title formatting for mutating commands (create/update/close/dep/edit) +# 0 = hide titles, N > 0 = truncate to N characters +# output: +# title-length: 255 + +# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) +# actor: "" + +# Export events (audit trail) to .beads/events.jsonl on each flush/sync +# When enabled, new events are appended incrementally using a high-water mark. +# Use 'bd export --events' to trigger manually regardless of this setting. +# events-export: false + +# Multi-repo configuration (experimental - bd-307) +# Allows hydrating from multiple repositories and routing writes to the correct database +# repos: +# primary: "." # Primary repo (where this database lives) +# additional: # Additional repos to hydrate from (read-only) +# - ~/beads-planning # Personal planning repo +# - ~/work-planning # Work planning repo + +# Dolt-native backup (periodic backup for off-machine recovery) +# This is full database backup only. Cross-machine sync uses Dolt remotes. +# backup: +# enabled: false # Disable auto-backup entirely +# interval: 15m # Minimum time between auto-backups +# git-push: false # Disable git push (backup locally only) +# git-repo: "" # Separate git repo for backups (default: project repo) + +# Optional JSONL auto-export for viewers, interchange, and issue-level migration. +# Disabled by default; enable only when an integration needs fresh .beads/issues.jsonl. +# Use relative paths under .beads/ for JSONL import/export filenames. +# export: +# auto: false +# path: issues.jsonl +# interval: 60s +# git-add: false +# import: +# path: issues.jsonl + +# Integration settings (access with 'bd config get/set') +# Non-secret keys (stored in the database): +# - jira.url, jira.project +# - linear.team_id +# - github.org, github.repo +# +# Secret keys (stored in this file but prefer env vars to avoid git exposure): +# - linear.api_key โ†’ use LINEAR_API_KEY env var instead +# - github.token โ†’ use GITHUB_TOKEN env var instead + +sync.remote: "git+https://github.com/pacphi/emailibrium.git" \ No newline at end of file diff --git a/.beads/hooks/applypatch-msg b/.beads/hooks/applypatch-msg new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/applypatch-msg @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/commit-msg b/.beads/hooks/commit-msg new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/commit-msg @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-applypatch b/.beads/hooks/post-applypatch new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/post-applypatch @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-checkout b/.beads/hooks/post-checkout new file mode 100755 index 0000000..c66cbfc --- /dev/null +++ b/.beads/hooks/post-checkout @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" + +# --- BEGIN BEADS INTEGRATION v1.1.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + _bd_used_perl=0 + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run post-checkout "$@" + _bd_exit=$? + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$_bd_timeout" bd hooks run post-checkout "$@" + _bd_exit=$? + elif command -v perl >/dev/null 2>&1; then + _bd_used_perl=1 + perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run post-checkout "$@" + _bd_exit=$? + else + echo >&2 "beads: hook 'post-checkout' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" + bd hooks run post-checkout "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then + echo >&2 "beads: hook 'post-checkout' timed out after ${_bd_timeout}s โ€” continuing without beads" + _bd_exit=0 + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized โ€” skipping hook 'post-checkout'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/post-commit b/.beads/hooks/post-commit new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/post-commit @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-merge b/.beads/hooks/post-merge new file mode 100755 index 0000000..18216de --- /dev/null +++ b/.beads/hooks/post-merge @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" + +# --- BEGIN BEADS INTEGRATION v1.1.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + _bd_used_perl=0 + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run post-merge "$@" + _bd_exit=$? + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$_bd_timeout" bd hooks run post-merge "$@" + _bd_exit=$? + elif command -v perl >/dev/null 2>&1; then + _bd_used_perl=1 + perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run post-merge "$@" + _bd_exit=$? + else + echo >&2 "beads: hook 'post-merge' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" + bd hooks run post-merge "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then + echo >&2 "beads: hook 'post-merge' timed out after ${_bd_timeout}s โ€” continuing without beads" + _bd_exit=0 + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized โ€” skipping hook 'post-merge'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/post-rewrite b/.beads/hooks/post-rewrite new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/post-rewrite @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-applypatch b/.beads/hooks/pre-applypatch new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/pre-applypatch @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-auto-gc b/.beads/hooks/pre-auto-gc new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/pre-auto-gc @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-commit b/.beads/hooks/pre-commit new file mode 100755 index 0000000..3a721b1 --- /dev/null +++ b/.beads/hooks/pre-commit @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Pre-commit gate โ€” mirrors the GitHub CI workflow quality checks so +# violations are caught locally before they ever reach CI. +# +# Fails fast on the first error so the user gets back to a shell prompt +# instead of hanging. Missing tools = hard failure with an install hint. +# Bash 3.2 compatible (macOS default). + +set -euo pipefail + +# 1) Prettier + ESLint on staged files (via lint-staged + package.json). +pnpm lint-staged + +staged_list() { git diff --cached --name-only --diff-filter=ACMR -- "$@" | tr '\n' '\0' | xargs -0 -I{} echo "{}"; } + +STAGED_MD=$(git diff --cached --name-only --diff-filter=ACMR -- '*.md') +STAGED_YAML=$(git diff --cached --name-only --diff-filter=ACMR -- '*.yaml' '*.yml' | grep -v 'pnpm-lock.yaml$' || true) +STAGED_SH=$(git diff --cached --name-only --diff-filter=ACMR -- '*.sh') +STAGED_BACKEND=$(git diff --cached --name-only --diff-filter=ACMR -- 'backend/') + +# 2) Markdownlint (if .md staged). +if [ -n "$STAGED_MD" ]; then + if ! command -v markdownlint-cli2 >/dev/null 2>&1; then + echo "โœ— markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2" >&2 + exit 1 + fi + COUNT=$(echo "$STAGED_MD" | wc -l | tr -d ' ') + echo "โ†’ markdownlint on $COUNT staged .md file(s)..." + echo "$STAGED_MD" | xargs markdownlint-cli2 +fi + +# 3) yamllint (if .yaml/.yml staged, excluding lockfiles). +if [ -n "$STAGED_YAML" ]; then + if ! command -v yamllint >/dev/null 2>&1; then + echo "โœ— yamllint not installed. Run: pip install yamllint" >&2 + exit 1 + fi + COUNT=$(echo "$STAGED_YAML" | wc -l | tr -d ' ') + echo "โ†’ yamllint on $COUNT staged YAML file(s)..." + echo "$STAGED_YAML" | xargs yamllint -c .yamllint.yaml +fi + +# 4) shellcheck (if .sh staged). +if [ -n "$STAGED_SH" ]; then + if ! command -v shellcheck >/dev/null 2>&1; then + echo "โœ— shellcheck not installed. Run: brew install shellcheck" >&2 + exit 1 + fi + COUNT=$(echo "$STAGED_SH" | wc -l | tr -d ' ') + echo "โ†’ shellcheck on $COUNT staged shell script(s)..." + echo "$STAGED_SH" | xargs shellcheck +fi + +# 5) Rust fmt + clippy (if backend/ touched). Matches CI strictness: deny +# all warnings but allow dead_code/unused_* so pre-existing noise does +# not block commits. +if [ -n "$STAGED_BACKEND" ]; then + echo "โ†’ cargo fmt + clippy on backend/..." + ( + cd backend + cargo fmt --package emailibrium -- --check + cargo clippy -p emailibrium --all-targets -- \ + -D warnings \ + -A dead_code -A unused_variables -A unused_imports -A unused_mut + ) +fi + +echo "โœ“ pre-commit checks passed." + +# --- BEGIN BEADS INTEGRATION v1.1.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + _bd_used_perl=0 + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run pre-commit "$@" + _bd_exit=$? + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$_bd_timeout" bd hooks run pre-commit "$@" + _bd_exit=$? + elif command -v perl >/dev/null 2>&1; then + _bd_used_perl=1 + perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run pre-commit "$@" + _bd_exit=$? + else + echo >&2 "beads: hook 'pre-commit' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" + bd hooks run pre-commit "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then + echo >&2 "beads: hook 'pre-commit' timed out after ${_bd_timeout}s โ€” continuing without beads" + _bd_exit=0 + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized โ€” skipping hook 'pre-commit'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/pre-merge-commit b/.beads/hooks/pre-merge-commit new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/pre-merge-commit @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-push b/.beads/hooks/pre-push new file mode 100755 index 0000000..ecc4b88 --- /dev/null +++ b/.beads/hooks/pre-push @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" + +# --- BEGIN BEADS INTEGRATION v1.1.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + _bd_used_perl=0 + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run pre-push "$@" + _bd_exit=$? + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$_bd_timeout" bd hooks run pre-push "$@" + _bd_exit=$? + elif command -v perl >/dev/null 2>&1; then + _bd_used_perl=1 + perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run pre-push "$@" + _bd_exit=$? + else + echo >&2 "beads: hook 'pre-push' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" + bd hooks run pre-push "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then + echo >&2 "beads: hook 'pre-push' timed out after ${_bd_timeout}s โ€” continuing without beads" + _bd_exit=0 + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized โ€” skipping hook 'pre-push'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/pre-rebase b/.beads/hooks/pre-rebase new file mode 100755 index 0000000..6aa68db --- /dev/null +++ b/.beads/hooks/pre-rebase @@ -0,0 +1,3 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/prepare-commit-msg b/.beads/hooks/prepare-commit-msg new file mode 100755 index 0000000..af1d640 --- /dev/null +++ b/.beads/hooks/prepare-commit-msg @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. +export PATH="$PWD/node_modules/.bin:$PATH" + +# --- BEGIN BEADS INTEGRATION v1.1.0 --- +# This section is managed by beads. Do not remove these markers. +if command -v bd >/dev/null 2>&1; then + export BD_GIT_HOOK=1 + _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} + _bd_used_perl=0 + if command -v timeout >/dev/null 2>&1; then + timeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + elif command -v perl >/dev/null 2>&1; then + _bd_used_perl=1 + perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + else + echo >&2 "beads: hook 'prepare-commit-msg' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" + bd hooks run prepare-commit-msg "$@" + _bd_exit=$? + fi + if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then + echo >&2 "beads: hook 'prepare-commit-msg' timed out after ${_bd_timeout}s โ€” continuing without beads" + _bd_exit=0 + fi + if [ $_bd_exit -eq 3 ]; then + echo >&2 "beads: database not initialized โ€” skipping hook 'prepare-commit-msg'" + _bd_exit=0 + fi + if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi +fi +# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/.beads/metadata.json b/.beads/metadata.json new file mode 100644 index 0000000..0a62889 --- /dev/null +++ b/.beads/metadata.json @@ -0,0 +1,7 @@ +{ + "database": "dolt", + "backend": "dolt", + "dolt_mode": "embedded", + "dolt_database": "emailibrium", + "project_id": "337e044a-6ee3-4107-aadc-941af8be48fd" +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index e64c31f..e4fb47c 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,9 @@ docker-compose.override.yml # RuVector / ruflo local agent databases -- machine artifacts, not library content agentdb.rvf agentdb.rvf.lock +.autopilot/queued/ + +# Beads / Dolt files (added by bd init) +.dolt/ +.beads-credential-key +.beads/proxieddb/ diff --git a/AGENTS.md b/AGENTS.md index c134a5b..5221fe0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,103 +1,539 @@ - +# emailibrium -# GitNexus โ€” Code Intelligence +> Multi-agent orchestration framework for agentic coding -This project is indexed by GitNexus as **emailibrium** (228117 symbols, 447347 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +## Project Overview -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. +A Claude Flow powered project -## Always Do +**Tech Stack**: TypeScript, Node.js +**Architecture**: Domain-Driven Design with bounded contexts -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol โ€” callers, callees, which execution flows it participates in โ€” use `gitnexus_context({name: "symbolName"})`. +## Quick Start -## When Debugging +### Installation +```bash +npm install +``` + +### Build +```bash +npm run build +``` + +### Test +```bash +npm test +``` + +### Development +```bash +npm run dev +``` + +## Agent Coordination + +### Swarm Configuration + +This project uses hierarchical swarm coordination for complex tasks: + +| Setting | Value | Purpose | +|---------|-------|---------| +| Topology | `hierarchical` | Queen-led coordination (anti-drift) | +| Max Agents | 8 | Optimal team size | +| Strategy | `specialized` | Clear role boundaries | +| Consensus | `raft` | Leader-based consistency | + +### When to Use Swarms + +**Invoke swarm for:** +- Multi-file changes (3+ files) +- New feature implementation +- Cross-module refactoring +- API changes with tests +- Security-related changes +- Performance optimization + +**Skip swarm for:** +- Single file edits +- Simple bug fixes (1-2 lines) +- Documentation updates +- Configuration changes + +### Available Skills + +Use `$skill-name` syntax to invoke: + +| Skill | Use Case | +|-------|----------| +| `$swarm-orchestration` | Multi-agent task coordination | +| `$memory-management` | Pattern storage and retrieval | +| `$sparc-methodology` | Structured development workflow | +| `$security-audit` | Security scanning and CVE detection | +| `$performance-analysis` | Profiling and optimization | +| `$github-automation` | CI/CD and PR management | + +### Agent Types + +| Type | Role | Use Case | +|------|------|----------| +| `researcher` | Requirements analysis | Understanding scope | +| `architect` | System design | Planning structure | +| `coder` | Implementation | Writing code | +| `tester` | Test creation | Quality assurance | +| `reviewer` | Code review | Security and quality | + +## Execution Model + +- **claude-flow** = LEDGER (coordinates: memory, routing, swarm state) +- **Codex** = EXECUTOR (writes code, runs tests, creates files) + +**Critical rule:** DON'T STOP after calling claude-flow commands. Coordination commands return instantly โ€” continue immediately with the next implementation step. + +## Ruflo + Codex Automated Workflow + +Ruflo is the coordination ledger and policy decision point; Codex workers execute code, tests, and commands. A Ruflo coordination call records work but never replaces implementation. + +Use `guidance_brain({ mode: "recommend", task: "..." })` when the task can +benefit from Ruflo-specific capabilities. Its live registry is authoritative +for tool presence; registration alone does not prove configuration, +reachability, health, or authorization. If it is not registered, use compatible +`guidance_recommend`, CLI discovery, and repository instructions. + +1. **Recall** โ€” search AgentDB memory and relevant ADRs for patterns and constraints. +2. **Inspect** โ€” read source, runtime, dependency, policy, and health state. +3. **Route** โ€” choose the smallest capable topology, agents, skills, and tools. +4. **Plan** โ€” define acceptance criteria, safety envelope, ownership, and validation. +5. **Execute** โ€” Codex workers implement in isolated scopes; Ruflo records coordination. +6. **Test** โ€” run focused tests, regression tests, and failure-path checks. +7. **Validate** โ€” check types, security, policy, compatibility, and artifact integrity. +8. **Benchmark** โ€” compare a source-bound candidate with a source-bound baseline. +9. **Optimize** โ€” improve measured bottlenecks without weakening the safety envelope. +10. **Receipt** โ€” bind claims, evidence, and decisions to exact source/build inputs. +11. **Handoff** โ€” reconcile concurrent work and disclose unresolved limitations. +12. **Publish** โ€” only an independently authorized release gate may publish immutable artifacts. + +### Concurrency and authority invariants + +- Never allow two writers in one worktree. +- Read-only research agents may share a checkout; writing agents may not. +- A child may drop capabilities but can never add tools, servers, namespaces, network access, spend, concurrency, or delegation depth. +- Cancel dependent and not-yet-started sibling work when policy denies an action or a required dependency fails. +- MetaHarness may benchmark candidates concurrently, but it cannot promote, serve, or expand its own SafetyEnvelope. +- Only the integration agent changes shared manifests or lockfiles. +- Do not auto-commit, push, merge, release, or delete worktrees unless the user authorized that operation. +- Every consequential action must produce a policy decision receipt; production, destructive, spend, and promotion actions may require human approval. + +### Repository harness adapter -1. `gitnexus_query({query: ""})` โ€” find execution flows related to the issue -2. `gitnexus_context({name: ""})` โ€” see all callers, callees, and process participation -3. `READ gitnexus://repo/emailibrium/process/{processName}` โ€” trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` โ€” see what your branch changed +When tracked repository instructions define a local collaboration harness: -## When Refactoring +1. Assign the isolated worktree before starting a writing session. +2. Start or register the session, inspect current claims, and acquire only the + exact paths, resources, and development ports needed for the task. +3. Renew leases during long work, check acknowledged inbox messages at integration + boundaries, and release claims when handing off or ending. +4. Record focused and integration evidence against the exact source state, + then let the designated integration owner decide release. -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview โ€” graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. +A repository lease coordinates ownership; it does not grant authorization. +In-memory reference adapters demonstrate semantics but are not distributed, +restart-durable release authorities. +The worker still needs the current ADR-324/325 action capability and fencing +epoch for every protected side effect. Heartbeat and lease expiry establish +liveness; a PID is diagnostic only. HEAD alone is not an exact source-state +identity when tracked or untracked changes exist, so a release receipt must +bind a clean commit or an immutable snapshot including those changes. -## Never Do -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace โ€” use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. +## MCP Integration -## Tools Quick Reference +Use MCP tools for coordination, then keep coding: -| Tool | When to use | Command | -| ---------------- | ----------------------------- | ----------------------------------------------------------------------- | -| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | -| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | -| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | -| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | -| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | -| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | +| Tool | Purpose | Example | +|------|---------|---------| +| `swarm_init` | Start coordination | `swarm_init({topology: "hierarchical"})` | +| `memory_store` | Save patterns | `memory_store({key: "auth", value: "JWT"})` | +| `memory_search` | Find patterns | `memory_search({query: "auth patterns"})` | +| `task_orchestrate` | Assign work | `task_orchestrate({task: "implement"})` | -## Impact Risk Levels +## Code Standards -| Depth | Meaning | Action | -| ----- | ------------------------------------- | --------------------- | -| d=1 | WILL BREAK โ€” direct callers/importers | MUST update these | -| d=2 | LIKELY AFFECTED โ€” indirect deps | Should test | -| d=3 | MAY NEED TESTING โ€” transitive | Test if critical path | +### File Organization +- **NEVER** save to root folder +- `/src` - Source code files +- `/tests` - Test files +- `/docs` - Documentation +- `/config` - Configuration files -## Resources +### Quality Rules +- Files under 500 lines +- No hardcoded secrets +- Input validation at boundaries +- Typed interfaces for public APIs +- TDD London School (mock-first) preferred -| Resource | Use for | -| -------------------------------------------- | ---------------------------------------- | -| `gitnexus://repo/emailibrium/context` | Codebase overview, check index freshness | -| `gitnexus://repo/emailibrium/clusters` | All functional areas | -| `gitnexus://repo/emailibrium/processes` | All execution flows | -| `gitnexus://repo/emailibrium/process/{name}` | Step-by-step execution trace | +### Commit Messages +``` +(): + +[optional body] +``` -## Self-Check Before Finishing +Types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `chore` -Before completing any code modification task, verify: +Do not add a `Co-Authored-By` trailer unless the repository explicitly +configures and authorizes that attribution. -1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated +## Security -## Keeping the Index Fresh +### Critical Rules +- NEVER commit secrets, credentials, or .env files +- NEVER hardcode API keys +- Always validate user input +- Use parameterized queries for SQL +- Sanitize output to prevent XSS -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: +### Path Security +- Validate all file paths +- Prevent directory traversal (../) +- Use absolute paths internally +## Memory System + +### Storing Patterns ```bash -npx gitnexus analyze +npx @claude-flow/cli memory store \ + --key "pattern-name" \ + --value "pattern description" \ + --namespace patterns ``` -If the index previously included embeddings, preserve them by adding `--embeddings`: +### Searching Memory +```bash +npx @claude-flow/cli memory search \ + --query "search terms" \ + --namespace patterns +``` + +## Quick Commands ```bash -npx gitnexus analyze --embeddings +npx @claude-flow/cli memory search --query "relevant patterns" +npx @claude-flow/cli hooks route --task "current task description" +npx @claude-flow/cli swarm init --topology hierarchical +npx @claude-flow/cli hooks pre-task --description "task summary" ``` -To check whether embeddings exist, inspect `.gitnexus/meta.json` โ€” the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** +## Links + +- Documentation: https://github.com/ruvnet/ruflo +- Issues: https://github.com/ruvnet/ruflo/issues + +## Performance Targets + +| Metric | Target | Notes | +|--------|--------|-------| +| HNSW Search | 150x-12,500x faster | Vector operations | +| Memory Reduction | 50-75% | Int8 quantization | +| MCP Response | <100ms | API latency | +| CLI Startup | <500ms | Cold start | +| SONA Adaptation | <0.05ms | Neural learning | + +## Testing + +### Running Tests +```bash +# Unit tests +npm test + +# Integration tests +npm run test:integration + +# Coverage +npm run test:coverage + +# Security tests +npm run test:security +``` + +### Test Philosophy +- TDD London School (mock-first) +- Unit tests for business logic +- Integration tests for boundaries +- E2E tests for critical paths +- Security tests for sensitive operations + +### Coverage Requirements +- Minimum 80% line coverage +- 100% coverage for security-critical code +- All public APIs must have tests + +## MCP Integration + +Claude Flow exposes tools via Model Context Protocol: + +```bash +# Start MCP server +npx ruflo mcp start + +# List available tools +npx ruflo mcp tools +``` + +### Available Tools + +| Tool | Purpose | Example | +|------|---------|---------| +| `swarm_init` | Initialize swarm coordination | `swarm_init({topology: "hierarchical"})` | +| `agent_spawn` | Spawn new agents | `agent_spawn({type: "coder", name: "dev-1"})` | +| `memory_store` | Store in AgentDB | `memory_store({key: "pattern", value: "..."})` | +| `memory_search` | Semantic search | `memory_search({query: "auth patterns"})` | +| `task_orchestrate` | Task coordination | `task_orchestrate({task: "implement feature"})` | +| `neural_train` | Train neural patterns | `neural_train({iterations: 10})` | +| `benchmark_run` | Performance benchmarks | `benchmark_run({type: "all"})` | + +## Hooks System + +Claude Flow uses hooks for lifecycle automation: + +### Core Hooks + +| Hook | Trigger | Purpose | +|------|---------|---------| +| `pre-task` | Before task starts | Get context, load patterns | +| `post-task` | After task completes | Record completion, train | +| `pre-edit` | Before file changes | Validate, backup | +| `post-edit` | After file changes | Train patterns, verify | +| `pre-command` | Before shell commands | Security check | +| `post-command` | After shell commands | Log results | + +### Session Hooks + +| Hook | Purpose | +|------|---------| +| `session-start` | Initialize context, load memory | +| `session-end` | Export metrics, consolidate memory | +| `session-restore` | Resume from checkpoint | +| `notify` | Send notifications | -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. +### Intelligence Hooks + +| Hook | Purpose | +|------|---------| +| `route` | Route task to appropriate agents | +| `explain` | Generate explanations | +| `pretrain` | Pre-train neural patterns | +| `build-agents` | Build specialized agents | +| `transfer` | Transfer learning between domains | + +### Example Usage +```bash +# Before starting a task +npx @claude-flow/cli hooks pre-task \ + --description "implementing authentication" + +# After completing a task +npx @claude-flow/cli hooks post-task \ + --task-id "task-123" \ + --success true + +# Route a task to agents +npx @claude-flow/cli hooks route \ + --task "implement OAuth2 login flow" +``` + +## Background Workers + +12 background workers provide continuous optimization: + +| Worker | Priority | Purpose | +|--------|----------|---------| +| `ultralearn` | normal | Deep knowledge acquisition | +| `optimize` | high | Performance optimization | +| `consolidate` | low | Memory consolidation | +| `predict` | normal | Predictive preloading | +| `audit` | critical | Security analysis | +| `map` | normal | Codebase mapping | +| `preload` | low | Resource preloading | +| `deepdive` | normal | Deep code analysis | +| `document` | normal | Auto-documentation | +| `refactor` | normal | Refactoring suggestions | +| `benchmark` | normal | Performance benchmarking | +| `testgaps` | normal | Test coverage analysis | + +### Managing Workers +```bash +# List workers +npx @claude-flow/cli hooks worker list + +# Trigger specific worker +npx @claude-flow/cli hooks worker dispatch --trigger audit + +# Check worker status +npx @claude-flow/cli hooks worker status +``` + +## Intelligence System + +The RuVector Intelligence System provides neural learning: + +### Components +- **SONA**: Self-Optimizing Neural Architecture (<0.05ms adaptation) +- **MoE**: Mixture of Experts for specialized routing +- **HNSW**: Hierarchical Navigable Small World for fast search +- **EWC++**: Elastic Weight Consolidation (prevents forgetting) +- **Flash Attention**: Optimized attention mechanism + +### 4-Step Pipeline +1. **RETRIEVE** - Fetch relevant patterns via HNSW +2. **JUDGE** - Evaluate with verdicts (success/failure) +3. **DISTILL** - Extract key learnings via LoRA +4. **CONSOLIDATE** - Prevent catastrophic forgetting via EWC++ + +## Debugging + +### Log Levels +```bash +# Set log level +export CLAUDE_FLOW_LOG_LEVEL=debug + +# Enable verbose mode +npx @claude-flow/cli --verbose +``` + +### Health Checks +```bash +# Run diagnostics +npx @claude-flow/cli doctor --fix + +# Check system status +npx @claude-flow/cli status +``` + +--- + +# Quality Engineering Standards (Agentic QE) + +## AQE MCP Server + +This project uses Agentic QE for AI-powered quality engineering. The AQE MCP server provides tools for test generation, coverage analysis, quality assessment, and learning. + +## Setup + +Always call `fleet_init` before using other AQE tools to initialize the QE fleet. + +## Available Tools + +### Test Generation +- `test_generate_enhanced` โ€” AI-powered test generation with pattern recognition and anti-pattern detection +- Supports unit, integration, and e2e test types + +### Coverage Analysis +- `coverage_analyze_sublinear` โ€” O(log n) coverage gap detection with ML-powered analysis +- Target: 80% statement coverage minimum, focus on risk-weighted coverage + +### Quality Assessment +- `quality_assess` โ€” Quality gate evaluation with configurable thresholds +- Run before marking tasks complete + +### Security Scanning +- `security_scan_comprehensive` โ€” SAST/DAST vulnerability scanning +- Run after changes to auth, security, or middleware code + +### Defect Prediction +- `defect_predict` โ€” AI analysis of code complexity and change history + +### Learning & Memory +- `memory_store` โ€” Store patterns and learnings for future reference +- `memory_query` โ€” Query past patterns before starting work +- Always store successful patterns after task completion + +## Best Practices + +1. **Test Pyramid**: 70% unit, 20% integration, 10% e2e +2. **AAA Pattern**: Arrange-Act-Assert for clear test structure +3. **One assertion per test**: Test one behavior at a time +4. **Descriptive names**: `should_returnValue_when_condition` +5. **Mock at boundaries**: Only mock external dependencies +6. **Edge cases first**: Test boundary conditions, not just happy paths + + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +``` + +### Rules + +- Use `bd` for ALL task tracking โ€” do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge โ€” do NOT use MEMORY.md files + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion + +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. + +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status + + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + bd dolt push + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step + +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + + + +## Beads Issue Tracker + +Use Beads (`bd`) for durable task tracking in repositories that include it. Use the `beads` skill at `.agents/skills/beads/SKILL.md` (project install) or `~/.agents/skills/beads/SKILL.md` (global install) for Beads workflow guidance, then use the `bd` CLI for issue operations. + +### Quick Reference + +```bash +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work +bd prime # Refresh Beads context +``` -## CLI +### Rules -| Task | Read this skill file | -| -------------------------------------------- | ----------------------------------------------------------- | -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | +- Use `bd` for all task tracking; do not create markdown TODO lists. +- Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. +- Keep persistent project memory in Beads via `bd remember`; do not create ad hoc memory files. - +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. + diff --git a/CLAUDE.md b/CLAUDE.md index d4cbaa2..354d97f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,46 +1,72 @@ - + # emailibrium -Vector-native, local-first email intelligence: semantic search, clustering, classification, and inbox cleanup over 10k+ emails with no cloud processing. Rust backend + React SPA. +## Swarm Config -## Layout +- **Topology**: hierarchical-mesh (anti-drift) +- **Max Agents**: 15 +- **Memory**: hybrid -| Path | What | Stack | -| ---------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | -| `backend/` | Axum REST + SSE API and the intelligence layer | Rust 1.96 (edition 2021), SQLx/SQLite, Moka, Redis | -| `backend/src/vectors/` | 22+ vector intelligence modules: embedding, HNSW search, SONA learning, clustering, RAG, encryption | โ€” | -| `backend/src/mcp/` | MCP server exposing email tools | โ€” | -| `backend/migrations/` | Numbered SQLite migrations โ€” **append the next number, never edit an applied one** | โ€” | -| `frontend/` | pnpm + Turborepo monorepo (app in `apps/web/`) | React 19, TS 5.9, Vite 8, TanStack Router/Query, Zustand, Tailwind | -| `ruvector/` | **Git submodule** (ruvnet/ruvector) โ€” the vector engine. Treat as vendored: don't edit; backend depends on it via path | Rust workspace | -| `docs/` | `architecture.md`, `ADRs/`, `DDDs/`, evaluation, setup/oauth guides | โ€” | -| `config/`, `secrets/` | Runtime config + dev secrets (never commit secrets) | โ€” | +```bash +ruflo swarm init --topology hierarchical --max-agents 15 --strategy specialized +``` + +## Agentic QE v3 + -## Build & Test โ€” Makefile-driven, not npm -The root `package.json` only wires Husky; **do not run `npm build`/`npm test`**. Use `make`: + +## Beads Issue Tracker + +This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. + +### Quick Reference ```bash -make ci # format-check + lint + typecheck + test (run before committing) -make test # backend (cargo) + frontend (Vitest) -make build # build everything -make dev # full stack: backend :8080, frontend :3000 -make lint # code + docs (markdownlint, yamllint) -make audit # cargo-audit + npm audit -make help # all targets +bd ready # Find available work +bd show # View issue details +bd update --claim # Claim work +bd close # Complete work ``` -Backend-only: `cd backend && cargo test` / `cargo clippy`. Frontend-only: `cd frontend && pnpm test` / `pnpm lint` / `pnpm typecheck`. +### Rules + +- Use `bd` for ALL task tracking โ€” do NOT use TodoWrite, TaskCreate, or markdown TODO lists +- Run `bd prime` for detailed command reference and session close protocol +- Use `bd remember` for persistent knowledge โ€” do NOT use MEMORY.md files + +**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. + +## Agent Context Profiles + +The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. + +- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. +- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. +- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. + +## Session Completion -Backend Cargo features: `vectors` (default), `builtin-llm` (llama-cpp, opt-in), `proptest`. +This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. -## Conventions +1. **File issues for remaining work** - Create beads for anything that needs follow-up +2. **Run quality gates** (if code changed) - Tests, linters, builds +3. **Update issue status** - Close finished work, update in-progress items +4. **Handle git/sync by active profile**: + ```bash + # Conservative/minimal/default: report status and proposed commands; wait for approval. + git status -- **Decisions are ADR/DDD-gated.** Before changing architecture, vector storage, learning, or AI providers, check `docs/ADRs/` and `docs/DDDs/` โ€” e.g. ADR-003 fixes RuVector as the primary vector store. Record new decisions as an ADR. -- **Privacy is a hard guarantee, not a setting.** Embeddings/models run and stay local; cloud AI is strictly opt-in. Don't add code paths that send email content off-machine by default. -- Encryption at rest is AES-256-GCM + Argon2id โ€” keep crypto changes within `backend/src/vectors/encryption.rs` and consent-gated. -- For code navigation, impact analysis, and safe refactors, use the GitNexus MCP tools per `AGENTS.md`. + # Team-maintainer opt-in only, unless current instructions forbid it: + git pull --rebase + git push + git status + ``` +5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step - - +**Critical rules:** +- Explicit user or orchestrator instructions override this Beads block. +- Do not commit or push without clear authority from the active profile or the current user request. +- If a required sync or push is blocked, stop and report the exact command and error. + From e12223ec1099809ac5ba798688d0b07585281b89 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 10:27:37 -0700 Subject: [PATCH 02/24] Revert "bd init: initialize beads issue tracking" This reverts commit 840c88d6bf9413d6b66c40cadb6e9678b87e6689. --- .beads/.gitignore | 77 ----- .beads/README.md | 81 ----- .beads/config.yaml | 68 ---- .beads/hooks/applypatch-msg | 3 - .beads/hooks/commit-msg | 3 - .beads/hooks/post-applypatch | 3 - .beads/hooks/post-checkout | 36 -- .beads/hooks/post-commit | 3 - .beads/hooks/post-merge | 36 -- .beads/hooks/post-rewrite | 3 - .beads/hooks/pre-applypatch | 3 - .beads/hooks/pre-auto-gc | 3 - .beads/hooks/pre-commit | 101 ------ .beads/hooks/pre-merge-commit | 3 - .beads/hooks/pre-push | 36 -- .beads/hooks/pre-rebase | 3 - .beads/hooks/prepare-commit-msg | 36 -- .beads/interactions.jsonl | 0 .beads/metadata.json | 7 - .gitignore | 6 - AGENTS.md | 574 ++++---------------------------- CLAUDE.md | 88 ++--- 22 files changed, 100 insertions(+), 1073 deletions(-) delete mode 100644 .beads/.gitignore delete mode 100644 .beads/README.md delete mode 100644 .beads/config.yaml delete mode 100755 .beads/hooks/applypatch-msg delete mode 100755 .beads/hooks/commit-msg delete mode 100755 .beads/hooks/post-applypatch delete mode 100755 .beads/hooks/post-checkout delete mode 100755 .beads/hooks/post-commit delete mode 100755 .beads/hooks/post-merge delete mode 100755 .beads/hooks/post-rewrite delete mode 100755 .beads/hooks/pre-applypatch delete mode 100755 .beads/hooks/pre-auto-gc delete mode 100755 .beads/hooks/pre-commit delete mode 100755 .beads/hooks/pre-merge-commit delete mode 100755 .beads/hooks/pre-push delete mode 100755 .beads/hooks/pre-rebase delete mode 100755 .beads/hooks/prepare-commit-msg delete mode 100644 .beads/interactions.jsonl delete mode 100644 .beads/metadata.json diff --git a/.beads/.gitignore b/.beads/.gitignore deleted file mode 100644 index f773858..0000000 --- a/.beads/.gitignore +++ /dev/null @@ -1,77 +0,0 @@ -# Dolt database (managed by Dolt, not git) -dolt/ -embeddeddolt/ -proxieddb/ - -# Runtime files -bd.sock -bd.sock.startlock -sync-state.json -last-touched -.exclusive-lock - -# Daemon runtime (lock, log, pid) -daemon.* - -# Push state (runtime, per-machine) -push-state.json - -# Lock files (various runtime locks) -*.lock - -# Credential key (encryption key for federation peer auth โ€” never commit) -.beads-credential-key - -# Local version tracking (prevents upgrade notification spam after git ops) -.local_version - -proxied_server_client_info.json - -# Worktree redirect file (contains relative path to main repo's .beads/) -# Must not be committed as paths would be wrong in other clones -redirect - -# Sync state (local-only, per-machine) -# These files are machine-specific and should not be shared across clones -.sync.lock -export-state/ -export-state.json -last_pull - -# Ephemeral store (SQLite - wisps/molecules, intentionally not versioned) -ephemeral.sqlite3 -ephemeral.sqlite3-journal -ephemeral.sqlite3-wal -ephemeral.sqlite3-shm - -# Dolt server management (auto-started by bd) -dolt-server.pid -dolt-server.log -dolt-server.lock -dolt-server.port -dolt-server.activity - -# Debug-mode pprof artifacts (written when dolt.debug: true in config.yaml) -dolt-pprof/ - -# Corrupt backup directories (created by bd doctor --fix recovery) -*.corrupt.backup/ - -# Backup data (auto-exported JSONL, local-only) -backup/ - -# Per-project environment file (Dolt connection config, GH#2520) -.env - -# Legacy files (from pre-Dolt versions) -*.db -*.db?* -*.db-journal -*.db-wal -*.db-shm -db.sqlite -bd.db -# NOTE: Do NOT add negation patterns here. -# They would override fork protection in .git/info/exclude. -# Config files (metadata.json, config.yaml) are tracked by git by default -# since no pattern above ignores them. diff --git a/.beads/README.md b/.beads/README.md deleted file mode 100644 index 63e8f4c..0000000 --- a/.beads/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Beads - AI-Native Issue Tracking - -Welcome to Beads! This repository uses **Beads** for issue tracking - a modern, AI-native tool designed to live directly in your codebase alongside your code. - -## What is Beads? - -Beads is issue tracking that lives in your repo, making it perfect for AI coding agents and developers who want their issues close to their code. No web UI required - everything works through the CLI and integrates seamlessly with git. - -**Learn more:** [github.com/steveyegge/beads](https://github.com/steveyegge/beads) - -## Quick Start - -### Essential Commands - -```bash -# Create new issues -bd create "Add user authentication" - -# View all issues -bd list - -# View issue details -bd show - -# Update issue status -bd update --claim -bd update --status done - -# Sync with Dolt remote -bd dolt push -``` - -### Working with Issues - -Issues in Beads are: -- **Git-native**: Stored in Dolt database with version control and branching -- **AI-friendly**: CLI-first design works perfectly with AI coding agents -- **Branch-aware**: Issues can follow your branch workflow -- **Sync-ready**: Uses Dolt remotes for backup and team sharing - -## Why Beads? - -โœจ **AI-Native Design** -- Built specifically for AI-assisted development workflows -- CLI-first interface works seamlessly with AI coding agents -- No context switching to web UIs - -๐Ÿš€ **Developer Focused** -- Issues live in your repo, right next to your code -- Works offline, syncs when you push -- Fast, lightweight, and stays out of your way - -๐Ÿ”ง **Git Integration** -- Dolt-native sync via bd dolt push / bd dolt pull -- Branch-aware issue tracking -- Dolt-native three-way merge resolution - -## Get Started with Beads - -Try Beads in your own projects: - -```bash -# Install Beads -curl -sSL https://raw.githubusercontent.com/steveyegge/beads/main/scripts/install.sh | bash - -# Initialize in your repo -bd init - -# Create your first issue -bd create "Try out Beads" -``` - -## Learn More - -- **Documentation**: [github.com/steveyegge/beads/docs](https://github.com/steveyegge/beads/tree/main/docs) -- **Quick Start Guide**: Run `bd quickstart` -- **Examples**: [github.com/steveyegge/beads/examples](https://github.com/steveyegge/beads/tree/main/examples) - ---- - -*Beads: Issue tracking that moves at the speed of thought* โšก diff --git a/.beads/config.yaml b/.beads/config.yaml deleted file mode 100644 index af0fa36..0000000 --- a/.beads/config.yaml +++ /dev/null @@ -1,68 +0,0 @@ -# Beads Configuration File -# This file configures default behavior for all bd commands in this repository -# All settings can also be set via environment variables (BD_* prefix) -# or overridden with command-line flags - -# Issue prefix for this repository (used by bd init) -# If not set, bd init will auto-detect from directory name -# Example: issue-prefix: "myproject" creates issues like "myproject-1", "myproject-2", etc. -# issue-prefix: "" - -# Use no-db mode: JSONL-only, no Dolt database -# When true, .beads/issues.jsonl is the only local store -# no-db: false - -# Enable JSON output by default -# json: false - -# Feedback title formatting for mutating commands (create/update/close/dep/edit) -# 0 = hide titles, N > 0 = truncate to N characters -# output: -# title-length: 255 - -# Default actor for audit trails (overridden by BEADS_ACTOR or --actor) -# actor: "" - -# Export events (audit trail) to .beads/events.jsonl on each flush/sync -# When enabled, new events are appended incrementally using a high-water mark. -# Use 'bd export --events' to trigger manually regardless of this setting. -# events-export: false - -# Multi-repo configuration (experimental - bd-307) -# Allows hydrating from multiple repositories and routing writes to the correct database -# repos: -# primary: "." # Primary repo (where this database lives) -# additional: # Additional repos to hydrate from (read-only) -# - ~/beads-planning # Personal planning repo -# - ~/work-planning # Work planning repo - -# Dolt-native backup (periodic backup for off-machine recovery) -# This is full database backup only. Cross-machine sync uses Dolt remotes. -# backup: -# enabled: false # Disable auto-backup entirely -# interval: 15m # Minimum time between auto-backups -# git-push: false # Disable git push (backup locally only) -# git-repo: "" # Separate git repo for backups (default: project repo) - -# Optional JSONL auto-export for viewers, interchange, and issue-level migration. -# Disabled by default; enable only when an integration needs fresh .beads/issues.jsonl. -# Use relative paths under .beads/ for JSONL import/export filenames. -# export: -# auto: false -# path: issues.jsonl -# interval: 60s -# git-add: false -# import: -# path: issues.jsonl - -# Integration settings (access with 'bd config get/set') -# Non-secret keys (stored in the database): -# - jira.url, jira.project -# - linear.team_id -# - github.org, github.repo -# -# Secret keys (stored in this file but prefer env vars to avoid git exposure): -# - linear.api_key โ†’ use LINEAR_API_KEY env var instead -# - github.token โ†’ use GITHUB_TOKEN env var instead - -sync.remote: "git+https://github.com/pacphi/emailibrium.git" \ No newline at end of file diff --git a/.beads/hooks/applypatch-msg b/.beads/hooks/applypatch-msg deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/applypatch-msg +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/commit-msg b/.beads/hooks/commit-msg deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/commit-msg +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-applypatch b/.beads/hooks/post-applypatch deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/post-applypatch +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-checkout b/.beads/hooks/post-checkout deleted file mode 100755 index c66cbfc..0000000 --- a/.beads/hooks/post-checkout +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" - -# --- BEGIN BEADS INTEGRATION v1.1.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - _bd_used_perl=0 - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run post-checkout "$@" - _bd_exit=$? - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout "$_bd_timeout" bd hooks run post-checkout "$@" - _bd_exit=$? - elif command -v perl >/dev/null 2>&1; then - _bd_used_perl=1 - perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run post-checkout "$@" - _bd_exit=$? - else - echo >&2 "beads: hook 'post-checkout' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" - bd hooks run post-checkout "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then - echo >&2 "beads: hook 'post-checkout' timed out after ${_bd_timeout}s โ€” continuing without beads" - _bd_exit=0 - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized โ€” skipping hook 'post-checkout'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/post-commit b/.beads/hooks/post-commit deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/post-commit +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/post-merge b/.beads/hooks/post-merge deleted file mode 100755 index 18216de..0000000 --- a/.beads/hooks/post-merge +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" - -# --- BEGIN BEADS INTEGRATION v1.1.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - _bd_used_perl=0 - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run post-merge "$@" - _bd_exit=$? - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout "$_bd_timeout" bd hooks run post-merge "$@" - _bd_exit=$? - elif command -v perl >/dev/null 2>&1; then - _bd_used_perl=1 - perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run post-merge "$@" - _bd_exit=$? - else - echo >&2 "beads: hook 'post-merge' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" - bd hooks run post-merge "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then - echo >&2 "beads: hook 'post-merge' timed out after ${_bd_timeout}s โ€” continuing without beads" - _bd_exit=0 - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized โ€” skipping hook 'post-merge'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/post-rewrite b/.beads/hooks/post-rewrite deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/post-rewrite +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-applypatch b/.beads/hooks/pre-applypatch deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/pre-applypatch +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-auto-gc b/.beads/hooks/pre-auto-gc deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/pre-auto-gc +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-commit b/.beads/hooks/pre-commit deleted file mode 100755 index 3a721b1..0000000 --- a/.beads/hooks/pre-commit +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Pre-commit gate โ€” mirrors the GitHub CI workflow quality checks so -# violations are caught locally before they ever reach CI. -# -# Fails fast on the first error so the user gets back to a shell prompt -# instead of hanging. Missing tools = hard failure with an install hint. -# Bash 3.2 compatible (macOS default). - -set -euo pipefail - -# 1) Prettier + ESLint on staged files (via lint-staged + package.json). -pnpm lint-staged - -staged_list() { git diff --cached --name-only --diff-filter=ACMR -- "$@" | tr '\n' '\0' | xargs -0 -I{} echo "{}"; } - -STAGED_MD=$(git diff --cached --name-only --diff-filter=ACMR -- '*.md') -STAGED_YAML=$(git diff --cached --name-only --diff-filter=ACMR -- '*.yaml' '*.yml' | grep -v 'pnpm-lock.yaml$' || true) -STAGED_SH=$(git diff --cached --name-only --diff-filter=ACMR -- '*.sh') -STAGED_BACKEND=$(git diff --cached --name-only --diff-filter=ACMR -- 'backend/') - -# 2) Markdownlint (if .md staged). -if [ -n "$STAGED_MD" ]; then - if ! command -v markdownlint-cli2 >/dev/null 2>&1; then - echo "โœ— markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2" >&2 - exit 1 - fi - COUNT=$(echo "$STAGED_MD" | wc -l | tr -d ' ') - echo "โ†’ markdownlint on $COUNT staged .md file(s)..." - echo "$STAGED_MD" | xargs markdownlint-cli2 -fi - -# 3) yamllint (if .yaml/.yml staged, excluding lockfiles). -if [ -n "$STAGED_YAML" ]; then - if ! command -v yamllint >/dev/null 2>&1; then - echo "โœ— yamllint not installed. Run: pip install yamllint" >&2 - exit 1 - fi - COUNT=$(echo "$STAGED_YAML" | wc -l | tr -d ' ') - echo "โ†’ yamllint on $COUNT staged YAML file(s)..." - echo "$STAGED_YAML" | xargs yamllint -c .yamllint.yaml -fi - -# 4) shellcheck (if .sh staged). -if [ -n "$STAGED_SH" ]; then - if ! command -v shellcheck >/dev/null 2>&1; then - echo "โœ— shellcheck not installed. Run: brew install shellcheck" >&2 - exit 1 - fi - COUNT=$(echo "$STAGED_SH" | wc -l | tr -d ' ') - echo "โ†’ shellcheck on $COUNT staged shell script(s)..." - echo "$STAGED_SH" | xargs shellcheck -fi - -# 5) Rust fmt + clippy (if backend/ touched). Matches CI strictness: deny -# all warnings but allow dead_code/unused_* so pre-existing noise does -# not block commits. -if [ -n "$STAGED_BACKEND" ]; then - echo "โ†’ cargo fmt + clippy on backend/..." - ( - cd backend - cargo fmt --package emailibrium -- --check - cargo clippy -p emailibrium --all-targets -- \ - -D warnings \ - -A dead_code -A unused_variables -A unused_imports -A unused_mut - ) -fi - -echo "โœ“ pre-commit checks passed." - -# --- BEGIN BEADS INTEGRATION v1.1.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - _bd_used_perl=0 - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run pre-commit "$@" - _bd_exit=$? - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout "$_bd_timeout" bd hooks run pre-commit "$@" - _bd_exit=$? - elif command -v perl >/dev/null 2>&1; then - _bd_used_perl=1 - perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run pre-commit "$@" - _bd_exit=$? - else - echo >&2 "beads: hook 'pre-commit' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" - bd hooks run pre-commit "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then - echo >&2 "beads: hook 'pre-commit' timed out after ${_bd_timeout}s โ€” continuing without beads" - _bd_exit=0 - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized โ€” skipping hook 'pre-commit'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/pre-merge-commit b/.beads/hooks/pre-merge-commit deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/pre-merge-commit +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/pre-push b/.beads/hooks/pre-push deleted file mode 100755 index ecc4b88..0000000 --- a/.beads/hooks/pre-push +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" - -# --- BEGIN BEADS INTEGRATION v1.1.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - _bd_used_perl=0 - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run pre-push "$@" - _bd_exit=$? - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout "$_bd_timeout" bd hooks run pre-push "$@" - _bd_exit=$? - elif command -v perl >/dev/null 2>&1; then - _bd_used_perl=1 - perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run pre-push "$@" - _bd_exit=$? - else - echo >&2 "beads: hook 'pre-push' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" - bd hooks run pre-push "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then - echo >&2 "beads: hook 'pre-push' timed out after ${_bd_timeout}s โ€” continuing without beads" - _bd_exit=0 - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized โ€” skipping hook 'pre-push'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/hooks/pre-rebase b/.beads/hooks/pre-rebase deleted file mode 100755 index 6aa68db..0000000 --- a/.beads/hooks/pre-rebase +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" \ No newline at end of file diff --git a/.beads/hooks/prepare-commit-msg b/.beads/hooks/prepare-commit-msg deleted file mode 100755 index af1d640..0000000 --- a/.beads/hooks/prepare-commit-msg +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env sh -# Injected by beads (GH#3132): husky helper layout not mirrored into this dir. -export PATH="$PWD/node_modules/.bin:$PATH" - -# --- BEGIN BEADS INTEGRATION v1.1.0 --- -# This section is managed by beads. Do not remove these markers. -if command -v bd >/dev/null 2>&1; then - export BD_GIT_HOOK=1 - _bd_timeout=${BEADS_HOOK_TIMEOUT:-300} - _bd_used_perl=0 - if command -v timeout >/dev/null 2>&1; then - timeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - elif command -v gtimeout >/dev/null 2>&1; then - gtimeout "$_bd_timeout" bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - elif command -v perl >/dev/null 2>&1; then - _bd_used_perl=1 - perl -e 'alarm shift; exec @ARGV' "$_bd_timeout" bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - else - echo >&2 "beads: hook 'prepare-commit-msg' running without timeout; install coreutils or perl to enable BEADS_HOOK_TIMEOUT" - bd hooks run prepare-commit-msg "$@" - _bd_exit=$? - fi - if [ $_bd_exit -eq 124 ] || { [ $_bd_used_perl -eq 1 ] && [ $_bd_exit -eq 142 ]; }; then - echo >&2 "beads: hook 'prepare-commit-msg' timed out after ${_bd_timeout}s โ€” continuing without beads" - _bd_exit=0 - fi - if [ $_bd_exit -eq 3 ]; then - echo >&2 "beads: database not initialized โ€” skipping hook 'prepare-commit-msg'" - _bd_exit=0 - fi - if [ $_bd_exit -ne 0 ]; then exit $_bd_exit; fi -fi -# --- END BEADS INTEGRATION v1.1.0 --- diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl deleted file mode 100644 index e69de29..0000000 diff --git a/.beads/metadata.json b/.beads/metadata.json deleted file mode 100644 index 0a62889..0000000 --- a/.beads/metadata.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "database": "dolt", - "backend": "dolt", - "dolt_mode": "embedded", - "dolt_database": "emailibrium", - "project_id": "337e044a-6ee3-4107-aadc-941af8be48fd" -} \ No newline at end of file diff --git a/.gitignore b/.gitignore index e4fb47c..e64c31f 100644 --- a/.gitignore +++ b/.gitignore @@ -119,9 +119,3 @@ docker-compose.override.yml # RuVector / ruflo local agent databases -- machine artifacts, not library content agentdb.rvf agentdb.rvf.lock -.autopilot/queued/ - -# Beads / Dolt files (added by bd init) -.dolt/ -.beads-credential-key -.beads/proxieddb/ diff --git a/AGENTS.md b/AGENTS.md index 5221fe0..c134a5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,539 +1,103 @@ -# emailibrium + -> Multi-agent orchestration framework for agentic coding +# GitNexus โ€” Code Intelligence -## Project Overview +This project is indexed by GitNexus as **emailibrium** (228117 symbols, 447347 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. -A Claude Flow powered project +> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. -**Tech Stack**: TypeScript, Node.js -**Architecture**: Domain-Driven Design with bounded contexts +## Always Do -## Quick Start +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. +- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. +- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol โ€” callers, callees, which execution flows it participates in โ€” use `gitnexus_context({name: "symbolName"})`. -### Installation -```bash -npm install -``` - -### Build -```bash -npm run build -``` - -### Test -```bash -npm test -``` - -### Development -```bash -npm run dev -``` - -## Agent Coordination - -### Swarm Configuration - -This project uses hierarchical swarm coordination for complex tasks: - -| Setting | Value | Purpose | -|---------|-------|---------| -| Topology | `hierarchical` | Queen-led coordination (anti-drift) | -| Max Agents | 8 | Optimal team size | -| Strategy | `specialized` | Clear role boundaries | -| Consensus | `raft` | Leader-based consistency | - -### When to Use Swarms - -**Invoke swarm for:** -- Multi-file changes (3+ files) -- New feature implementation -- Cross-module refactoring -- API changes with tests -- Security-related changes -- Performance optimization - -**Skip swarm for:** -- Single file edits -- Simple bug fixes (1-2 lines) -- Documentation updates -- Configuration changes - -### Available Skills - -Use `$skill-name` syntax to invoke: - -| Skill | Use Case | -|-------|----------| -| `$swarm-orchestration` | Multi-agent task coordination | -| `$memory-management` | Pattern storage and retrieval | -| `$sparc-methodology` | Structured development workflow | -| `$security-audit` | Security scanning and CVE detection | -| `$performance-analysis` | Profiling and optimization | -| `$github-automation` | CI/CD and PR management | - -### Agent Types - -| Type | Role | Use Case | -|------|------|----------| -| `researcher` | Requirements analysis | Understanding scope | -| `architect` | System design | Planning structure | -| `coder` | Implementation | Writing code | -| `tester` | Test creation | Quality assurance | -| `reviewer` | Code review | Security and quality | - -## Execution Model - -- **claude-flow** = LEDGER (coordinates: memory, routing, swarm state) -- **Codex** = EXECUTOR (writes code, runs tests, creates files) - -**Critical rule:** DON'T STOP after calling claude-flow commands. Coordination commands return instantly โ€” continue immediately with the next implementation step. - -## Ruflo + Codex Automated Workflow - -Ruflo is the coordination ledger and policy decision point; Codex workers execute code, tests, and commands. A Ruflo coordination call records work but never replaces implementation. - -Use `guidance_brain({ mode: "recommend", task: "..." })` when the task can -benefit from Ruflo-specific capabilities. Its live registry is authoritative -for tool presence; registration alone does not prove configuration, -reachability, health, or authorization. If it is not registered, use compatible -`guidance_recommend`, CLI discovery, and repository instructions. - -1. **Recall** โ€” search AgentDB memory and relevant ADRs for patterns and constraints. -2. **Inspect** โ€” read source, runtime, dependency, policy, and health state. -3. **Route** โ€” choose the smallest capable topology, agents, skills, and tools. -4. **Plan** โ€” define acceptance criteria, safety envelope, ownership, and validation. -5. **Execute** โ€” Codex workers implement in isolated scopes; Ruflo records coordination. -6. **Test** โ€” run focused tests, regression tests, and failure-path checks. -7. **Validate** โ€” check types, security, policy, compatibility, and artifact integrity. -8. **Benchmark** โ€” compare a source-bound candidate with a source-bound baseline. -9. **Optimize** โ€” improve measured bottlenecks without weakening the safety envelope. -10. **Receipt** โ€” bind claims, evidence, and decisions to exact source/build inputs. -11. **Handoff** โ€” reconcile concurrent work and disclose unresolved limitations. -12. **Publish** โ€” only an independently authorized release gate may publish immutable artifacts. - -### Concurrency and authority invariants - -- Never allow two writers in one worktree. -- Read-only research agents may share a checkout; writing agents may not. -- A child may drop capabilities but can never add tools, servers, namespaces, network access, spend, concurrency, or delegation depth. -- Cancel dependent and not-yet-started sibling work when policy denies an action or a required dependency fails. -- MetaHarness may benchmark candidates concurrently, but it cannot promote, serve, or expand its own SafetyEnvelope. -- Only the integration agent changes shared manifests or lockfiles. -- Do not auto-commit, push, merge, release, or delete worktrees unless the user authorized that operation. -- Every consequential action must produce a policy decision receipt; production, destructive, spend, and promotion actions may require human approval. - -### Repository harness adapter +## When Debugging -When tracked repository instructions define a local collaboration harness: +1. `gitnexus_query({query: ""})` โ€” find execution flows related to the issue +2. `gitnexus_context({name: ""})` โ€” see all callers, callees, and process participation +3. `READ gitnexus://repo/emailibrium/process/{processName}` โ€” trace the full execution flow step by step +4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` โ€” see what your branch changed -1. Assign the isolated worktree before starting a writing session. -2. Start or register the session, inspect current claims, and acquire only the - exact paths, resources, and development ports needed for the task. -3. Renew leases during long work, check acknowledged inbox messages at integration - boundaries, and release claims when handing off or ending. -4. Record focused and integration evidence against the exact source state, - then let the designated integration owner decide release. +## When Refactoring -A repository lease coordinates ownership; it does not grant authorization. -In-memory reference adapters demonstrate semantics but are not distributed, -restart-durable release authorities. -The worker still needs the current ADR-324/325 action capability and fencing -epoch for every protected side effect. Heartbeat and lease expiry establish -liveness; a PID is diagnostic only. HEAD alone is not an exact source-state -identity when tracked or untracked changes exist, so a release receipt must -bind a clean commit or an immutable snapshot including those changes. +- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview โ€” graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. +- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. +- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. +## Never Do -## MCP Integration +- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. +- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. +- NEVER rename symbols with find-and-replace โ€” use `gitnexus_rename` which understands the call graph. +- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. -Use MCP tools for coordination, then keep coding: +## Tools Quick Reference -| Tool | Purpose | Example | -|------|---------|---------| -| `swarm_init` | Start coordination | `swarm_init({topology: "hierarchical"})` | -| `memory_store` | Save patterns | `memory_store({key: "auth", value: "JWT"})` | -| `memory_search` | Find patterns | `memory_search({query: "auth patterns"})` | -| `task_orchestrate` | Assign work | `task_orchestrate({task: "implement"})` | +| Tool | When to use | Command | +| ---------------- | ----------------------------- | ----------------------------------------------------------------------- | +| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | +| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | +| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | +| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | +| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | +| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | -## Code Standards +## Impact Risk Levels -### File Organization -- **NEVER** save to root folder -- `/src` - Source code files -- `/tests` - Test files -- `/docs` - Documentation -- `/config` - Configuration files +| Depth | Meaning | Action | +| ----- | ------------------------------------- | --------------------- | +| d=1 | WILL BREAK โ€” direct callers/importers | MUST update these | +| d=2 | LIKELY AFFECTED โ€” indirect deps | Should test | +| d=3 | MAY NEED TESTING โ€” transitive | Test if critical path | -### Quality Rules -- Files under 500 lines -- No hardcoded secrets -- Input validation at boundaries -- Typed interfaces for public APIs -- TDD London School (mock-first) preferred +## Resources -### Commit Messages -``` -(): - -[optional body] -``` +| Resource | Use for | +| -------------------------------------------- | ---------------------------------------- | +| `gitnexus://repo/emailibrium/context` | Codebase overview, check index freshness | +| `gitnexus://repo/emailibrium/clusters` | All functional areas | +| `gitnexus://repo/emailibrium/processes` | All execution flows | +| `gitnexus://repo/emailibrium/process/{name}` | Step-by-step execution trace | -Types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `chore` +## Self-Check Before Finishing -Do not add a `Co-Authored-By` trailer unless the repository explicitly -configures and authorizes that attribution. +Before completing any code modification task, verify: -## Security +1. `gitnexus_impact` was run for all modified symbols +2. No HIGH/CRITICAL risk warnings were ignored +3. `gitnexus_detect_changes()` confirms changes match expected scope +4. All d=1 (WILL BREAK) dependents were updated -### Critical Rules -- NEVER commit secrets, credentials, or .env files -- NEVER hardcode API keys -- Always validate user input -- Use parameterized queries for SQL -- Sanitize output to prevent XSS +## Keeping the Index Fresh -### Path Security -- Validate all file paths -- Prevent directory traversal (../) -- Use absolute paths internally +After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: -## Memory System - -### Storing Patterns ```bash -npx @claude-flow/cli memory store \ - --key "pattern-name" \ - --value "pattern description" \ - --namespace patterns +npx gitnexus analyze ``` -### Searching Memory -```bash -npx @claude-flow/cli memory search \ - --query "search terms" \ - --namespace patterns -``` - -## Quick Commands +If the index previously included embeddings, preserve them by adding `--embeddings`: ```bash -npx @claude-flow/cli memory search --query "relevant patterns" -npx @claude-flow/cli hooks route --task "current task description" -npx @claude-flow/cli swarm init --topology hierarchical -npx @claude-flow/cli hooks pre-task --description "task summary" +npx gitnexus analyze --embeddings ``` -## Links - -- Documentation: https://github.com/ruvnet/ruflo -- Issues: https://github.com/ruvnet/ruflo/issues - -## Performance Targets - -| Metric | Target | Notes | -|--------|--------|-------| -| HNSW Search | 150x-12,500x faster | Vector operations | -| Memory Reduction | 50-75% | Int8 quantization | -| MCP Response | <100ms | API latency | -| CLI Startup | <500ms | Cold start | -| SONA Adaptation | <0.05ms | Neural learning | - -## Testing - -### Running Tests -```bash -# Unit tests -npm test - -# Integration tests -npm run test:integration - -# Coverage -npm run test:coverage - -# Security tests -npm run test:security -``` - -### Test Philosophy -- TDD London School (mock-first) -- Unit tests for business logic -- Integration tests for boundaries -- E2E tests for critical paths -- Security tests for sensitive operations - -### Coverage Requirements -- Minimum 80% line coverage -- 100% coverage for security-critical code -- All public APIs must have tests - -## MCP Integration - -Claude Flow exposes tools via Model Context Protocol: - -```bash -# Start MCP server -npx ruflo mcp start - -# List available tools -npx ruflo mcp tools -``` - -### Available Tools - -| Tool | Purpose | Example | -|------|---------|---------| -| `swarm_init` | Initialize swarm coordination | `swarm_init({topology: "hierarchical"})` | -| `agent_spawn` | Spawn new agents | `agent_spawn({type: "coder", name: "dev-1"})` | -| `memory_store` | Store in AgentDB | `memory_store({key: "pattern", value: "..."})` | -| `memory_search` | Semantic search | `memory_search({query: "auth patterns"})` | -| `task_orchestrate` | Task coordination | `task_orchestrate({task: "implement feature"})` | -| `neural_train` | Train neural patterns | `neural_train({iterations: 10})` | -| `benchmark_run` | Performance benchmarks | `benchmark_run({type: "all"})` | - -## Hooks System - -Claude Flow uses hooks for lifecycle automation: - -### Core Hooks - -| Hook | Trigger | Purpose | -|------|---------|---------| -| `pre-task` | Before task starts | Get context, load patterns | -| `post-task` | After task completes | Record completion, train | -| `pre-edit` | Before file changes | Validate, backup | -| `post-edit` | After file changes | Train patterns, verify | -| `pre-command` | Before shell commands | Security check | -| `post-command` | After shell commands | Log results | - -### Session Hooks - -| Hook | Purpose | -|------|---------| -| `session-start` | Initialize context, load memory | -| `session-end` | Export metrics, consolidate memory | -| `session-restore` | Resume from checkpoint | -| `notify` | Send notifications | +To check whether embeddings exist, inspect `.gitnexus/meta.json` โ€” the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** -### Intelligence Hooks - -| Hook | Purpose | -|------|---------| -| `route` | Route task to appropriate agents | -| `explain` | Generate explanations | -| `pretrain` | Pre-train neural patterns | -| `build-agents` | Build specialized agents | -| `transfer` | Transfer learning between domains | - -### Example Usage -```bash -# Before starting a task -npx @claude-flow/cli hooks pre-task \ - --description "implementing authentication" - -# After completing a task -npx @claude-flow/cli hooks post-task \ - --task-id "task-123" \ - --success true - -# Route a task to agents -npx @claude-flow/cli hooks route \ - --task "implement OAuth2 login flow" -``` - -## Background Workers - -12 background workers provide continuous optimization: - -| Worker | Priority | Purpose | -|--------|----------|---------| -| `ultralearn` | normal | Deep knowledge acquisition | -| `optimize` | high | Performance optimization | -| `consolidate` | low | Memory consolidation | -| `predict` | normal | Predictive preloading | -| `audit` | critical | Security analysis | -| `map` | normal | Codebase mapping | -| `preload` | low | Resource preloading | -| `deepdive` | normal | Deep code analysis | -| `document` | normal | Auto-documentation | -| `refactor` | normal | Refactoring suggestions | -| `benchmark` | normal | Performance benchmarking | -| `testgaps` | normal | Test coverage analysis | - -### Managing Workers -```bash -# List workers -npx @claude-flow/cli hooks worker list - -# Trigger specific worker -npx @claude-flow/cli hooks worker dispatch --trigger audit - -# Check worker status -npx @claude-flow/cli hooks worker status -``` - -## Intelligence System - -The RuVector Intelligence System provides neural learning: - -### Components -- **SONA**: Self-Optimizing Neural Architecture (<0.05ms adaptation) -- **MoE**: Mixture of Experts for specialized routing -- **HNSW**: Hierarchical Navigable Small World for fast search -- **EWC++**: Elastic Weight Consolidation (prevents forgetting) -- **Flash Attention**: Optimized attention mechanism - -### 4-Step Pipeline -1. **RETRIEVE** - Fetch relevant patterns via HNSW -2. **JUDGE** - Evaluate with verdicts (success/failure) -3. **DISTILL** - Extract key learnings via LoRA -4. **CONSOLIDATE** - Prevent catastrophic forgetting via EWC++ - -## Debugging - -### Log Levels -```bash -# Set log level -export CLAUDE_FLOW_LOG_LEVEL=debug - -# Enable verbose mode -npx @claude-flow/cli --verbose -``` - -### Health Checks -```bash -# Run diagnostics -npx @claude-flow/cli doctor --fix - -# Check system status -npx @claude-flow/cli status -``` - ---- - -# Quality Engineering Standards (Agentic QE) - -## AQE MCP Server - -This project uses Agentic QE for AI-powered quality engineering. The AQE MCP server provides tools for test generation, coverage analysis, quality assessment, and learning. - -## Setup - -Always call `fleet_init` before using other AQE tools to initialize the QE fleet. - -## Available Tools - -### Test Generation -- `test_generate_enhanced` โ€” AI-powered test generation with pattern recognition and anti-pattern detection -- Supports unit, integration, and e2e test types - -### Coverage Analysis -- `coverage_analyze_sublinear` โ€” O(log n) coverage gap detection with ML-powered analysis -- Target: 80% statement coverage minimum, focus on risk-weighted coverage - -### Quality Assessment -- `quality_assess` โ€” Quality gate evaluation with configurable thresholds -- Run before marking tasks complete - -### Security Scanning -- `security_scan_comprehensive` โ€” SAST/DAST vulnerability scanning -- Run after changes to auth, security, or middleware code - -### Defect Prediction -- `defect_predict` โ€” AI analysis of code complexity and change history - -### Learning & Memory -- `memory_store` โ€” Store patterns and learnings for future reference -- `memory_query` โ€” Query past patterns before starting work -- Always store successful patterns after task completion - -## Best Practices - -1. **Test Pyramid**: 70% unit, 20% integration, 10% e2e -2. **AAA Pattern**: Arrange-Act-Assert for clear test structure -3. **One assertion per test**: Test one behavior at a time -4. **Descriptive names**: `should_returnValue_when_condition` -5. **Mock at boundaries**: Only mock external dependencies -6. **Edge cases first**: Test boundary conditions, not just happy paths - - -## Beads Issue Tracker - -This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. - -### Quick Reference - -```bash -bd ready # Find available work -bd show # View issue details -bd update --claim # Claim work -bd close # Complete work -``` - -### Rules - -- Use `bd` for ALL task tracking โ€” do NOT use TodoWrite, TaskCreate, or markdown TODO lists -- Run `bd prime` for detailed command reference and session close protocol -- Use `bd remember` for persistent knowledge โ€” do NOT use MEMORY.md files - -**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. - -## Agent Context Profiles - -The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. - -- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. -- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. -- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. - -## Session Completion - -This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. - -1. **File issues for remaining work** - Create beads for anything that needs follow-up -2. **Run quality gates** (if code changed) - Tests, linters, builds -3. **Update issue status** - Close finished work, update in-progress items -4. **Handle git/sync by active profile**: - ```bash - # Conservative/minimal/default: report status and proposed commands; wait for approval. - git status - - # Team-maintainer opt-in only, unless current instructions forbid it: - git pull --rebase - bd dolt push - git push - git status - ``` -5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step - -**Critical rules:** -- Explicit user or orchestrator instructions override this Beads block. -- Do not commit or push without clear authority from the active profile or the current user request. -- If a required sync or push is blocked, stop and report the exact command and error. - - - -## Beads Issue Tracker - -Use Beads (`bd`) for durable task tracking in repositories that include it. Use the `beads` skill at `.agents/skills/beads/SKILL.md` (project install) or `~/.agents/skills/beads/SKILL.md` (global install) for Beads workflow guidance, then use the `bd` CLI for issue operations. - -### Quick Reference - -```bash -bd ready # Find available work -bd show # View issue details -bd update --claim # Claim work -bd close # Complete work -bd prime # Refresh Beads context -``` +> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. -### Rules +## CLI -- Use `bd` for all task tracking; do not create markdown TODO lists. -- Run `bd prime` when Beads context is missing or stale. Codex 0.129.0+ can load Beads context automatically through native hooks; use `/hooks` to inspect or toggle them. -- Keep persistent project memory in Beads via `bd remember`; do not create ad hoc memory files. +| Task | Read this skill file | +| -------------------------------------------- | ----------------------------------------------------------- | +| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | -**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. - + diff --git a/CLAUDE.md b/CLAUDE.md index 354d97f..d4cbaa2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,72 +1,46 @@ - + # emailibrium -## Swarm Config +Vector-native, local-first email intelligence: semantic search, clustering, classification, and inbox cleanup over 10k+ emails with no cloud processing. Rust backend + React SPA. -- **Topology**: hierarchical-mesh (anti-drift) -- **Max Agents**: 15 -- **Memory**: hybrid +## Layout -```bash -ruflo swarm init --topology hierarchical --max-agents 15 --strategy specialized -``` - -## Agentic QE v3 - +| Path | What | Stack | +| ---------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | +| `backend/` | Axum REST + SSE API and the intelligence layer | Rust 1.96 (edition 2021), SQLx/SQLite, Moka, Redis | +| `backend/src/vectors/` | 22+ vector intelligence modules: embedding, HNSW search, SONA learning, clustering, RAG, encryption | โ€” | +| `backend/src/mcp/` | MCP server exposing email tools | โ€” | +| `backend/migrations/` | Numbered SQLite migrations โ€” **append the next number, never edit an applied one** | โ€” | +| `frontend/` | pnpm + Turborepo monorepo (app in `apps/web/`) | React 19, TS 5.9, Vite 8, TanStack Router/Query, Zustand, Tailwind | +| `ruvector/` | **Git submodule** (ruvnet/ruvector) โ€” the vector engine. Treat as vendored: don't edit; backend depends on it via path | Rust workspace | +| `docs/` | `architecture.md`, `ADRs/`, `DDDs/`, evaluation, setup/oauth guides | โ€” | +| `config/`, `secrets/` | Runtime config + dev secrets (never commit secrets) | โ€” | +## Build & Test โ€” Makefile-driven, not npm - -## Beads Issue Tracker - -This project uses **bd (beads)** for issue tracking. Run `bd prime` to see full workflow context and commands. - -### Quick Reference +The root `package.json` only wires Husky; **do not run `npm build`/`npm test`**. Use `make`: ```bash -bd ready # Find available work -bd show # View issue details -bd update --claim # Claim work -bd close # Complete work +make ci # format-check + lint + typecheck + test (run before committing) +make test # backend (cargo) + frontend (Vitest) +make build # build everything +make dev # full stack: backend :8080, frontend :3000 +make lint # code + docs (markdownlint, yamllint) +make audit # cargo-audit + npm audit +make help # all targets ``` -### Rules - -- Use `bd` for ALL task tracking โ€” do NOT use TodoWrite, TaskCreate, or markdown TODO lists -- Run `bd prime` for detailed command reference and session close protocol -- Use `bd remember` for persistent knowledge โ€” do NOT use MEMORY.md files - -**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/SYNC_CONCEPTS.md for details and anti-patterns. - -## Agent Context Profiles - -The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. - -- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. -- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. -- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. - -## Session Completion +Backend-only: `cd backend && cargo test` / `cargo clippy`. Frontend-only: `cd frontend && pnpm test` / `pnpm lint` / `pnpm typecheck`. -This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. +Backend Cargo features: `vectors` (default), `builtin-llm` (llama-cpp, opt-in), `proptest`. -1. **File issues for remaining work** - Create beads for anything that needs follow-up -2. **Run quality gates** (if code changed) - Tests, linters, builds -3. **Update issue status** - Close finished work, update in-progress items -4. **Handle git/sync by active profile**: - ```bash - # Conservative/minimal/default: report status and proposed commands; wait for approval. - git status +## Conventions - # Team-maintainer opt-in only, unless current instructions forbid it: - git pull --rebase - git push - git status - ``` -5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step +- **Decisions are ADR/DDD-gated.** Before changing architecture, vector storage, learning, or AI providers, check `docs/ADRs/` and `docs/DDDs/` โ€” e.g. ADR-003 fixes RuVector as the primary vector store. Record new decisions as an ADR. +- **Privacy is a hard guarantee, not a setting.** Embeddings/models run and stay local; cloud AI is strictly opt-in. Don't add code paths that send email content off-machine by default. +- Encryption at rest is AES-256-GCM + Argon2id โ€” keep crypto changes within `backend/src/vectors/encryption.rs` and consent-gated. +- For code navigation, impact analysis, and safe refactors, use the GitNexus MCP tools per `AGENTS.md`. -**Critical rules:** -- Explicit user or orchestrator instructions override this Beads block. -- Do not commit or push without clear authority from the active profile or the current user request. -- If a required sync or push is blocked, stop and report the exact command and error. - + + From dbc75c883eda8c3434ef7b8fd087cb122ce32d30 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 10:32:53 -0700 Subject: [PATCH 03/24] fix(frontend): stop swallowing test/audit failures in Makefile `make test` and `make audit` (and thus root `make test`/`make audit`) could report green even when Vitest tests failed or pnpm audit found vulnerabilities, because both targets masked their exit code (`|| true`, `2>/dev/null || echo ...`). Found while wiring up the autopilot quality gate, which depends on these commands being honest. --- frontend/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/Makefile b/frontend/Makefile index 647de12..1f62e84 100644 --- a/frontend/Makefile +++ b/frontend/Makefile @@ -91,7 +91,7 @@ clean: ## Clean build artifacts .PHONY: test test: ## Run unit tests (Vitest) - @$(TURBO) test || true + @$(TURBO) test .PHONY: test-e2e test-e2e: ## Run E2E tests (Playwright) @@ -127,7 +127,7 @@ deadcode: ## Check for unused exports .PHONY: audit audit: ## Security audit - @$(PNPM) audit --prod 2>/dev/null || echo "$(YELLOW)pnpm audit completed with findings$(RESET)" + @$(PNPM) audit --prod # ============================================================================ # Dependency Management From 4c595ba71c23fe2f4de461d1fcb02e39a23dc489 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 10:32:55 -0700 Subject: [PATCH 04/24] ci: run PR checks on PRs into develop, not just main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The autopilot ci-build-optimization pipeline uses develop as its integration branch (base) in pr_ci mode. Without develop in pull_request.branches, phase PRs into develop would trigger zero CI checks, and autopilot's merge guard treats that as SKIPPED rather than green โ€” it would refuse to ever merge a phase. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 747767d..8353bbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, develop] pull_request: - branches: [main] + branches: [main, develop] workflow_dispatch: concurrency: From 789d5d58493f6574638b945bb1b7c70678a731f4 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 10:33:11 -0700 Subject: [PATCH 05/24] chore(autopilot): scaffold ci-build-optimization pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the autopilot stack profile, the 7-phase pipeline plan derived from OPTIMIZATION_SPEC.md (rust-optimizer's CI/build/Docker/dependency audit), and the seeded run ledger. autonomy: pr_ci, base: develop (created on orchestrate's first firing), trunk: main โ€” phase PRs land on develop; a single develop -> main PR is the human review point for the full body of work. --- .autopilot/pipeline.yml | 187 +++++++++++++++ .autopilot/profile.yml | 132 +++++++++++ .autopilot/runs/ci-build-optimization.jsonl | 2 + OPTIMIZATION_SPEC.md | 238 ++++++++++++++++++++ 4 files changed, 559 insertions(+) create mode 100644 .autopilot/pipeline.yml create mode 100644 .autopilot/profile.yml create mode 100644 .autopilot/runs/ci-build-optimization.jsonl create mode 100644 OPTIMIZATION_SPEC.md diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml new file mode 100644 index 0000000..9d3b8ed --- /dev/null +++ b/.autopilot/pipeline.yml @@ -0,0 +1,187 @@ +# pipeline.yml โ€” the autopilot feature pipeline manifest. +# Lives at .autopilot/pipeline.yml in the TARGET repo. +# +# Generated by autopilot:plan on 2026-07-31 from OPTIMIZATION_SPEC.md (produced by +# rust-optimizer:optimize's CI/build/Docker/dependency audit; checkpoint 1 โ€” curating +# the findings โ€” was already done by the user before this plan was authored). + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# YOU SUPPLY THIS +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +feature_id: "ci-build-optimization" +goal: "Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md." +spec: "OPTIMIZATION_SPEC.md" + +references: + prd: "" + plan: "" + adr_dir: "docs/ADRs" + ddd_dir: "docs/DDDs" + extra: [] + +# Branch model. trunk is NEVER merged autonomously โ€” a human always merges the final PR. +# Revised 2026-07-31 (user request): each phase branches off `develop`, PRs into `develop`, and +# auto-merges once CI is green. When every phase has landed, orchestrate opens ONE `develop -> main` +# PR and STOPS โ€” that PR is where a human reviews the full body of work from all phases before it +# ever reaches `main`. `develop` did not exist as a branch before this pipeline; orchestrate's STEP A +# creates it from `main` (locally + pushed to origin) idempotently on its first firing โ€” nothing to +# do here. .github/workflows/ci.yml's `pull_request.branches` was widened to `[main, develop]` in the +# same sitting (2026-07-31) so phase PRs into `develop` actually get CI-checked โ€” without that, every +# phase PR would show zero checks and orchestrate's anti-vacuous-green guard would refuse to merge. +trunk: main +base: develop + +# Autonomy mode: pr_ci โ€” branch -> PR -> CI -> bounded fix-loop -> squash-merge, no per-phase +# human checkpoint. NOTE: OPTIMIZATION_SPEC.md's own preamble recommends `reviewed` for the first +# run on this pipeline; the user explicitly chose pr_ci instead when confirming this plan +# (2026-07-31). CI covers PRs into base `develop` (see branch-model note above; profile.yml +# ci.base_coverage: covered), and qe-court (accelerators.qe_court, 3 vendors) will convene on +# risk_phases and the final develop->main integration PR as an extra adversarial check given +# there's no human checkpoint per phase. +autonomy: pr_ci + +fix_budget: 5 + +max_parallel: 1 +requeue_budget: 2 + +# F-5 (nextest archive restructuring), F-2 (Docker build context/submodules), and F-1 (Docker +# publish stage) carry real blast radius (release artifact correctness, CI restructuring) โ€” see +# OPTIMIZATION_SPEC.md's own risk_phases: [F-5, F-2, F-1]. +risk_phases: [4, 5, 6] + +court: auto + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# GENERATED by `autopilot:plan` โ€” edit freely +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +phases: + - id: 0 + goal: "backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps." + track: "" + deliverables: + - "Remove the unused `ruvector-collections` path dependency (0 source refs; a submodule path-crate compiled on every build for nothing)." + - "Remove the unused `encoding_rs` optional dependency, including its `dep:encoding_rs` reference in the `builtin-llm` feature." + - "Add a `[package.metadata.cargo-machete]` `ignored = [\"apalis\", \"apalis-sql\"]` allowlist with a rationale comment for the planned-but-not-yet-wired job queue deps." + definition_of_done: + - "cmd: cargo machete backend" + - "cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm" + - "cmd: cargo test --manifest-path backend/Cargo.toml" + - "grep:absent: ruvector-collections in backend/Cargo.toml" + - "grep:absent: dep:encoding_rs in backend/Cargo.toml" + - "grep: cargo-machete in backend/Cargo.toml" + conventions: "" + depends_on: [] + touches: ["backend/Cargo.toml"] + adrs: [] + ddd: [] + + - id: 1 + goal: "CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures." + track: "" + deliverables: + - "Set `CARGO_INCREMENTAL=0` and `CARGO_PROFILE_TEST_DEBUG=0` in ci.yml's workflow-level `env:` block." + - "Replace `cargo install --locked cargo-audit` (compiles from source) with a prebuilt binary via `taiki-e/install-action` in the rust-audit job." + - "Add `if-no-files-found: ignore` to the changelog `upload-artifact` step in release.yml so a missing path never fails the release job." + definition_of_done: + - "grep: 'CARGO_INCREMENTAL: ?0' in .github/workflows/ci.yml" + - "grep: 'CARGO_PROFILE_TEST_DEBUG: ?0' in .github/workflows/ci.yml" + - "grep: taiki-e/install-action in .github/workflows/ci.yml" + - "grep:absent: 'cargo install --locked cargo-audit' in .github/workflows/**" + - "grep: 'if-no-files-found: ignore' in .github/workflows/release.yml" + conventions: "" + depends_on: [] + touches: [".github/workflows/ci.yml", ".github/workflows/release.yml"] + adrs: [] + ddd: [] + + - id: 2 + goal: "Per-PR CI stops running unenforced 'performance gate' theater; Lighthouse becomes a real, on-demand, non-theater check." + track: "" + deliverables: + - "Remove the `lighthouse` (was `continue-on-error: true`) and `bundlewatch` (warn-only, no config file) jobs from the per-PR ci.yml โ€” neither ever gated anything." + - "Create .github/workflows/lighthouse.yml, triggered by `workflow_dispatch` only: install, `pnpm turbo build`, then `treosh/lighthouse-ci-action` against the retained frontend/lighthouserc.json, without `continue-on-error`." + - "Fix or delete the misleading `# Performance gates (ADR-005)` comment in ci.yml (ADR-005 is actually the Tauriโ†’SPA migration)." + definition_of_done: + - "grep:absent: continue-on-error in .github/workflows/ci.yml" + - "grep:absent: bundlewatch in .github/workflows/ci.yml" + - "grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml" + - "grep:absent: ADR-005 in .github/workflows/ci.yml" + - "grep: workflow_dispatch in .github/workflows/lighthouse.yml" + - "grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml" + - "grep:absent: continue-on-error in .github/workflows/lighthouse.yml" + - "cmd: test -f frontend/lighthouserc.json" + conventions: "This also resolves the frontend half of F-5's redundant-build concern: removing both jobs drops 2 of the 3 full `pnpm install` + `pnpm turbo build` runs per PR (only frontend-quality's build remains)." + depends_on: [] + touches: [".github/workflows/ci.yml", ".github/workflows/lighthouse.yml", "frontend/lighthouserc.json"] + adrs: [] + ddd: [] + + - id: 3 + goal: "Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners." + track: "" + deliverables: + - "Make backend/rust-toolchain.toml (MSRV 1.96.0) the single source of truth: pin ci.yml/release.yml's `dtolnay/rust-toolchain` step to that version instead of bare `@stable`." + - "Install `lld` in the Rust CI jobs and set `-Clink-arg=-fuse-ld=lld` (via RUSTFLAGS or backend/.cargo/config.toml) to cut incremental link time." + - "Point heavy jobs (rust-test, Docker builds) at `runs-on: \\${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }}` so a runner upgrade is opt-in and safe by default." + definition_of_done: + - "grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**" + - "grep: '1.96' in .github/workflows/**" + - "grep: rust-toolchain.toml in backend" + - "grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml" + - "grep: lld in .github/workflows/**" + - "grep: vars.HEAVY_RUNNER in .github/workflows/**" + conventions: "" + depends_on: [] + touches: [".github/workflows/ci.yml", ".github/workflows/release.yml", "backend/.cargo/config.toml"] + adrs: [] + ddd: [] + + - id: 4 + goal: "The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates." + track: "" + deliverables: + - "A build-once job producing `cargo nextest archive --archive-file nextest.tar.zst`, with nextest installed via `taiki-e/install-action` and a new `.config/nextest.toml`." + - "The test job consumes the archive via `cargo nextest run --archive-file ...` instead of recompiling from scratch." + definition_of_done: + - "grep: 'nextest archive' in .github/workflows/**" + - "grep: archive-file in .github/workflows/**" + - "cmd: cargo test --manifest-path backend/Cargo.toml" + conventions: "Removing the separate compile may expose a latent flaky test โ€” treat that as exposed, not caused, and fix it deterministically rather than reintroducing the redundant compile (OPTIMIZATION_SPEC.md F-5 validation note). Baseline warm time-to-green: 311s median; re-measure after." + depends_on: [] + touches: [".github/workflows/ci.yml", ".config/nextest.toml"] + adrs: [] + ddd: [] + + - id: 5 + goal: "The Docker build context actually contains everything the `runtime` target needs to compile โ€” the ruvector submodule is no longer silently missing." + track: "" + deliverables: + - "Rework the Docker build context to include the vendored `ruvector` submodule crates (e.g. repo-root `context:`, with Dockerfile `COPY` lines bringing in both backend/ and ruvector/)." + - "Add `submodules: true` (or recursive) to the checkout step of every job that builds a Docker image, in docker.yml and release.yml." + definition_of_done: + - "grep: 'submodules: true' in .github/workflows/docker.yml" + - "grep: 'submodules: true' in .github/workflows/release.yml" + - "grep: ruvector in backend/Dockerfile" + - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from repo root with the reworked context)" + conventions: "ruvector is a vendored third-party git submodule consumed via path deps โ€” do not edit it, only fix how the build context references it (OPTIMIZATION_SPEC.md preamble)." + depends_on: [] + touches: [".github/workflows/docker.yml", ".github/workflows/release.yml", "backend/Dockerfile"] + adrs: [] + ddd: [] + + - id: 6 + goal: "The published images are the runnable `runtime` stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable `development` dev-tooling stage." + track: "" + deliverables: + - "Every `docker/build-push-action` step (both jobs in docker.yml, and release.yml) passes `target: runtime`, for both the backend and frontend images." + definition_of_done: + - "grep: 'target: runtime' in .github/workflows/docker.yml" + - "grep: 'target: runtime' in .github/workflows/release.yml" + - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help (exits 0 โ€” confirms the runtime stage ships the compiled binary, not the cargo-watch dev stage)" + - "cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend (exits 0 โ€” confirms the frontend image builds from the nginx runtime stage, not the node dev stage)" + conventions: "`cmd:` lines that invoke cargo must use `--manifest-path backend/Cargo.toml` (or an equivalent path arg) so they run correctly from the repo root (OPTIMIZATION_SPEC.md preamble)." + depends_on: [5] + touches: [".github/workflows/docker.yml", ".github/workflows/release.yml"] + adrs: [] + ddd: [] diff --git a/.autopilot/profile.yml b/.autopilot/profile.yml new file mode 100644 index 0000000..f0c608e --- /dev/null +++ b/.autopilot/profile.yml @@ -0,0 +1,132 @@ +# profile.yml โ€” the stack profile for autopilot's quality gate. +# Lives at .autopilot/profile.yml in the TARGET repo. +# +# Generated by autopilot:detect and confirmed by the user on 2026-07-31. +# Stack: Rust/Cargo backend (backend/, package `emailibrium`) + pnpm/Turborepo +# React frontend (frontend/apps/web + frontend/packages), orchestrated by the +# root Makefile. Edit anything that drifts from reality โ€” the gate trusts this file. + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# COMMANDS the gate runs. Leave a value empty ("") to skip that check. +# These are placeholders the gate template resolves as {{commands.}}. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +commands: + # Bring up anything the tests need (db, services). Runs once before the gate. Optional. + infra_up: "make docker-up-dev" # postgres, redis, qdrant, backend, frontend (hot-reload) + # Fast formatting check (non-mutating). The gate fails if this fails. + format_check: "make format-check" # backend cargo fmt --check + frontend prettier --check + docs + # Static analysis / linters. + lint: "make lint" # backend cargo clippy -D warnings + frontend eslint + docs lint + # Build everything (the phase must compile/bundle). + build: "make build" # backend cargo build + frontend turbo build + # Primary/unit test suite (fast, no external infra ideally). + # NOTE: frontend/Makefile's `test` and `audit` targets used to swallow failures + # (`... || true` / `... 2>/dev/null || echo ...`), which would have let a broken + # phase pass the gate silently. Fixed directly in frontend/Makefile during detect + # (2026-07-31); the split commands below no longer depend on that fix holding. + test: "make -C backend test && (cd frontend && pnpm turbo test)" + # Integration tests (may need infra_up). Optional. + test_integration: "make -C backend test-integration" # cargo test --test '*' + # Frontend/UI tests, only relevant if the phase touches the UI. Optional. + test_frontend: "cd frontend && pnpm turbo test" # vitest run (non-watch) via turbo + # Dependency / vulnerability audit. Optional. + audit: "make -C backend audit && (cd frontend && pnpm audit --prod)" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# CONVENTIONS โ€” free-text house style, injected into the runner's 'match the +# existing code' step so new code looks native. Be specific; point at exemplar files. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +conventions: | + backend (Rust, Cargo workspace, package `emailibrium`, edition per backend/rust-toolchain.toml): + - DDD-ish layering per domain module, e.g. src/cleanup/{domain,repository,api,orchestrator}/: + domain = pure logic + ports (traits), repository = persistence adapters, api = HTTP handlers, + orchestrator = use-case coordination. Other top-level modules: api, cache, content, db, + email, events, mcp, middleware, rules, tools, vectors. + - Tests: integration + property tests live in backend/tests/*.rs (api_integration.rs, + mcp_integration.rs, proptest_*.rs, *_evaluation.rs); unit tests are inline `#[cfg(test)]` + modules alongside the code they test. + - Lint is strict (clippy -D warnings) but allows dead_code/unused_variables/unused_imports/ + unused_mut during active development โ€” see Makefile lint target. + + frontend (React/TypeScript, pnpm + Turborepo monorepo under frontend/): + - apps/web is the SPA; feature-slice layout under src/features//{components,hooks}; + shared code in src/shared; cross-cutting services in src/services; app shell in src/app. + - Tests colocated in __tests__/ subdirectories or as *.test.ts(x) next to source (see + src/services/ai/__tests__/, src/features/settings/hooks/__tests__/, + src/features/email/utils/groupBySender.test.ts). Run via Vitest โ€” always `vitest run` + (non-watch); `test:watch` is the separate opt-in script. + - E2E via Playwright (apps/web/e2e, `make test-e2e` โ€” not part of the default gate `test`). + + docs: architecture decisions in docs/ADRs/ADR-NNN-*.md, domain design in docs/DDDs/DDD-NNN-*.md + (start with DDD-000-context-map.md for the bounded-context overview). Rich corpus โ€” worth + consulting for `plan` decomposition of any feature touching an existing bounded context. + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# CI โ€” the merge authority in pr_ci mode. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +ci: + provider: github # github is the supported provider today + # When true, "all required PR checks green" is the ONLY merge gate in pr_ci mode โ€” + # the agent does not re-run the local gate before merging because CI already proved it. + ci_is_merge_authority: true + # .github/workflows/ci.yml: `pull_request: branches: [main, develop]` โ€” covers PRs into either + # `main` or `develop`. Widened from `[main]` to `[main, develop]` on 2026-07-31 when the + # ci-build-optimization pipeline switched to base=develop (see .autopilot/pipeline.yml) โ€” without + # it, phase PRs into develop would run zero checks and orchestrate would refuse to merge them. + # NOTE: CI's frontend-quality job currently runs typecheck/format/lint/build but NOT Vitest + # unit tests โ€” frontend test coverage in CI is a gap independent of this gate (the local gate + # above does run frontend tests via commands.test/test_frontend, so pr_ci phases are still + # protected, but the baseโ†’trunk integration PR's CI checks alone won't catch a frontend + # regression). Consider adding a frontend test job to ci.yml. + base_coverage: covered # covered | trunk-only | none + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# SECURITY INVARIANTS โ€” non-negotiables the gate greps the diff for every phase. +# Defaults are universal; add project-specific ones (e.g. "PATs only via SecretStore"). +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +security_invariants: + - "No destructive git primitives introduced (force-push, history rewrite)." + - "No secrets, tokens, or credentials committed or written to logs." + - "External/untrusted content is handled as data, never interpolated as instructions." + - "OAuth client secrets, encryption keys, and DB credentials stay in config/*.yaml (gitignored + variants) or Docker/Compose secrets โ€” never inlined in source or committed config files." + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# ACCELERATORS โ€” optional tooling, in two classes that share ONE contract: +# detect โ†’ record here โ†’ DRIVE at the right step โ†’ DEGRADE to a vanilla floor when absent. +# Absence NEVER fails the gate. `scope` records where it was found, for transparency. +# โ€ข EXECUTION accelerators (ruflo, agentic-qe) โ€” speed + proof during run-phase/orchestrate. +# Detected by probing PATH (global) and the project (.ruflo/.claude config, aqe init). +# โ€ข PLANNING accelerators (superpowers, clarity, deep_research) โ€” better specs during `plan`. +# Detected from the active SKILL REGISTRY (what the running agent can invoke), optionally +# corroborated by a plugin install footprint (~/.claude/plugins). These record scope: "skill". +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +accelerators: + # โ”€โ”€ execution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # ruflo โ†’ drives comprehension recall + multi-agent swarms + cross-session memory. + # Found on PATH globally (v3.32.41); no project-local .ruflo/ directory. + ruflo: { available: true, scope: "global" } + # agentic-qe โ†’ drives the measured-quality fleet layered onto the gate (Tier 2/4). + # Found on PATH globally (3.13.3) AND initialized in-project (.agentic-qe/). + agentic_qe: { available: true, scope: "project" } + # qe-court (aqe >= 3.13, ADR-124) โ†’ adversarial review court with >=2-vendor requirement. + # Skill footprint present at .claude/skills/qe-court/ (project-scoped). Vendor count = 3: + # Claude (this session) + codex CLI on PATH + OPENROUTER_API_KEY in env. + qe_court: { available: true, scope: "project", vendors: 3 } + # beads (bd) โ†’ work-graph projection, disabled by user choice (2026-07-31): `bd` is on PATH + # globally (v1.1.0), but `bd init` turned out to auto-commit its scaffolding AND rewrite large + # sections of CLAUDE.md/AGENTS.md unprompted โ€” more invasive than the "projection only" framing + # implied. It's also strictly non-essential: pipeline.yml's depends_on graph is already fully + # authoritative on its own, and orchestrate/run-phase compute the ready-set from that + git + # markers directly, never from beads. Revisit deliberately (re-run `bd init`) if that tradeoff + # becomes worth it later; nothing about autopilot's correctness depends on it either way. + beads: { available: false, scope: "" } + # the /code-review skill/command โ€” the Tier-3 floor, assumed present in Claude Code. + code_review: { available: true, scope: "global" } + # โ”€โ”€ planning (skill-based; scope: "skill" | "") โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + # superpowers โ†’ active in this session's skill registry (brainstorming, TDD, debugging, etc). + superpowers: { available: true, scope: "skill" } + # clarity โ†’ active in this session's skill registry (spec generation from references). + clarity: { available: true, scope: "skill" } + # deep_research โ†’ not present in the active skill registry. + deep_research: { available: false, scope: "" } diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl new file mode 100644 index 0000000..bfb8283 --- /dev/null +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -0,0 +1,2 @@ +{"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"main","autonomy":"pr_ci","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T09:56:11-07:00"} +{"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: base changed from main to develop (user request 2026-07-31) so phase PRs land on a dedicated integration branch and a single develop->main PR gives a human review of the full body of work. .github/workflows/ci.yml pull_request.branches widened to [main, develop] in the same sitting.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T10:27:37-07:00"} diff --git a/OPTIMIZATION_SPEC.md b/OPTIMIZATION_SPEC.md new file mode 100644 index 0000000..2ad62b6 --- /dev/null +++ b/OPTIMIZATION_SPEC.md @@ -0,0 +1,238 @@ +# OPTIMIZATION_SPEC.md + +> Generated by rust-optimizer. Two human checkpoints: (1) curate these findings โ€” **done**; +> (2) approve the autopilot plan. Recommended first run: `autonomy: reviewed`. +> +> Repo: `pacphi/emailibrium` (personal, **PUBLIC**). Rust crate lives in `backend/` (single crate, +> not a workspace); `ruvector` is a vendored git submodule consumed via path deps โ€” **do not edit it**. +> `cmd:` Definition-of-Done lines that touch cargo use `--manifest-path backend/Cargo.toml` (or a path +> arg) so they run from the repo root. +> +> **Baked-in decisions from checkpoint 1:** +> +> - **F-4:** keep `apalis` / `apalis-sql` (planned, not-yet-wired job queue) via a documented +> `cargo-machete` ignore; remove only `ruvector-collections` and `encoding_rs`. +> - **F-3:** remove the perf-gate theater from the per-PR path rather than enforce it. **Lighthouse +> moves to a new, manually-triggerable (`workflow_dispatch`) `lighthouse.yml` workflow** โ€” capability +> preserved, per-PR cost removed. **bundlewatch** was pure warn-only theater with no config file, so +> it is deleted. No real ADR mandates these โ€” the `(ADR-005)` comment in `ci.yml` is a mis-citation +> (ADR-005 is the Tauriโ†’SPA migration); fix that comment as part of the change. + +--- + +### F-1 โ€” Docker build/push must target the `runtime` stage, not `development` + +- **severity:** critical +- **evidence:** .github/workflows/docker.yml:16-22, .github/workflows/docker.yml:30-36, .github/workflows/release.yml:169-175, backend/Dockerfile:21-25, frontend/Dockerfile:18 +- **deliverable:** Every `docker/build-push-action` step passes `target: runtime` so buildx builds the + slim runtime stage instead of the last-defined `development` stage. Currently the published + `ghcr.io/pacphi/emailibrium/backend` image is the `rust:1.96-slim` + `cargo-watch` dev stage with no + compiled binary, no `entrypoint.sh`, and no `migrations/` โ€” it cannot run. Same defect for frontend + (ships the node dev stage, not `nginx:alpine`). +- **definition_of_done:** + - grep: target: runtime in .github/workflows/docker.yml + - grep: target: runtime in .github/workflows/release.yml + - grep:absent: cargo-watch in (verified via F-2 smoke run) +- **validation:** `push: false` smoke build of both images (see F-2), then + `docker run --rm /app/emailibrium --help` (or equivalent) must find the binary; the + frontend image must serve via nginx, not a dev server. +- **est_impact:** correctness โ€” restores a runnable published release artifact; makes the `docker.yml` + build a real compile gate instead of a `CACHED` no-op. +- **risk:** high +- **depends_on:** [F-2] + +### F-2 โ€” Make the runtime image buildable (ruvector context + submodules) + +- **severity:** critical +- **evidence:** backend/Cargo.toml:104-106 (path deps into `../ruvector/...`), backend/Dockerfile:1-7 (context assumes `./backend` only), .github/workflows/docker.yml:14 (checkout without `submodules: true`) +- **deliverable:** Rework the Docker build so the `ruvector` submodule crates are inside the build + context (e.g. set the build `context:` to the repo root and update the Dockerfile `COPY` lines to + bring in both `backend/` and `ruvector/`), and add `submodules: true` (or `recursive`) to the + checkout of every job that builds a Docker image (`docker.yml`, `release.yml`). This is the latent + blocker that the `development`-stage shortcut hides โ€” **exposed by F-1, not caused by it.** +- **definition_of_done:** + - grep: submodules: true in .github/workflows/docker.yml + - grep: submodules: true in .github/workflows/release.yml + - grep: ruvector in backend/Dockerfile + - cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from repo root, with the reworked context) +- **validation:** `push: false` smoke build on the default amd64 runner; confirm the built image + contains `/app/emailibrium`, `/app/migrations`, and `/app/entrypoint.sh`. +- **est_impact:** correctness โ€” precondition for F-1; without it, `target: runtime` fails to resolve + the ruvector path deps. +- **risk:** high +- **depends_on:** [] + +### F-3 โ€” Move Lighthouse to a manual workflow; delete bundlewatch theater + +- **severity:** medium +- **evidence:** .github/workflows/ci.yml:156 (stale `(ADR-005)` comment), .github/workflows/ci.yml:157-182 (lighthouse job, `continue-on-error: true`), .github/workflows/ci.yml:184-222 (bundlewatch job, oversized check only `echo`s a warning) +- **deliverable:** Remove the `lighthouse` and `bundlewatch` jobs from the per-PR `ci.yml`. Neither + gated anything โ€” Lighthouse was `continue-on-error: true`, bundlewatch never exited non-zero. + - **Lighthouse:** re-create as a new standalone `.github/workflows/lighthouse.yml` triggered by + `workflow_dispatch` (manual, on-demand). It checks out, does one `pnpm install` + `pnpm turbo build`, + then runs `treosh/lighthouse-ci-action` against the retained `frontend/lighthouserc.json` โ€” **without + `continue-on-error`**, so a manual run surfaces real pass/fail instead of theater. + - **bundlewatch:** delete (pure warn-only, no config file to preserve). + - Fix or delete the misleading `# Performance gates (ADR-005)` comment in `ci.yml`. + - **Bonus:** removes both jobs from every PR, eliminating 2 of the 3 redundant full + `pnpm install` + `pnpm turbo build` runs per PR โ€” this resolves the frontend half of F-5. +- **definition_of_done:** + - grep:absent: continue-on-error in .github/workflows/ci.yml + - grep:absent: bundlewatch in .github/workflows/ci.yml + - grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml + - grep:absent: ADR-005 in .github/workflows/ci.yml + - grep: workflow_dispatch in .github/workflows/lighthouse.yml + - grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml + - grep:absent: continue-on-error in .github/workflows/lighthouse.yml + - cmd: test -f frontend/lighthouserc.json +- **validation:** Per-PR CI still green and the frontend builds exactly once per PR (in + `frontend-quality`); manually dispatching `lighthouse.yml` runs a Lighthouse audit and reports real + status. +- **est_impact:** deterministic: frontend full builds per PR 3 โ†’ 1; removes 2 jobs from every PR run; + Lighthouse cost moves to on-demand only. +- **risk:** low +- **depends_on:** [] + +### F-4 โ€” Dependency hygiene: remove 2 unused, document-ignore 2 planned + +- **severity:** medium +- **evidence:** backend/Cargo.toml:105 (`ruvector-collections`, 0 source refs), backend/Cargo.toml:13 & :111 (`encoding_rs` โ€” dead `builtin-llm` feature stub, 0 refs), backend/Cargo.toml:79-80 (`apalis` / `apalis-sql`), backend/src/content/jobs.rs:15,176 (apalis referenced in comments only) +- **deliverable:** Remove `ruvector-collections` (a submodule path-crate compiled on every build for + nothing) and `encoding_rs` (drop `dep:encoding_rs` from the `builtin-llm` feature and the optional + dep). Keep `apalis` / `apalis-sql` (planned persistent job queue) but silence the machete false-red + with a documented allowlist in `backend/Cargo.toml`: + `[package.metadata.cargo-machete]` `ignored = ["apalis", "apalis-sql"]` plus a rationale comment. +- **definition_of_done:** + - cmd: cargo machete backend + - cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm + - cmd: cargo test --manifest-path backend/Cargo.toml + - grep:absent: ruvector-collections in backend/Cargo.toml + - grep:absent: dep:encoding_rs in backend/Cargo.toml + - grep: cargo-machete in backend/Cargo.toml +- **validation:** `cargo build`/`cargo test` still pass; `Cargo.lock` shrinks; `ruvector-collections` + no longer appears in `cargo tree`. +- **est_impact:** deterministic: unused-dep candidates 4 โ†’ 0 (2 removed, 2 documented); drops a + submodule crate from every compile; smaller `Cargo.lock` and supply-chain surface. +- **risk:** low +- **depends_on:** [] + +### F-5 โ€” Build the Rust crate once, share via nextest archive + +- **severity:** medium +- **evidence:** .github/workflows/ci.yml:47-52 (`cargo clippy --workspace --all-targets` โ€” full compile), .github/workflows/ci.yml:66-71 (`cargo test --workspace` + `cargo bench --no-run` โ€” second full compile) +- **deliverable:** A build-once job producing `cargo nextest archive --archive-file nextest.tar.zst` + that the test job consumes with `cargo nextest run --archive-file โ€ฆ`; install nextest via + `taiki-e/install-action`; add `.config/nextest.toml`. Each of clippy and test currently recompiles + the full crate **and** the heavy ruvector path crates in a separate job/cache. +- **definition_of_done:** + - grep: nextest archive in .github/workflows/** + - grep: archive-file in .github/workflows/** + - cmd: cargo test --manifest-path backend/Cargo.toml +- **validation:** CI green; the test job no longer compiles the crate from scratch. **Removing the + separate compile may expose a latent flaky test โ€” frame as exposed, not caused**, and fix + deterministically if one surfaces. +- **est_impact:** deterministic: full-workspace-compiles-per-PR 2 โ†’ 1 (ruvector is heavy, so this is + a real wall-clock win). empirical: re-measure warm time-to-green after (baseline median 311s). +- **risk:** medium +- **depends_on:** [] + +### F-6 โ€” Set `CARGO_INCREMENTAL=0` and `CARGO_PROFILE_TEST_DEBUG=0` + +- **severity:** medium +- **evidence:** .github/workflows/ci.yml:14-16 (env block sets neither) +- **deliverable:** Add both to the workflow-level `env:` so `target/` stays lean and `rust-cache` + save/restore is faster. +- **definition_of_done:** + - grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml + - grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml +- **validation:** CI green; cache artifact size drops on the next warm run. +- **est_impact:** deterministic: smaller `target/`; faster cache steps. +- **risk:** low +- **depends_on:** [] + +### F-7 โ€” Install `cargo-audit` from a prebuilt binary + +- **severity:** medium +- **evidence:** .github/workflows/ci.yml:82-83 (`cargo install --locked cargo-audit` โ€” compiles from source each run) +- **deliverable:** Replace with `taiki-e/install-action` (prebuilt `cargo-audit`). +- **definition_of_done:** + - grep: taiki-e/install-action in .github/workflows/ci.yml + - grep:absent: cargo install --locked cargo-audit in .github/workflows/** +- **validation:** `rust-audit` job still runs `cargo audit` and stays green; job wall-clock drops. +- **est_impact:** deterministic: removes a from-source tool compile per audit run. +- **risk:** low +- **depends_on:** [] + +### F-8 โ€” Single Rust-toolchain source of truth + +- **severity:** low +- **evidence:** backend/rust-toolchain.toml (pins `1.96.0`), .github/workflows/ci.yml:27,41,62 & .github/workflows/release.yml:76 (`dtolnay/rust-toolchain@stable`) +- **deliverable:** Make `backend/rust-toolchain.toml` (MSRV 1.96.0) the single source of truth โ€” pin + the action to that version (or have it read the toolchain file) instead of installing bare `@stable`, + which silently drifts and is overridden by the file inside `backend/` anyway. +- **definition_of_done:** + - grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/** + - grep: 1.96 in .github/workflows/** + - grep: rust-toolchain.toml in backend +- **validation:** CI green; `rustc --version` in CI matches the pinned MSRV. +- **est_impact:** correctness/maintenance โ€” no fmt/clippy version drift between CI and local. +- **risk:** low +- **depends_on:** [] + +### F-9 โ€” Add a fast linker (lld) + +- **severity:** low +- **evidence:** no `fuse-ld=lld` present in workflows or `backend/.cargo/config.toml` +- **deliverable:** Install `lld` in the Rust jobs and set `-Clink-arg=-fuse-ld=lld` (via `RUSTFLAGS` + or `backend/.cargo/config.toml`) to cut link time on incremental rebuilds. +- **definition_of_done:** + - grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml + - grep: lld in .github/workflows/** +- **validation:** CI green; link phase faster on warm runs. +- **est_impact:** medium/low โ€” link-time reduction on incremental builds. +- **risk:** low +- **depends_on:** [] + +### F-10 โ€” Parameterize heavy runners + +- **severity:** low +- **evidence:** .github/workflows/ci.yml:22,36,55 (heavy jobs hardcode `runs-on: ubuntu-latest`) +- **deliverable:** Point heavy jobs (`rust-test`, Docker builds) at + `runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }}`. On this PUBLIC repo the var can later be set + to the free `ubuntu-24.04-arm`; the default keeps every other context safe. A nonexistent label + queues forever, so this parameterization is what makes an upgrade safe. +- **definition_of_done:** + - grep: vars.HEAVY_RUNNER in .github/workflows/** +- **validation:** CI green with the var unset (falls back to `ubuntu-latest`). +- **est_impact:** low/medium โ€” opt-in bigger/native runner with a safe default. +- **risk:** low +- **depends_on:** [] + +### F-11 โ€” `upload-artifact` with `if-no-files-found: ignore` + +- **severity:** low +- **evidence:** .github/workflows/release.yml:128 (changelog `upload-artifact` without the guard) +- **deliverable:** Add `if-no-files-found: ignore` so a missing path never fails the release job. +- **definition_of_done:** + - grep: if-no-files-found: ignore in .github/workflows/release.yml +- **validation:** Release job green even when the artifact path is absent. +- **est_impact:** low โ€” flakiness guard. +- **risk:** low +- **depends_on:** [] + +--- + +## N/A โ€” out of scope (do not plan) + +- **D1 native arm64 split โ€” N/A:** no QEMU/multi-arch build exists; Docker is single-arch amd64. + (The PUBLIC repo *could* use the free `ubuntu-24.04-arm`, but there is no arm64 build to accelerate. + Revisit only if multi-arch publishing is added โ€” F-10 leaves the door open safely.) +- **Larger x64 runners โ€” N/A:** personal account; Team/Enterprise only. +- **A2 (sccache+Swatinem), A5 (double test run), B1 (disk hacks), C1 (concurrency) โ€” N/A:** not present + / already correct. + +--- + +suggested_sequence: [F-4, F-6, F-7, F-11, F-3, F-8, F-9, F-10, F-5, F-2, F-1] + +risk_phases: [F-5, F-2, F-1] From d1f79017696a93523c79791db31fbdc76fdc4937 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 11:01:07 -0700 Subject: [PATCH 06/24] =?UTF-8?q?chore(autopilot:ci-build-optimization):?= =?UTF-8?q?=20re-plan=20=E2=80=94=20add=20phase=207=20(Make=20->=20Just=20?= =?UTF-8?q?migration)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User request: convert the project's Make-based task runner to just, scoped as a phase to run after everything else. Investigated first: just 1.57.0 is already installed locally; no CI workflow, Dockerfile, or husky hook invokes make directly today โ€” only .autopilot/profile.yml's commands.* and human-facing docs do. That's what makes phase 7 need to be strictly last (depends_on: [0..6]): it rewrites profile.yml's make commands to just commands, which every earlier phase's gate is still relying on while those phases run. --- .autopilot/pipeline.yml | 30 +++++++++++++++++++++ .autopilot/runs/ci-build-optimization.jsonl | 1 + 2 files changed, 31 insertions(+) diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml index 9d3b8ed..bef018c 100644 --- a/.autopilot/pipeline.yml +++ b/.autopilot/pipeline.yml @@ -185,3 +185,33 @@ phases: touches: [".github/workflows/docker.yml", ".github/workflows/release.yml"] adrs: [] ddd: [] + + - id: 7 + goal: "just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow." + track: "" + deliverables: + - "A root `justfile` plus `backend/justfile` and `frontend/justfile` that reproduce every load-bearing recipe from the current three Makefiles (root ~61 targets incl. build/test/lint/format-check/audit/ci/docker-*/setup-*/release-*; backend ~15 incl. build/test/test-integration/lint/format-check/audit/bench; frontend ~15 incl. build/test/test-e2e/lint/format-check/typecheck/audit/deadcode), preserving the existing recursive structure (root delegates to backend/frontend โ€” via `just`'s module system (`mod backend`, `mod frontend`; just 1.57.0 is already available and supports it) or an equivalent recipe-delegation pattern, implementer's call) and the `help` target's category listing." + - "Delete `Makefile`, `backend/Makefile`, and `frontend/Makefile`." + - "Update `.autopilot/profile.yml`'s `commands.*` (infra_up, format_check, lint, build, test, test_integration, audit) from `make ...` to the equivalent `just ...` invocations โ€” this MUST land in this phase, not after, since the very next gate run depends on it." + - "Update the actively-maintained docs that instruct readers to run `make` commands: README.md, QUICKSTART.md, CLAUDE.md, docs/setup-guide.md, docs/maintainer-guide.md, docs/deployment-guide.md, docs/releasing.md, docs/user-guide.md, docs/oauth-setup-guide.md. Leave CHANGELOG.md, docs/ADRs/**, docs/plan/**, and docs/research/** untouched โ€” those are historical/point-in-time records, not live instructions." + - "Add a new ADR (docs/ADRs/ADR-0NN-make-to-just-migration.md, next available number) documenting the decision, matching this repo's existing ADR format and its 31 predecessors." + definition_of_done: + - "cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile" + - "cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile" + - "cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list (each parses without error)" + - "cmd: just build (equivalent to old `make build`; must succeed)" + - "cmd: just test (equivalent to old `make test`; must succeed, honoring the frontend swallow-bug fix already landed โ€” a real failure must fail this)" + - "cmd: just lint (equivalent to old `make lint`; must succeed)" + - "cmd: just format-check (equivalent to old `make format-check`; must succeed)" + - "grep:absent: 'make build' in README.md QUICKSTART.md CLAUDE.md" + - "grep:absent: 'make test' in README.md QUICKSTART.md CLAUDE.md" + - "grep: 'just ' in README.md" + - "grep: 'just' in .autopilot/profile.yml" + - "grep:absent: 'make ' in .autopilot/profile.yml" + - "cmd: ls docs/ADRs | grep -qi just" + - "prose: every recipe present in the three deleted Makefiles has a behaviorally equivalent `just` recipe โ€” no silent drops. Cite the mapping for any renamed recipe." + conventions: "This phase touches nearly every file the other six phases also touch, and profile.yml's commands.* are load-bearing for every future gate run โ€” that's why it's last and depends on all of 0-6, not just file-level `touches` overlap. Preserve exact recipe names where sensible (`just build`, `just test`, `just lint`, `just format-check`, `just audit`, `just ci`) so muscle memory and any external scripts/docs need minimal changes. cargo-machete's `[package.metadata.cargo-machete]` block from phase 0 and the audit-ignore file are untouched by this phase โ€” it's a task-runner swap, not a dependency change. Also a good moment to fix the discovered backend/Makefile audit-swallow bug (pl-mk-audit-swallow, .autopilot/discovered/ci-build-optimization.jsonl) โ€” it disappears for free once backend/Makefile is deleted, but the just recipe must not reintroduce an equivalent swallow." + depends_on: [0, 1, 2, 3, 4, 5, 6] + touches: ["Makefile", "justfile", "backend/Makefile", "backend/justfile", "frontend/Makefile", "frontend/justfile", ".autopilot/profile.yml", "README.md", "QUICKSTART.md", "CLAUDE.md", "docs/setup-guide.md", "docs/maintainer-guide.md", "docs/deployment-guide.md", "docs/releasing.md", "docs/user-guide.md", "docs/oauth-setup-guide.md", "docs/ADRs/**"] + adrs: [] + ddd: [] diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index bfb8283..0393991 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -1,2 +1,3 @@ {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"main","autonomy":"pr_ci","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T09:56:11-07:00"} {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: base changed from main to develop (user request 2026-07-31) so phase PRs land on a dedicated integration branch and a single develop->main PR gives a human review of the full body of work. .github/workflows/ci.yml pull_request.branches widened to [main, develop] in the same sitting.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T10:27:37-07:00"} +{"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: added phase 7 (Make -> Just task-runner migration, user request 2026-07-31). Ordered last via depends_on:[0..6] since it touches nearly every file the other phases touch AND rewrites .autopilot/profile.yml commands.* (make -> just), which every prior phase's gate still depends on while they run. Investigated first: just 1.57.0 already installed locally; no CI workflow, Dockerfile, or husky hook currently invokes make (only profile.yml commands.* and human-facing docs do), which is what makes a same-phase profile.yml update both necessary and sufficient.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]},{"id":7,"goal":"just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow.","definition_of_done":["cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile","cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile","cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list","cmd: just build","cmd: just test","cmd: just lint","cmd: just format-check","grep:absent: make build in README.md QUICKSTART.md CLAUDE.md","grep:absent: make test in README.md QUICKSTART.md CLAUDE.md","grep: just in README.md","grep: just in .autopilot/profile.yml","grep:absent: make in .autopilot/profile.yml","cmd: ls docs/ADRs | grep -qi just"]}],"at":"2026-07-31T10:53:31-07:00"} From 57206df10e43367ecdd1a7382d738565e6303841 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 11:13:15 -0700 Subject: [PATCH 07/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=200=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../discovered/ci-build-optimization.jsonl | 1 + .autopilot/runs/ci-build-optimization.jsonl | 1 + OPTIMIZATION_SPEC.md | 22 +++++++++---------- backend/Cargo.lock | 19 ---------------- backend/Cargo.toml | 13 +++++++---- 5 files changed, 22 insertions(+), 34 deletions(-) create mode 100644 .autopilot/discovered/ci-build-optimization.jsonl diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl new file mode 100644 index 0000000..ee99557 --- /dev/null +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -0,0 +1 @@ +{"id":"pl-mk-audit-swallow","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":0,"at":"2026-07-31T10:33:11-07:00"},"discovered_by":"gate","note":"backend/Makefile's audit target swallows its exit code (`cargo audit 2>/dev/null || echo \"cargo-audit not installed...\"`), so it can never fail even when real vulnerabilities are found โ€” same defect class as the frontend/Makefile test/audit swallow fixed during autopilot:detect. Out of scope for phase 0 (Cargo.toml dependency hygiene only); ran `cargo audit` directly to get an honest signal for this gate instead.","status":"open"} diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 0393991..1a2f9d0 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -1,3 +1,4 @@ {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"main","autonomy":"pr_ci","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T09:56:11-07:00"} {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: base changed from main to develop (user request 2026-07-31) so phase PRs land on a dedicated integration branch and a single develop->main PR gives a human review of the full body of work. .github/workflows/ci.yml pull_request.branches widened to [main, develop] in the same sitting.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T10:27:37-07:00"} +{"phase":0,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” tests passed without it)","test_integration (subsumed โ€” cargo test already runs all tests/*.rs binaries)","test_frontend (not applicable โ€” backend-only phase)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"88958a4","at":"2026-07-31T10:51:03-07:00","summary":"Removed unused ruvector-collections + encoding_rs deps from backend/Cargo.toml; added cargo-machete ignore for apalis/apalis-sql. 39/39 tests pass; machete/fmt/clippy/audit clean. Tier-3: reviewer subagent + manual pass, 0 issues. 1 parking-lot item: backend/Makefile audit target swallows exit code (pl-mk-audit-swallow)."} {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: added phase 7 (Make -> Just task-runner migration, user request 2026-07-31). Ordered last via depends_on:[0..6] since it touches nearly every file the other phases touch AND rewrites .autopilot/profile.yml commands.* (make -> just), which every prior phase's gate still depends on while they run. Investigated first: just 1.57.0 already installed locally; no CI workflow, Dockerfile, or husky hook currently invokes make (only profile.yml commands.* and human-facing docs do), which is what makes a same-phase profile.yml update both necessary and sufficient.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]},{"id":7,"goal":"just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow.","definition_of_done":["cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile","cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile","cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list","cmd: just build","cmd: just test","cmd: just lint","cmd: just format-check","grep:absent: make build in README.md QUICKSTART.md CLAUDE.md","grep:absent: make test in README.md QUICKSTART.md CLAUDE.md","grep: just in README.md","grep: just in .autopilot/profile.yml","grep:absent: make in .autopilot/profile.yml","cmd: ls docs/ADRs | grep -qi just"]}],"at":"2026-07-31T10:53:31-07:00"} diff --git a/OPTIMIZATION_SPEC.md b/OPTIMIZATION_SPEC.md index 2ad62b6..063383a 100644 --- a/OPTIMIZATION_SPEC.md +++ b/OPTIMIZATION_SPEC.md @@ -20,7 +20,7 @@ --- -### F-1 โ€” Docker build/push must target the `runtime` stage, not `development` +## F-1 โ€” Docker build/push must target the `runtime` stage, not `development` - **severity:** critical - **evidence:** .github/workflows/docker.yml:16-22, .github/workflows/docker.yml:30-36, .github/workflows/release.yml:169-175, backend/Dockerfile:21-25, frontend/Dockerfile:18 @@ -41,7 +41,7 @@ - **risk:** high - **depends_on:** [F-2] -### F-2 โ€” Make the runtime image buildable (ruvector context + submodules) +## F-2 โ€” Make the runtime image buildable (ruvector context + submodules) - **severity:** critical - **evidence:** backend/Cargo.toml:104-106 (path deps into `../ruvector/...`), backend/Dockerfile:1-7 (context assumes `./backend` only), .github/workflows/docker.yml:14 (checkout without `submodules: true`) @@ -62,7 +62,7 @@ - **risk:** high - **depends_on:** [] -### F-3 โ€” Move Lighthouse to a manual workflow; delete bundlewatch theater +## F-3 โ€” Move Lighthouse to a manual workflow; delete bundlewatch theater - **severity:** medium - **evidence:** .github/workflows/ci.yml:156 (stale `(ADR-005)` comment), .github/workflows/ci.yml:157-182 (lighthouse job, `continue-on-error: true`), .github/workflows/ci.yml:184-222 (bundlewatch job, oversized check only `echo`s a warning) @@ -93,7 +93,7 @@ - **risk:** low - **depends_on:** [] -### F-4 โ€” Dependency hygiene: remove 2 unused, document-ignore 2 planned +## F-4 โ€” Dependency hygiene: remove 2 unused, document-ignore 2 planned - **severity:** medium - **evidence:** backend/Cargo.toml:105 (`ruvector-collections`, 0 source refs), backend/Cargo.toml:13 & :111 (`encoding_rs` โ€” dead `builtin-llm` feature stub, 0 refs), backend/Cargo.toml:79-80 (`apalis` / `apalis-sql`), backend/src/content/jobs.rs:15,176 (apalis referenced in comments only) @@ -116,7 +116,7 @@ - **risk:** low - **depends_on:** [] -### F-5 โ€” Build the Rust crate once, share via nextest archive +## F-5 โ€” Build the Rust crate once, share via nextest archive - **severity:** medium - **evidence:** .github/workflows/ci.yml:47-52 (`cargo clippy --workspace --all-targets` โ€” full compile), .github/workflows/ci.yml:66-71 (`cargo test --workspace` + `cargo bench --no-run` โ€” second full compile) @@ -136,7 +136,7 @@ - **risk:** medium - **depends_on:** [] -### F-6 โ€” Set `CARGO_INCREMENTAL=0` and `CARGO_PROFILE_TEST_DEBUG=0` +## F-6 โ€” Set `CARGO_INCREMENTAL=0` and `CARGO_PROFILE_TEST_DEBUG=0` - **severity:** medium - **evidence:** .github/workflows/ci.yml:14-16 (env block sets neither) @@ -150,7 +150,7 @@ - **risk:** low - **depends_on:** [] -### F-7 โ€” Install `cargo-audit` from a prebuilt binary +## F-7 โ€” Install `cargo-audit` from a prebuilt binary - **severity:** medium - **evidence:** .github/workflows/ci.yml:82-83 (`cargo install --locked cargo-audit` โ€” compiles from source each run) @@ -163,7 +163,7 @@ - **risk:** low - **depends_on:** [] -### F-8 โ€” Single Rust-toolchain source of truth +## F-8 โ€” Single Rust-toolchain source of truth - **severity:** low - **evidence:** backend/rust-toolchain.toml (pins `1.96.0`), .github/workflows/ci.yml:27,41,62 & .github/workflows/release.yml:76 (`dtolnay/rust-toolchain@stable`) @@ -179,7 +179,7 @@ - **risk:** low - **depends_on:** [] -### F-9 โ€” Add a fast linker (lld) +## F-9 โ€” Add a fast linker (lld) - **severity:** low - **evidence:** no `fuse-ld=lld` present in workflows or `backend/.cargo/config.toml` @@ -193,7 +193,7 @@ - **risk:** low - **depends_on:** [] -### F-10 โ€” Parameterize heavy runners +## F-10 โ€” Parameterize heavy runners - **severity:** low - **evidence:** .github/workflows/ci.yml:22,36,55 (heavy jobs hardcode `runs-on: ubuntu-latest`) @@ -208,7 +208,7 @@ - **risk:** low - **depends_on:** [] -### F-11 โ€” `upload-artifact` with `if-no-files-found: ignore` +## F-11 โ€” `upload-artifact` with `if-no-files-found: ignore` - **severity:** low - **evidence:** .github/workflows/release.yml:128 (changelog `upload-artifact` without the guard) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c35d329..4fcdfb2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1738,7 +1738,6 @@ dependencies = [ "criterion", "dirs", "emailibrium", - "encoding_rs", "fastembed", "figment", "futures", @@ -1755,7 +1754,6 @@ dependencies = [ "regex", "reqwest", "rmcp", - "ruvector-collections", "ruvector-core", "ruvector-gnn", "schemars", @@ -4656,7 +4654,6 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", - "futures-channel", "futures-core", "futures-util", "h2", @@ -4895,21 +4892,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "ruvector-collections" -version = "2.2.2" -dependencies = [ - "bincode 2.0.1", - "chrono", - "dashmap", - "parking_lot", - "ruvector-core", - "serde", - "serde_json", - "thiserror 2.0.18", - "uuid", -] - [[package]] name = "ruvector-core" version = "2.2.2" @@ -4928,7 +4910,6 @@ dependencies = [ "rand_distr", "rayon", "redb", - "reqwest", "rkyv", "serde", "serde_json", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7451b3f..36cfe6c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -6,11 +6,17 @@ rust-version = "1.97" description = "Vector-native email intelligence platform" license = "MIT" +# apalis/apalis-sql are a planned background-job queue (ADR-006) referenced only in comments +# (backend/src/content/jobs.rs) until the queue is wired up โ€” cargo-machete would otherwise flag +# them as unused. Revisit this allowlist once jobs.rs actually depends on them. +[package.metadata.cargo-machete] +ignored = ["apalis", "apalis-sql"] + [features] default = ["vectors"] vectors = [] test-vectors = ["vectors"] -builtin-llm = ["dep:llama-cpp-4", "dep:hf-hub", "dep:encoding_rs"] +builtin-llm = ["dep:llama-cpp-4", "dep:hf-hub"] proptest = ["dep:proptest"] [lints.rust] @@ -75,7 +81,8 @@ ammonia = "4" scraper = "0.27" pdf-extract = "0.12" -# Background jobs (ADR-006) +# Background jobs (ADR-006) โ€” planned, not yet wired (see backend/src/content/jobs.rs). +# cargo-machete flags these as unused since nothing calls into them yet; kept intentionally. apalis = { version = "1.0.0-rc.9" } apalis-sql = "1.0.0-rc.9" @@ -105,13 +112,11 @@ schemars = "1.2" # Vector database (ADR-003: RuVector as primary) ruvector-core = { path = "../ruvector/crates/ruvector-core", default-features = false, features = ["hnsw", "storage", "simd", "parallel"] } -ruvector-collections = { path = "../ruvector/crates/ruvector-collections" } ruvector-gnn = { path = "../ruvector/crates/ruvector-gnn", default-features = false } # Built-in local LLM (ADR-021 addendum, opt-in via --features builtin-llm) llama-cpp-4 = { version = "0.3.1", optional = true } hf-hub = { version = "0.5", optional = true } -encoding_rs = { version = "0.8", optional = true } shellexpand = "3" dirs = "6" proptest = { version = "1", optional = true } From 1295d390900f4abf3f244abb3016a4f5f9170ff2 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 11:51:10 -0700 Subject: [PATCH 08/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=201=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .autopilot/discovered/ci-build-optimization.jsonl | 1 + .autopilot/runs/ci-build-optimization.jsonl | 1 + .github/workflows/ci.yml | 11 ++++++++++- .github/workflows/release.yml | 14 ++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index ee99557..6c5761d 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -1 +1,2 @@ {"id":"pl-mk-audit-swallow","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":0,"at":"2026-07-31T10:33:11-07:00"},"discovered_by":"gate","note":"backend/Makefile's audit target swallows its exit code (`cargo audit 2>/dev/null || echo \"cargo-audit not installed...\"`), so it can never fail even when real vulnerabilities are found โ€” same defect class as the frontend/Makefile test/audit swallow fixed during autopilot:detect. Out of scope for phase 0 (Cargo.toml dependency hygiene only); ran `cargo audit` directly to get an honest signal for this gate instead.","status":"open"} +{"id":"pl-test-debuginfo-tradeoff","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":1,"at":"2026-07-31T11:13:15-07:00"},"discovered_by":"reviewer","note":"CARGO_PROFILE_TEST_DEBUG=0 (added this phase per OPTIMIZATION_SPEC F-6) strips debuginfo from the test profile, so RUST_BACKTRACE=1 output in CI loses every `at file:line` frame โ€” only symbol names remain (verified locally on cargo 1.96 by the Tier-3 reviewer). The panic-location line still survives via #[track_caller], so assert failures still point at the right line, but deep backtraces are harder to read. Shipped as specified because the curated spec explicitly asked for it and the target/-size + cache-speed win is real; revert this one env var if a CI test failure ever proves hard to diagnose.","status":"open"} diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 1a2f9d0..a45a3ab 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -2,3 +2,4 @@ {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: base changed from main to develop (user request 2026-07-31) so phase PRs land on a dedicated integration branch and a single develop->main PR gives a human review of the full body of work. .github/workflows/ci.yml pull_request.branches widened to [main, develop] in the same sitting.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]}],"at":"2026-07-31T10:27:37-07:00"} {"phase":0,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” tests passed without it)","test_integration (subsumed โ€” cargo test already runs all tests/*.rs binaries)","test_frontend (not applicable โ€” backend-only phase)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"88958a4","at":"2026-07-31T10:51:03-07:00","summary":"Removed unused ruvector-collections + encoding_rs deps from backend/Cargo.toml; added cargo-machete ignore for apalis/apalis-sql. 39/39 tests pass; machete/fmt/clippy/audit clean. Tier-3: reviewer subagent + manual pass, 0 issues. 1 parking-lot item: backend/Makefile audit target swallows exit code (pl-mk-audit-swallow)."} {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: added phase 7 (Make -> Just task-runner migration, user request 2026-07-31). Ordered last via depends_on:[0..6] since it touches nearly every file the other phases touch AND rewrites .autopilot/profile.yml commands.* (make -> just), which every prior phase's gate still depends on while they run. Investigated first: just 1.57.0 already installed locally; no CI workflow, Dockerfile, or husky hook currently invokes make (only profile.yml commands.* and human-facing docs do), which is what makes a same-phase profile.yml update both necessary and sufficient.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]},{"id":7,"goal":"just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow.","definition_of_done":["cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile","cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile","cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list","cmd: just build","cmd: just test","cmd: just lint","cmd: just format-check","grep:absent: make build in README.md QUICKSTART.md CLAUDE.md","grep:absent: make test in README.md QUICKSTART.md CLAUDE.md","grep: just in README.md","grep: just in .autopilot/profile.yml","grep:absent: make in .autopilot/profile.yml","cmd: ls docs/ADRs | grep -qi just"]}],"at":"2026-07-31T10:53:31-07:00"} +{"phase":1,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority: diff is .github/workflows/*.yml only and cannot affect compilation)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"ca35144","at":"2026-07-31T11:35:15-07:00","summary":"CI env (CARGO_INCREMENTAL/PROFILE_TEST_DEBUG=0), cargo-audit via taiki-e/install-action, and release-notes fallback + if-no-files-found. DoD 5/5; yamllint+parse green. Tier-3 caught that F-11 as-specified would have moved a failure downstream โ€” fixed properly. 1 parking-lot item (pl-test-debuginfo-tradeoff)."} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8353bbd..6d1abfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,11 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + # Incremental compilation only pays off across repeated local builds; in CI + # every job starts from a restored cache, so it just bloats target/ and slows + # the cache save/restore steps. Same for test-profile debuginfo. + CARGO_INCREMENTAL: 0 + CARGO_PROFILE_TEST_DEBUG: 0 jobs: # Backend jobs (separate for clear failure attribution) @@ -79,8 +84,12 @@ jobs: # availability we cannot control, and its HTTP 500s from GitHub cause # spurious job failures unrelated to our security posture. - uses: actions/checkout@v7 + # Prebuilt binary. This previously used a from-source `cargo install`, + # which recompiled the tool on every run. - name: Install cargo-audit - run: cargo install --locked cargo-audit + uses: taiki-e/install-action@v2 + with: + tool: cargo-audit # Fetch the RUSTSEC advisory DB with retry so a transient GitHub 500 # does not fail the job. The audit step uses --no-fetch after this. - name: Fetch advisory database diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7512048..a6aaeee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -125,10 +125,24 @@ jobs: with: config: cliff.toml args: --latest --output RELEASE_NOTES.md + # create-release consumes RELEASE_NOTES.md unconditionally (download-artifact, + # body_path, and the CHANGELOG.md update all assume it). So `if-no-files-found: + # ignore` alone would not make a missing file harmless โ€” it would just move the + # failure downstream. Guarantee the file exists, then the guard is meaningful. + - name: Ensure release notes exist + env: + TAG: ${{ github.ref_name }} + run: | + if [ ! -s RELEASE_NOTES.md ]; then + echo "No changelog entries were generated for $TAG." > RELEASE_NOTES.md + fi - uses: actions/upload-artifact@v7 with: name: changelog path: RELEASE_NOTES.md + # Belt-and-braces: the step above guarantees the file, so this only + # matters if that step is ever removed. + if-no-files-found: ignore # โ”€โ”€ Job 4: Build Docker images โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ docker-images: From d1cd7c55e0742ebdd895f4eb6d5b8749508fc45a Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 12:13:54 -0700 Subject: [PATCH 09/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=202=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../discovered/ci-build-optimization.jsonl | 2 + .autopilot/runs/ci-build-optimization.jsonl | 1 + .github/workflows/ci.yml | 68 ------------------- .github/workflows/lighthouse.yml | 43 ++++++++++++ 4 files changed, 46 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/lighthouse.yml diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index 6c5761d..d5559ce 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -1,2 +1,4 @@ {"id":"pl-mk-audit-swallow","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":0,"at":"2026-07-31T10:33:11-07:00"},"discovered_by":"gate","note":"backend/Makefile's audit target swallows its exit code (`cargo audit 2>/dev/null || echo \"cargo-audit not installed...\"`), so it can never fail even when real vulnerabilities are found โ€” same defect class as the frontend/Makefile test/audit swallow fixed during autopilot:detect. Out of scope for phase 0 (Cargo.toml dependency hygiene only); ran `cargo audit` directly to get an honest signal for this gate instead.","status":"open"} {"id":"pl-test-debuginfo-tradeoff","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":1,"at":"2026-07-31T11:13:15-07:00"},"discovered_by":"reviewer","note":"CARGO_PROFILE_TEST_DEBUG=0 (added this phase per OPTIMIZATION_SPEC F-6) strips debuginfo from the test profile, so RUST_BACKTRACE=1 output in CI loses every `at file:line` frame โ€” only symbol names remain (verified locally on cargo 1.96 by the Tier-3 reviewer). The panic-location line still survives via #[track_caller], so assert failures still point at the right line, but deep backtraces are harder to read. Shipped as specified because the curated spec explicitly asked for it and the target/-size + cache-speed win is real; revert this one env var if a CI test failure ever proves hard to diagnose.","status":"open"} +{"id":"pl-lighthouse-warn-only","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"OPTIMIZATION_SPEC F-3 wanted the relocated Lighthouse workflow to surface \"real pass/fail instead of theater\", but dropping continue-on-error is cosmetic: all 8 assertions in frontend/lighthouserc.json are severity \"warn\" and lhci autorun exits 0 when only warn-level assertions trip. To actually gate, flip those to \"error\". Left to a human because it sets the project quality bar and current Lighthouse scores are unknown โ€” an unconditional flip could make every manual run fail. The workflow comment now states this honestly rather than claiming a gate that does not exist.","status":"open"} +{"id":"pl-ci-job-doc-drift","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"Docs drifted from CI reality after removing the lighthouse/bundlewatch jobs: docs/ADRs/ADR-005-tauri-to-web-spa-migration.md:65 still claims a \"Bundlewatch CI gate\" (which never had teeth โ€” it only echoed a warning), and docs/plan/march-2026-audit.v2.md:301 still counts 11 CI jobs (now 9). Out of scope for phase 2 (touches only the workflow files + lighthouserc.json); ADR/plan docs are historical records, so a human should decide whether to amend or leave them as point-in-time.","status":"open"} diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index a45a3ab..5faebef 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -3,3 +3,4 @@ {"phase":0,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” tests passed without it)","test_integration (subsumed โ€” cargo test already runs all tests/*.rs binaries)","test_frontend (not applicable โ€” backend-only phase)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"88958a4","at":"2026-07-31T10:51:03-07:00","summary":"Removed unused ruvector-collections + encoding_rs deps from backend/Cargo.toml; added cargo-machete ignore for apalis/apalis-sql. 39/39 tests pass; machete/fmt/clippy/audit clean. Tier-3: reviewer subagent + manual pass, 0 issues. 1 parking-lot item: backend/Makefile audit target swallows exit code (pl-mk-audit-swallow)."} {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: added phase 7 (Make -> Just task-runner migration, user request 2026-07-31). Ordered last via depends_on:[0..6] since it touches nearly every file the other phases touch AND rewrites .autopilot/profile.yml commands.* (make -> just), which every prior phase's gate still depends on while they run. Investigated first: just 1.57.0 already installed locally; no CI workflow, Dockerfile, or husky hook currently invokes make (only profile.yml commands.* and human-facing docs do), which is what makes a same-phase profile.yml update both necessary and sufficient.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]},{"id":7,"goal":"just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow.","definition_of_done":["cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile","cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile","cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list","cmd: just build","cmd: just test","cmd: just lint","cmd: just format-check","grep:absent: make build in README.md QUICKSTART.md CLAUDE.md","grep:absent: make test in README.md QUICKSTART.md CLAUDE.md","grep: just in README.md","grep: just in .autopilot/profile.yml","grep:absent: make in .autopilot/profile.yml","cmd: ls docs/ADRs | grep -qi just"]}],"at":"2026-07-31T10:53:31-07:00"} {"phase":1,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority: diff is .github/workflows/*.yml only and cannot affect compilation)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"ca35144","at":"2026-07-31T11:35:15-07:00","summary":"CI env (CARGO_INCREMENTAL/PROFILE_TEST_DEBUG=0), cargo-audit via taiki-e/install-action, and release-notes fallback + if-no-files-found. DoD 5/5; yamllint+parse green. Tier-3 caught that F-11 as-specified would have moved a failure downstream โ€” fixed properly. 1 parking-lot item (pl-test-debuginfo-tradeoff)."} +{"phase":2,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” diff is workflow YAML only)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"8efc74b","at":"2026-07-31T11:57:30-07:00","summary":"Deleted toothless lighthouse+bundlewatch PR jobs (11->9 CI jobs, -2 redundant frontend builds/PR); added manual-dispatch lighthouse.yml; killed stale ADR-005 comment. DoD 8/8. Tier-3: relocated Lighthouse still warn-only so cannot gate โ€” recorded pl-lighthouse-warn-only rather than silently changing the quality bar; also pl-ci-job-doc-drift."} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d1abfc..0a53af2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,74 +162,6 @@ jobs: working-directory: frontend run: pnpm turbo build - # Performance gates (ADR-005) - lighthouse: - name: Lighthouse CI - runs-on: ubuntu-latest - needs: frontend-quality - steps: - - uses: actions/checkout@v7 - - uses: pnpm/action-setup@v6 - with: - version: 11 - - uses: actions/setup-node@v7 - with: - node-version: 26 - cache: pnpm - cache-dependency-path: frontend/pnpm-lock.yaml - - name: Install - working-directory: frontend - run: pnpm install --frozen-lockfile - - name: Build - working-directory: frontend - run: pnpm turbo build - - name: Lighthouse CI - uses: treosh/lighthouse-ci-action@v12 - with: - configPath: frontend/lighthouserc.json - uploadArtifacts: true - continue-on-error: true - - bundlewatch: - name: Bundle Size Check - runs-on: ubuntu-latest - needs: frontend-quality - steps: - - uses: actions/checkout@v7 - - uses: pnpm/action-setup@v6 - with: - version: 11 - - uses: actions/setup-node@v7 - with: - node-version: 26 - cache: pnpm - cache-dependency-path: frontend/pnpm-lock.yaml - - name: Install - working-directory: frontend - run: pnpm install --frozen-lockfile - - name: Build - working-directory: frontend - run: pnpm turbo build - - name: Check bundle size - working-directory: frontend - run: | - echo "=== Bundle Size Report ===" - du -sh apps/web/dist/ 2>/dev/null || echo "No dist directory" - echo "" - echo "=== JS Bundle Sizes ===" - find apps/web/dist -name "*.js" -exec du -h {} + 2>/dev/null | sort -rh | head -20 - echo "" - echo "=== CSS Bundle Sizes ===" - find apps/web/dist -name "*.css" -exec du -h {} + 2>/dev/null | sort -rh | head -10 - echo "" - # Fail if any single JS chunk exceeds 500KB - OVERSIZED=$(find apps/web/dist -name "*.js" -size +500k 2>/dev/null) - if [ -n "$OVERSIZED" ]; then - echo "WARNING: The following JS bundles exceed 500KB:" - echo "$OVERSIZED" | xargs du -h - echo "Consider code-splitting or lazy loading to reduce bundle size." - fi - # Documentation validation validate-markdown: name: Validate Markdown diff --git a/.github/workflows/lighthouse.yml b/.github/workflows/lighthouse.yml new file mode 100644 index 0000000..8837987 --- /dev/null +++ b/.github/workflows/lighthouse.yml @@ -0,0 +1,43 @@ +name: Lighthouse + +# Manual only. This used to run on every PR in a non-blocking mode, so it could +# never fail anything โ€” it just spent a full `pnpm install` + `pnpm turbo build` +# per PR to produce a report nobody was gated on. Now it runs on demand. +# +# NOTE: this job still cannot fail on a budget miss. Every assertion in +# frontend/lighthouserc.json is severity "warn", and lhci exits 0 when only +# warn-level assertions trip. Flip those to "error" to make a manual run +# actually gate on the budgets โ€” a deliberate quality-bar call, so it is left +# to a human rather than changed here. +on: + workflow_dispatch: + +concurrency: + group: lighthouse-${{ github.ref }} + cancel-in-progress: true + +jobs: + lighthouse: + name: Lighthouse CI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + with: + version: 11 + - uses: actions/setup-node@v7 + with: + node-version: 26 + cache: pnpm + cache-dependency-path: frontend/pnpm-lock.yaml + - name: Install + working-directory: frontend + run: pnpm install --frozen-lockfile + - name: Build + working-directory: frontend + run: pnpm turbo build + - name: Lighthouse CI + uses: treosh/lighthouse-ci-action@v12 + with: + configPath: frontend/lighthouserc.json + uploadArtifacts: true From 8259e8c9cbd4b7f4b03626e4e29dcb8196298224 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 12:14:55 -0700 Subject: [PATCH 10/24] chore(autopilot:ci-build-optimization): record cold-cache finding for phase 4 baseline --- .autopilot/discovered/ci-build-optimization.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index d5559ce..5f5dc90 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -2,3 +2,4 @@ {"id":"pl-test-debuginfo-tradeoff","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":1,"at":"2026-07-31T11:13:15-07:00"},"discovered_by":"reviewer","note":"CARGO_PROFILE_TEST_DEBUG=0 (added this phase per OPTIMIZATION_SPEC F-6) strips debuginfo from the test profile, so RUST_BACKTRACE=1 output in CI loses every `at file:line` frame โ€” only symbol names remain (verified locally on cargo 1.96 by the Tier-3 reviewer). The panic-location line still survives via #[track_caller], so assert failures still point at the right line, but deep backtraces are harder to read. Shipped as specified because the curated spec explicitly asked for it and the target/-size + cache-speed win is real; revert this one env var if a CI test failure ever proves hard to diagnose.","status":"open"} {"id":"pl-lighthouse-warn-only","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"OPTIMIZATION_SPEC F-3 wanted the relocated Lighthouse workflow to surface \"real pass/fail instead of theater\", but dropping continue-on-error is cosmetic: all 8 assertions in frontend/lighthouserc.json are severity \"warn\" and lhci autorun exits 0 when only warn-level assertions trip. To actually gate, flip those to \"error\". Left to a human because it sets the project quality bar and current Lighthouse scores are unknown โ€” an unconditional flip could make every manual run fail. The workflow comment now states this honestly rather than claiming a gate that does not exist.","status":"open"} {"id":"pl-ci-job-doc-drift","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"Docs drifted from CI reality after removing the lighthouse/bundlewatch jobs: docs/ADRs/ADR-005-tauri-to-web-spa-migration.md:65 still claims a \"Bundlewatch CI gate\" (which never had teeth โ€” it only echoed a warning), and docs/plan/march-2026-audit.v2.md:301 still counts 11 CI jobs (now 9). Out of scope for phase 2 (touches only the workflow files + lighthouserc.json); ADR/plan docs are historical records, so a human should decide whether to amend or leave them as point-in-time.","status":"open"} +{"id":"pl-rust-cache-cold-after-env-change","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T12:13:54-07:00"},"discovered_by":"gate","note":"Rust Tests wall-clock trended 5m58s (phase 0) -> 10m48s (phase 1) -> 13m50s (phase 2) vs the spec baseline of 311s warm. Root cause found in the phase-2 job log: Swatinem/rust-cache reports \"No cache found.\" It runs with add-rust-environment-hash-key:true, so phase 1 adding CARGO_INCREMENTAL/CARGO_PROFILE_TEST_DEBUG to the workflow env changed the cache key; GitHub also scopes PR-branch caches to their own branch. The develop-scoped cache for the new key only populates once a push-triggered run on develop completes. Expected to self-resolve โ€” but phase 4 (F-5 nextest archive) must NOT compare against these cold numbers when it re-measures warm time-to-green.","status":"open"} From efffd0aa390fbca500b8b4f9ba747137f6680a99 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 12:46:58 -0700 Subject: [PATCH 11/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=203=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../discovered/ci-build-optimization.jsonl | 1 + .autopilot/pipeline.yml | 25 +++++++++--- .autopilot/runs/ci-build-optimization.jsonl | 1 + .github/workflows/ci.yml | 39 +++++++++++++++++-- .github/workflows/release.yml | 17 +++++++- 5 files changed, 72 insertions(+), 11 deletions(-) diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index 5f5dc90..23f5349 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -3,3 +3,4 @@ {"id":"pl-lighthouse-warn-only","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"OPTIMIZATION_SPEC F-3 wanted the relocated Lighthouse workflow to surface \"real pass/fail instead of theater\", but dropping continue-on-error is cosmetic: all 8 assertions in frontend/lighthouserc.json are severity \"warn\" and lhci autorun exits 0 when only warn-level assertions trip. To actually gate, flip those to \"error\". Left to a human because it sets the project quality bar and current Lighthouse scores are unknown โ€” an unconditional flip could make every manual run fail. The workflow comment now states this honestly rather than claiming a gate that does not exist.","status":"open"} {"id":"pl-ci-job-doc-drift","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"Docs drifted from CI reality after removing the lighthouse/bundlewatch jobs: docs/ADRs/ADR-005-tauri-to-web-spa-migration.md:65 still claims a \"Bundlewatch CI gate\" (which never had teeth โ€” it only echoed a warning), and docs/plan/march-2026-audit.v2.md:301 still counts 11 CI jobs (now 9). Out of scope for phase 2 (touches only the workflow files + lighthouserc.json); ADR/plan docs are historical records, so a human should decide whether to amend or leave them as point-in-time.","status":"open"} {"id":"pl-rust-cache-cold-after-env-change","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T12:13:54-07:00"},"discovered_by":"gate","note":"Rust Tests wall-clock trended 5m58s (phase 0) -> 10m48s (phase 1) -> 13m50s (phase 2) vs the spec baseline of 311s warm. Root cause found in the phase-2 job log: Swatinem/rust-cache reports \"No cache found.\" It runs with add-rust-environment-hash-key:true, so phase 1 adding CARGO_INCREMENTAL/CARGO_PROFILE_TEST_DEBUG to the workflow env changed the cache key; GitHub also scopes PR-branch caches to their own branch. The develop-scoped cache for the new key only populates once a push-triggered run on develop completes. Expected to self-resolve โ€” but phase 4 (F-5 nextest archive) must NOT compare against these cold numbers when it re-measures warm time-to-green.","status":"open"} +{"id":"pl-dockerfile-rust-version-third-source","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":3,"at":"2026-07-31T12:14:55-07:00"},"discovered_by":"reviewer","note":"backend/Dockerfile hardcodes `rust:1.97-slim` (lines ~2 and ~22), so the Rust version still has a THIRD source of truth alongside backend/rust-toolchain.toml and backend/Cargo.toml rust-version. Phase 3 made the CI workflows read the pin from rust-toolchain.toml, but did not touch the Dockerfile โ€” that file belongs to phase 5 (Docker build context), which is already rewriting its COPY/context lines. Worth folding the version single-sourcing into phase 5 (e.g. an ARG defaulted from the toolchain file, or at minimum a comment cross-referencing it) so a future MSRV bump does not leave the image on a stale compiler.","status":"open"} diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml index bef018c..3a7023a 100644 --- a/.autopilot/pipeline.yml +++ b/.autopilot/pipeline.yml @@ -121,17 +121,32 @@ phases: goal: "Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners." track: "" deliverables: - - "Make backend/rust-toolchain.toml (MSRV 1.96.0) the single source of truth: pin ci.yml/release.yml's `dtolnay/rust-toolchain` step to that version instead of bare `@stable`." - - "Install `lld` in the Rust CI jobs and set `-Clink-arg=-fuse-ld=lld` (via RUSTFLAGS or backend/.cargo/config.toml) to cut incremental link time." + - "Make backend/rust-toolchain.toml the single source of truth: ci.yml/release.yml must resolve the Rust version FROM that file instead of installing bare `@stable`." + - "Install `lld` and set `-Clink-arg=-fuse-ld=lld` in the CI job(s) that actually link, to cut link time." - "Point heavy jobs (rust-test, Docker builds) at `runs-on: \\${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }}` so a runner upgrade is opt-in and safe by default." definition_of_done: - "grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**" - - "grep: '1.96' in .github/workflows/**" + - "grep: rust-toolchain.toml in .github/workflows/**" + - "cmd: test \"$(sed -n 's/^channel *= *\"\\(.*\\)\"/\\1/p' backend/rust-toolchain.toml)\" = \"1.97.0\"" - "grep: rust-toolchain.toml in backend" - - "grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml" + - "grep: fuse-ld=lld in .github/workflows/**" - "grep: lld in .github/workflows/**" - "grep: vars.HEAVY_RUNNER in .github/workflows/**" - conventions: "" + conventions: | + DoD CORRECTED 2026-07-31 during phase-3 execution. The original line + `grep: '1.96' in .github/workflows/**` came from OPTIMIZATION_SPEC F-8's claim that the + MSRV is 1.96.0. That is stale: backend/rust-toolchain.toml pins 1.97.0 and + backend/Cargo.toml sets rust-version = "1.97". Hardcoding 1.96 into CI would BREAK the + build โ€” phase 0 hit exactly that error locally ("rustc 1.96.0 is not supported by the + following packages: emailibrium@0.2.0 requires rustc 1.97"). + The replacement DoD is strictly STRONGER, not weaker: instead of asserting a hardcoded + version string appears in the workflows (which would itself be a second source of truth + and could drift again), it asserts the workflows REFERENCE rust-toolchain.toml, and + separately pins the expected value in one place. This follows the deliverable's own + sanctioned alternative: "or have it read the toolchain file". + lld is deliberately NOT added to backend/.cargo/config.toml: that would make every local + build require lld to be installed. It is scoped to the rust-test job, the only job that + links (fmt/clippy emit metadata and never link). depends_on: [] touches: [".github/workflows/ci.yml", ".github/workflows/release.yml", "backend/.cargo/config.toml"] adrs: [] diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 5faebef..00fde02 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -4,3 +4,4 @@ {"type":"plan","feature_id":"ci-build-optimization","goal":"Fix the broken Docker publish (wrong build stage + missing submodule context) and optimize the CI/build/dependency pipeline per the rust-optimizer audit in OPTIMIZATION_SPEC.md.","trunk":"main","base":"develop","autonomy":"pr_ci","note":"re-plan: added phase 7 (Make -> Just task-runner migration, user request 2026-07-31). Ordered last via depends_on:[0..6] since it touches nearly every file the other phases touch AND rewrites .autopilot/profile.yml commands.* (make -> just), which every prior phase's gate still depends on while they run. Investigated first: just 1.57.0 already installed locally; no CI workflow, Dockerfile, or husky hook currently invokes make (only profile.yml commands.* and human-facing docs do), which is what makes a same-phase profile.yml update both necessary and sufficient.","phases":[{"id":0,"goal":"backend/Cargo.toml carries no unused dependencies and documents the ones it deliberately keeps.","definition_of_done":["cmd: cargo machete backend","cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm","cmd: cargo test --manifest-path backend/Cargo.toml","grep:absent: ruvector-collections in backend/Cargo.toml","grep:absent: dep:encoding_rs in backend/Cargo.toml","grep: cargo-machete in backend/Cargo.toml"]},{"id":1,"goal":"CI workflow env/tooling carries three small, independent, low-risk fixes: leaner cache, prebuilt cargo-audit, no spurious release failures.","definition_of_done":["grep: CARGO_INCREMENTAL: ?0 in .github/workflows/ci.yml","grep: CARGO_PROFILE_TEST_DEBUG: ?0 in .github/workflows/ci.yml","grep: taiki-e/install-action in .github/workflows/ci.yml","grep:absent: cargo install --locked cargo-audit in .github/workflows/**","grep: if-no-files-found: ignore in .github/workflows/release.yml"]},{"id":2,"goal":"Per-PR CI stops running unenforced performance-gate theater; Lighthouse becomes a real, on-demand, non-theater check.","definition_of_done":["grep:absent: continue-on-error in .github/workflows/ci.yml","grep:absent: bundlewatch in .github/workflows/ci.yml","grep:absent: treosh/lighthouse-ci-action in .github/workflows/ci.yml","grep:absent: ADR-005 in .github/workflows/ci.yml","grep: workflow_dispatch in .github/workflows/lighthouse.yml","grep: treosh/lighthouse-ci-action in .github/workflows/lighthouse.yml","grep:absent: continue-on-error in .github/workflows/lighthouse.yml","cmd: test -f frontend/lighthouserc.json"]},{"id":3,"goal":"Rust toolchain version and link step are single-sourced and faster, with an opt-in path to bigger/native runners.","definition_of_done":["grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/**","grep: 1.96 in .github/workflows/**","grep: rust-toolchain.toml in backend","grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml","grep: lld in .github/workflows/**","grep: vars.HEAVY_RUNNER in .github/workflows/**"]},{"id":4,"goal":"The Rust crate compiles once per CI run; clippy and test no longer each pay a full recompile of the crate plus the heavy vendored ruvector path crates.","definition_of_done":["grep: nextest archive in .github/workflows/**","grep: archive-file in .github/workflows/**","cmd: cargo test --manifest-path backend/Cargo.toml"]},{"id":5,"goal":"The Docker build context actually contains everything the runtime target needs to compile - the ruvector submodule is no longer silently missing.","definition_of_done":["grep: submodules: true in .github/workflows/docker.yml","grep: submodules: true in .github/workflows/release.yml","grep: ruvector in backend/Dockerfile","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check ."]},{"id":6,"goal":"The published images are the runnable runtime stage (with the compiled binary, migrations, and entrypoint) instead of the unrunnable development dev-tooling stage.","definition_of_done":["grep: target: runtime in .github/workflows/docker.yml","grep: target: runtime in .github/workflows/release.yml","cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help","cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend"]},{"id":7,"goal":"just replaces make as the project's task runner, with zero loss of functionality and no dangling references to the old make-based workflow.","definition_of_done":["cmd: test -f justfile && test -f backend/justfile && test -f frontend/justfile","cmd: test ! -f Makefile && test ! -f backend/Makefile && test ! -f frontend/Makefile","cmd: just --list && just --justfile backend/justfile --list && just --justfile frontend/justfile --list","cmd: just build","cmd: just test","cmd: just lint","cmd: just format-check","grep:absent: make build in README.md QUICKSTART.md CLAUDE.md","grep:absent: make test in README.md QUICKSTART.md CLAUDE.md","grep: just in README.md","grep: just in .autopilot/profile.yml","grep:absent: make in .autopilot/profile.yml","cmd: ls docs/ADRs | grep -qi just"]}],"at":"2026-07-31T10:53:31-07:00"} {"phase":1,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority: diff is .github/workflows/*.yml only and cannot affect compilation)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"ca35144","at":"2026-07-31T11:35:15-07:00","summary":"CI env (CARGO_INCREMENTAL/PROFILE_TEST_DEBUG=0), cargo-audit via taiki-e/install-action, and release-notes fallback + if-no-files-found. DoD 5/5; yamllint+parse green. Tier-3 caught that F-11 as-specified would have moved a failure downstream โ€” fixed properly. 1 parking-lot item (pl-test-debuginfo-tradeoff)."} {"phase":2,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” diff is workflow YAML only)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"8efc74b","at":"2026-07-31T11:57:30-07:00","summary":"Deleted toothless lighthouse+bundlewatch PR jobs (11->9 CI jobs, -2 redundant frontend builds/PR); added manual-dispatch lighthouse.yml; killed stale ADR-005 comment. DoD 8/8. Tier-3: relocated Lighthouse still warn-only so cannot gate โ€” recorded pl-lighthouse-warn-only rather than silently changing the quality bar; also pl-ci-job-doc-drift."} +{"phase":3,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” workflow-YAML-only diff; and this phase changes HOW CI installs the toolchain, so CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"9f24209","at":"2026-07-31T12:29:51-07:00","summary":"Workflows now read the Rust pin from rust-toolchain.toml (no more @stable drift); lld+RUSTFLAGS scoped to rust-test; HEAVY_RUNNER opt-in on rust-test+docker-images. Corrected a stale DoD demanding 1.96 (real pin 1.97.0 โ€” would have broken the build). Tier-3 caught that HEAVY_RUNNER could silently republish :latest as arm64-only; pinned platforms: linux/amd64. DoD 7/7."} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a53af2..834ef4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,16 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - - uses: dtolnay/rust-toolchain@stable + # backend/rust-toolchain.toml is the single source of truth for the Rust + # version. Installing bare @stable here drifts from it, and rustup would + # silently install the pinned toolchain a second time on first cargo use + # inside backend/ โ€” so read the pin and install exactly it, once. + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust.outputs.channel }} components: rustfmt - name: Check formatting working-directory: backend @@ -43,8 +51,12 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - - uses: dtolnay/rust-toolchain@stable + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust.outputs.channel }} components: clippy - uses: Swatinem/rust-cache@v2 with: @@ -58,13 +70,32 @@ jobs: rust-test: name: Rust Tests - runs-on: ubuntu-latest + # Opt-in bigger/native runner. Unset (the default) resolves to ubuntu-latest, + # so this is safe everywhere; setting the repo variable to a label that does + # not exist would queue forever, which is why it is parameterized rather than + # hardcoded to a non-default runner. + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} needs: [rust-format, rust-clippy] + # This is where the linking cost actually is: the test binaries and benches. + # (clippy does link build scripts and proc-macros, but not the test binaries, + # so it has little to gain.) RUSTFLAGS is scoped to this job so a missing + # linker can never break an unrelated one. + env: + RUSTFLAGS: -Clink-arg=-fuse-ld=lld steps: - uses: actions/checkout@v7 with: submodules: true - - uses: dtolnay/rust-toolchain@stable + # Assumes an apt-based Ubuntu runner โ€” true for ubuntu-latest and for the + # ubuntu-*-arm images HEAVY_RUNNER is expected to point at. + - name: Install lld + run: sudo apt-get update && sudo apt-get install -y lld + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust.outputs.channel }} - uses: Swatinem/rust-cache@v2 with: workspaces: backend diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6aaeee..6394530 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,8 +73,14 @@ jobs: submodules: true # Backend - - uses: dtolnay/rust-toolchain@stable + # backend/rust-toolchain.toml is the single source of truth for the Rust + # version โ€” read the pin rather than installing bare @stable, which drifts. + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master with: + toolchain: ${{ steps.rust.outputs.channel }} components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 with: @@ -148,7 +154,8 @@ jobs: docker-images: name: Docker ${{ matrix.app }} needs: [validate-tag, ci-gate] - runs-on: ubuntu-latest + # Opt-in bigger/native runner; unset resolves to ubuntu-latest. + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} strategy: matrix: include: @@ -185,6 +192,12 @@ jobs: context: ${{ matrix.context }} push: true tags: ${{ steps.tags.outputs.tags }} + # Pin the published architecture. Without this, buildx targets the + # RUNNER's arch โ€” so pointing HEAVY_RUNNER at an arm64 runner to speed + # up the Rust jobs would silently republish :latest as arm64-only. + # These images have always been single-arch amd64; make that explicit + # rather than an accident of runner selection. + platforms: linux/amd64 cache-from: type=gha,scope=${{ matrix.app }} cache-to: type=gha,mode=max,scope=${{ matrix.app }} From 17472c6c431da3338951adda6898e1349074d868 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 13:21:20 -0700 Subject: [PATCH 12/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=204=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../court/ci-build-optimization/phase-4.md | 119 +++++++++++++++++ .autopilot/runs/ci-build-optimization.jsonl | 2 + .github/workflows/ci.yml | 124 ++++++++++++++++-- .gitignore | 2 + backend/.config/nextest.toml | 32 +++++ 5 files changed, 266 insertions(+), 13 deletions(-) create mode 100644 .autopilot/court/ci-build-optimization/phase-4.md create mode 100644 backend/.config/nextest.toml diff --git a/.autopilot/court/ci-build-optimization/phase-4.md b/.autopilot/court/ci-build-optimization/phase-4.md new file mode 100644 index 0000000..ff0efac --- /dev/null +++ b/.autopilot/court/ci-build-optimization/phase-4.md @@ -0,0 +1,119 @@ +# qe-court record โ€” ci-build-optimization phase 4 + +**Delivery:** restructure Rust CI to build test binaries once into a `cargo-nextest` +archive consumed by `rust-test` (OPTIMIZATION_SPEC F-5). +**Date:** 2026-07-31 ยท **Phase:** 4 (in `risk_phases`) ยท **Court mode:** `auto` + +--- + +## โš ๏ธ Panel could NOT be seated as configured โ€” read this before trusting the verdict + +`.claude/skills/qe-court/config.json` routes 4 of 8 roles to **Cognitum** +(`prosecutor.devils-advocate`, `prosecutor.sherlock`, `prosecutor.security-scanner`, +and โ€” critically โ€” **`jury`**). Cognitum is **not configured in this environment**: +no `COGNITUM*` env var, and `aqe llm-router config` returns nothing. + +The skill also mandates calling `validateCourtConfig()` from +`src/skills/qe-court/referee.ts` before seating a panel. **That file is not present** +in this project's skill install (`.claude/skills/qe-court/` contains only +`config.json`, `evals`, `schemas`, `scripts`, `SKILL.md`), so the machine-checked +invariant validation could not be run at all. + +Per the skill's own instruction โ€” *"do not proceed with a degraded panel and do not +silently re-route around it"* โ€” this is recorded as a **partial court**, not a full +one. What actually ran is stated plainly below. **This verdict carries less weight +than a fully-seated court** and should be read as one strong cross-vendor lens plus +the Tier-3 floor, not as the ADR-124 protocol. + +| Role | Configured | Actually run | +|---|---|---| +| Defense | claude-code | โ€” (not run) | +| Prosecutor ยท devils-advocate | cognitum-mid | โŒ vendor unavailable | +| Prosecutor ยท brutal-honesty | claude-code | โœ… ran as the Tier-3 reviewer subagent | +| Prosecutor ยท sherlock | cognitum-high | โŒ vendor unavailable | +| Prosecutor ยท security-scanner | cognitum-mid | โŒ vendor unavailable | +| Prosecutor ยท mutation | ollama | โŒ not run | +| Prosecutor ยท codex-review | codex | โœ… ran (`codex exec`, GPT โ€” true cross-vendor) | +| Jury | cognitum-high | โŒ **vendor unavailable โ€” no independent jury seated** | +| Deeper reviewer / overturn | codex | โŒ overturn round not run | + +**Anti-collusion status:** 2 distinct vendors did file charges (Claude + GPT-via-codex), +satisfying `minDistinctVendors: 2`. But `writerIsNeverJuror` could not be *enforced* +because no jury was seated โ€” the author (Claude) adjudicated the charges. That is the +weakness in this record. The human judge is the real backstop. + +--- + +## Charges filed + +### Prosecutor: brutal-honesty / Tier-3 reviewer (Claude) โ€” 2 blockers, 1 false claim + +| # | Charge | Status | +|---|---|---| +| 1 | **Arch split.** `rust-build` used `HEAVY_RUNNER` while `rust-test` hardcoded `ubuntu-latest`. A nextest archive is host-bound (pins host target triple, ships host libstd). Setting `HEAVY_RUNNER` to `ubuntu-24.04-arm` โ€” its documented purpose โ€” would emit aarch64 binaries `rust-test` cannot exec. Impossible before the split, when build+run shared a runner. | **REPRODUCED โ†’ FIXED** (both jobs now share `runs-on`) | +| 2 | **Bench gated the artifact.** `cargo bench --no-run` ran before `upload-artifact`, so a bench-only break aborted the job pre-upload and skipped `rust-test`. | **REPRODUCED โ†’ FIXED** | +| 3 | **False claim in my own comment.** I wrote that benches were "nearly free" in `rust-build` because they share its target dir. Verified false: `cargo bench --no-run` builds the *optimized* `bench` profile into `target/release`, a separate full dep-graph build. | **CONFIRMED โ†’ comment removed, bench moved to its own job** | + +### Prosecutor: codex-review (GPT โ€” cross-vendor) โ€” 3 charges + +| # | Charge | Status | +|---|---|---| +| 1 | **Test-signal suppression.** Moving `upload-artifact` earlier fixes artifact *availability* but not job *conclusion*. A failing doctest step after the upload still fails `rust-build`; `rust-test` `needs:` it, so GitHub skips `rust-test` and all ~1177 tests go dark. **The upload-first comment was false.** | **REPRODUCED โ†’ FIXED.** `rust-build` now *ends* at upload; doctests moved to independent `rust-extra-checks`. | +| 2 | **Version-skew / supply-chain race.** Both jobs installed `cargo-nextest` unpinned via mutable `@v2`. A release landing between the jobs could mismatch archive producer and consumer; nextest's archiving docs require matching versions. | **VALID โ†’ FIXED.** Both pinned to `cargo-nextest@0.9.140`. | +| 3 | **Merge-gate regression.** Benches moved out of the `Rust Tests` context; if branch protection required only that context, a bench failure would no longer block merge. | **NOT CURRENTLY EXPLOITABLE.** Verified via `gh api`: neither `main` nor `develop` has branch protection, so there are no required contexts to regress. Recorded for whenever protection is added โ€” the new job names (`rust-build`, `rust-extra-checks`) must be included. | + +**Notable:** charge 1 from the cross-vendor prosecutor is exactly the kind of finding +the court exists for โ€” the Claude-side reviewer found the *adjacent* bug (ordering) +and I applied a fix that looked sufficient but wasn't. A second vendor caught that the +fix was incomplete. Single-lens review would have shipped this. + +--- + +## Kill round + +Not run (no blind refuter seated). All charges above were instead **verified directly +by reproduction or by reading the authoritative source** before being accepted: + +- Charge 3 (merge-gate) was *downgraded* by direct evidence (`gh api` โ†’ 404 "Branch not protected"). +- The bench-profile claim was confirmed by observing `cargo bench --no-run` build the optimized profile. + +## Verdict + +**REMAND โ†’ charges fixed โ†’ re-rendered as SHIP (partial court).** + +Every reproduced charge was fixed in-phase; none was waived. The verdict is marked +*partial* because no independent jury was seated and no overturn round ran, so the +asymmetric "SHIP must survive escalation" guarantee โ€” the mechanic that makes a court +harder to fool than a review โ€” **did not apply here**. + +## Evidence backing the delivery + +- Archive builds: 15 binaries, 106 files, **125 MB**. +- Run-from-archive in a *different directory*: **1177 tests run, 1177 passed, 7 skipped**. +- `cargo test --workspace`: **1177 passed, 7 ignored** โ€” exact parity, no lost tests. +- Doctests: 2 present, both `ignored`, **0 executed** either way โ†’ nextest's lack of + doctest support costs nothing today, and `rust-extra-checks` now guards future ones. +- `--workspace-remap .` and `--profile ci` both verified working locally. + +## Correction to an earlier record + +The phase-0 PR and ledger reported **"39/39 tests passing."** That was **wrong** โ€” it +summed only the tail of `cargo test` output and missed `lib.rs` (1004 unit tests) and +`main.rs` (65). The real figure was always **1177**. Corrected here for the record. + +## For the human judge + +**Strongest case FOR shipping:** exact 1177-test parity between the old and new paths, +verified by execution rather than inference; every charge from two vendors was +reproduced and fixed rather than argued away; the wall-clock win is structural +(`rust-build` runs concurrently with clippy instead of `rust-test` waiting for clippy +and only then compiling). + +**Strongest case AGAINST:** no independent jury and no overturn round โ€” the author +adjudicated charges against their own work. Two of the three most serious findings +were caught only *after* an initial fix looked adequate, which is evidence the +remaining unreviewed surface may hold more. The 125 MB artifact round-trip is new +per-run cost that partially offsets the scheduling win, and has not been measured in +CI. And the real wall-clock benefit is still **unproven in CI** โ€” the rust-cache has +been cold since phase 1 (`pl-rust-cache-cold-after-env-change`), so no trustworthy +warm baseline exists to compare against yet. diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 00fde02..47d33ef 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -5,3 +5,5 @@ {"phase":1,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (not needed โ€” no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority: diff is .github/workflows/*.yml only and cannot affect compilation)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"ca35144","at":"2026-07-31T11:35:15-07:00","summary":"CI env (CARGO_INCREMENTAL/PROFILE_TEST_DEBUG=0), cargo-audit via taiki-e/install-action, and release-notes fallback + if-no-files-found. DoD 5/5; yamllint+parse green. Tier-3 caught that F-11 as-specified would have moved a failure downstream โ€” fixed properly. 1 parking-lot item (pl-test-debuginfo-tradeoff)."} {"phase":2,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services involved)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” diff is workflow YAML only)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"8efc74b","at":"2026-07-31T11:57:30-07:00","summary":"Deleted toothless lighthouse+bundlewatch PR jobs (11->9 CI jobs, -2 redundant frontend builds/PR); added manual-dispatch lighthouse.yml; killed stale ADR-005 comment. DoD 8/8. Tier-3: relocated Lighthouse still warn-only so cannot gate โ€” recorded pl-lighthouse-warn-only rather than silently changing the quality bar; also pl-ci-job-doc-drift."} {"phase":3,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” workflow-YAML-only diff; and this phase changes HOW CI installs the toolchain, so CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"9f24209","at":"2026-07-31T12:29:51-07:00","summary":"Workflows now read the Rust pin from rust-toolchain.toml (no more @stable drift); lld+RUSTFLAGS scoped to rust-test; HEAVY_RUNNER opt-in on rust-test+docker-images. Corrected a stale DoD demanding 1.96 (real pin 1.97.0 โ€” would have broken the build). Tier-3 caught that HEAVY_RUNNER could silently republish :latest as arm64-only; pinned platforms: linux/amd64. DoD 7/7."} +{"phase":4,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test deferred to PR CI (workflow restructure โ€” CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":["qe_court (partial)"],"marker":"6d3296b","at":"2026-07-31T13:07:53-07:00","summary":"nextest archive: rust-build compiles once concurrently with clippy; rust-test runs from artifact. 1177-test parity verified by execution. Court REMANDed 4 charges (arch split, test-signal suppression, version skew, false bench claim) โ€” all fixed. Court PARTIAL: no jury (cognitum unconfigured), referee.ts absent."} +{"type":"court","phase":4,"verdict":"SHIP","charges_surviving":0,"overturn_rounds":0,"vendors":2,"record":".autopilot/court/ci-build-optimization/phase-4.md","at":"2026-07-31T13:07:53-07:00","note":"PARTIAL COURT โ€” no independent jury seated (cognitum unconfigured, routed 4/8 roles incl. jury); validateCourtConfig unavailable; overturn round not run. 2 vendors filed charges (Claude + GPT/codex). Author adjudicated. Weaker guarantee than a full court."} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 834ef4a..74a8106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,18 +68,18 @@ jobs: -D warnings \ -A dead_code -A unused_variables -A unused_imports -A unused_mut - rust-test: - name: Rust Tests - # Opt-in bigger/native runner. Unset (the default) resolves to ubuntu-latest, - # so this is safe everywhere; setting the repo variable to a label that does - # not exist would queue forever, which is why it is parameterized rather than - # hardcoded to a non-default runner. + # Compile the test binaries ONCE, here, and hand them to rust-test as a nextest + # archive. The win is scheduling, not fewer compiles: this job has no `needs:`, + # so it compiles CONCURRENTLY with rust-format/rust-clippy, whereas the old + # rust-test waited for clippy and only then started its own full compile of the + # workspace plus the heavy vendored ruvector crates. Wall-clock goes from + # (clippy THEN compile+run) to (max(clippy, compile) THEN run). + rust-build: + name: Rust Build (test archive) + # Opt-in bigger/native runner. Unset (the default) resolves to ubuntu-latest. + # This moved here from rust-test along with lld: after this change rust-test + # only executes prebuilt binaries, so the compile/link cost lives in THIS job. runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} - needs: [rust-format, rust-clippy] - # This is where the linking cost actually is: the test binaries and benches. - # (clippy does link build scripts and proc-macros, but not the test binaries, - # so it has little to gain.) RUSTFLAGS is scoped to this job so a missing - # linker can never break an unrelated one. env: RUSTFLAGS: -Clink-arg=-fuse-ld=lld steps: @@ -96,15 +96,113 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ steps.rust.outputs.channel }} + # Pin the version: the archive producer and consumer must be the same + # nextest, and an unpinned install lets a release landing between the two + # jobs mismatch them (nextest's own archiving docs call this out). + - uses: taiki-e/install-action@v2 + with: + tool: cargo-nextest@0.9.140 - uses: Swatinem/rust-cache@v2 with: workspaces: backend - - name: Run tests + - name: Build test archive working-directory: backend - run: cargo test --workspace + run: cargo nextest archive --workspace --archive-file nextest.tar.zst + # Upload IMMEDIATELY after archiving, before any other check. Anything that + # runs between the archive and the upload can abort the job and skip + # rust-test entirely, costing the signal from all ~1177 tests to diagnose + # an unrelated failure. + - uses: actions/upload-artifact@v7 + with: + name: nextest-archive + path: backend/nextest.tar.zst + # A missing archive MUST fail loudly โ€” rust-test cannot run without it. + if-no-files-found: error + # ~125MB; it is only needed for the life of this run. + retention-days: 1 + # NOTE: nothing else may run in this job after the upload. Uploading first + # guarantees the ARTIFACT exists, but a later failing step still fails the + # JOB โ€” and rust-test `needs:` this one, so GitHub would skip it and all + # ~1177 tests' signal would vanish while diagnosing something unrelated. + # That is why doctests live in rust-extra-checks, not here. + + # Compile-only checks that are NOT the primary test signal. They live in their + # own job for one specific reason: rust-test `needs: rust-build`, so ANY failing + # step in rust-build makes GitHub skip rust-test and destroy the signal from all + # ~1177 tests while you debug something unrelated. Uploading the artifact early + # does not help โ€” the artifact exists, but the job still fails and the dependent + # job is still skipped. Keeping these here means a broken bench or doctest + # reports itself and nothing else goes dark. + # + # `cargo bench --no-run` also builds the OPTIMIZED bench profile into + # target/release โ€” a separate build from the archive's target/debug, so putting + # it in rust-build would have cost real time on the critical path too. + rust-extra-checks: + name: Rust Extra Checks (benches, doctests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + submodules: true + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust.outputs.channel }} + - uses: Swatinem/rust-cache@v2 + with: + workspaces: backend - name: Verify benchmarks compile working-directory: backend run: cargo bench --no-run + # nextest cannot run doctests. Today that costs nothing (both doctests in + # this crate are `ignored`, so `cargo test --doc` executes 0), but without + # this a future REAL doctest would be silently never run. + - name: Doctests (nextest cannot run these) + working-directory: backend + run: cargo test --doc + + rust-test: + # MUST match rust-build's runner. A nextest archive is host-bound: it pins the + # host target triple and ships the host libstd, so binaries built on an arm64 + # HEAVY_RUNNER cannot execute on an amd64 runner. Before the archive split, + # build and run shared one machine and this could not happen. + name: Rust Tests + runs-on: ${{ vars.HEAVY_RUNNER || 'ubuntu-latest' }} + needs: [rust-format, rust-clippy, rust-build] + steps: + # submodules are load-bearing here even though nothing compiles: nextest + # resolves the archived workspace manifest, so the checkout must match. + - uses: actions/checkout@v7 + with: + submodules: true + - name: Resolve pinned toolchain + id: rust + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust.outputs.channel }} + # Must match rust-build's pinned version exactly โ€” the archive is produced + # and consumed by nextest, and a version skew between the two jobs is a + # confusing failure mode nextest's archiving docs warn about. + - uses: taiki-e/install-action@v2 + with: + tool: cargo-nextest@0.9.140 + - uses: actions/download-artifact@v8 + with: + name: nextest-archive + path: backend + # No rust-cache and no lld here on purpose: nothing is compiled or linked, + # so a cache restore would cost time and buy nothing. + # --workspace-remap is not strictly required while the build and test jobs + # happen to check out to the same $GITHUB_WORKSPACE path, but relying on + # that coincidence hard-errors ("workspace root manifest does not exist") + # on any runner that lays out paths differently. Passing it removes the + # whole failure class for free. + - name: Run tests from archive + working-directory: backend + run: cargo nextest run --profile ci --archive-file nextest.tar.zst --workspace-remap . rust-audit: name: Rust Security Audit diff --git a/.gitignore b/.gitignore index e64c31f..c0e5282 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,5 @@ docker-compose.override.yml # RuVector / ruflo local agent databases -- machine artifacts, not library content agentdb.rvf agentdb.rvf.lock +.autopilot/queued/ +nextest.tar.zst diff --git a/backend/.config/nextest.toml b/backend/.config/nextest.toml new file mode 100644 index 0000000..0f71ecd --- /dev/null +++ b/backend/.config/nextest.toml @@ -0,0 +1,32 @@ +# cargo-nextest configuration โ€” https://nexte.st/docs/configuration/ +# +# Lives at backend/.config/nextest.toml because nextest resolves this file +# relative to the CARGO WORKSPACE root, which is backend/, not the repo root. +# +# Note on doctests: nextest does not run them. That costs nothing here โ€” this +# crate has exactly 2 doctests and both are `ignored`, so `cargo test --doc` +# executes 0 of them (verified 2026-07-31). If a real doctest is ever added, +# CI needs a separate `cargo test --doc` step. + +[profile.default] +# Surface every failure in one run rather than stopping at the first, so a +# broken PR reports its full blast radius instead of one symptom at a time. +fail-fast = false + +# Deliberately NO retries. Retrying would paper over exactly the flakiness this +# phase might expose: dropping the separate recompile can surface a latent +# order- or timing-dependent test. OPTIMIZATION_SPEC F-5 calls for such a test +# to be treated as EXPOSED (and fixed deterministically), not silently retried. +retries = 0 + +# Flag pathologically slow tests instead of letting them quietly dominate CI. +# `terminate-after` bounds a genuinely hung test rather than burning the job. +slow-timeout = { period = "60s", terminate-after = 5 } + +[profile.ci] +# Inherits the above; CI wants failure detail inline and a quieter pass path. +fail-fast = false +retries = 0 +failure-output = "immediate-final" +status-level = "fail" +final-status-level = "slow" From 0a53d78b95d4e8d5942415f9b6309a3aff63d90d Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 13:21:34 -0700 Subject: [PATCH 13/24] chore(autopilot:ci-build-optimization): record measured CI timings after phase 4 --- .autopilot/discovered/ci-build-optimization.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index 23f5349..0d720bc 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -4,3 +4,4 @@ {"id":"pl-ci-job-doc-drift","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T11:51:10-07:00"},"discovered_by":"reviewer","note":"Docs drifted from CI reality after removing the lighthouse/bundlewatch jobs: docs/ADRs/ADR-005-tauri-to-web-spa-migration.md:65 still claims a \"Bundlewatch CI gate\" (which never had teeth โ€” it only echoed a warning), and docs/plan/march-2026-audit.v2.md:301 still counts 11 CI jobs (now 9). Out of scope for phase 2 (touches only the workflow files + lighthouserc.json); ADR/plan docs are historical records, so a human should decide whether to amend or leave them as point-in-time.","status":"open"} {"id":"pl-rust-cache-cold-after-env-change","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T12:13:54-07:00"},"discovered_by":"gate","note":"Rust Tests wall-clock trended 5m58s (phase 0) -> 10m48s (phase 1) -> 13m50s (phase 2) vs the spec baseline of 311s warm. Root cause found in the phase-2 job log: Swatinem/rust-cache reports \"No cache found.\" It runs with add-rust-environment-hash-key:true, so phase 1 adding CARGO_INCREMENTAL/CARGO_PROFILE_TEST_DEBUG to the workflow env changed the cache key; GitHub also scopes PR-branch caches to their own branch. The develop-scoped cache for the new key only populates once a push-triggered run on develop completes. Expected to self-resolve โ€” but phase 4 (F-5 nextest archive) must NOT compare against these cold numbers when it re-measures warm time-to-green.","status":"open"} {"id":"pl-dockerfile-rust-version-third-source","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":3,"at":"2026-07-31T12:14:55-07:00"},"discovered_by":"reviewer","note":"backend/Dockerfile hardcodes `rust:1.97-slim` (lines ~2 and ~22), so the Rust version still has a THIRD source of truth alongside backend/rust-toolchain.toml and backend/Cargo.toml rust-version. Phase 3 made the CI workflows read the pin from rust-toolchain.toml, but did not touch the Dockerfile โ€” that file belongs to phase 5 (Docker build context), which is already rewriting its COPY/context lines. Worth folding the version single-sourcing into phase 5 (e.g. an ARG defaulted from the toolchain file, or at minimum a comment cross-referencing it) so a future MSRV bump does not leave the image on a stale compiler.","status":"open"} +{"id":"pl-bench-job-now-longest-pole","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":4,"at":"2026-07-31T13:21:20-07:00"},"discovered_by":"gate","note":"MEASURED after phase 4 landed. Time-to-test-signal improved hugely: Rust Tests went 12m51s -> 58s (it no longer compiles), and with rust-build (4m39s) running concurrently with clippy (1m33s), the test verdict now arrives in ~5m37s instead of ~16m. BUT total workflow wall-clock is now bound by the new rust-extra-checks job at 11m7s, because `cargo bench --no-run` builds the OPTIMIZED bench profile from a cold cache โ€” a separate full dep-graph build from the archive`s target/debug. Options if PR wall-clock matters more than bench coverage: run benches only on develop/main pushes rather than every PR, cache the release profile separately, or drop `cargo bench --no-run` to a scheduled job. Not actioned in phase 4 because it was not in scope and benches compiling IS real signal.","status":"open"} From c5177432d78b54dc0cadfb87f1d3cd726967967d Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 13:52:37 -0700 Subject: [PATCH 14/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=205=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../discovered/ci-build-optimization.jsonl | 2 + .autopilot/pipeline.yml | 31 ++++- .autopilot/runs/ci-build-optimization.jsonl | 1 + .dockerignore | 57 +++++++++ .github/workflows/docker.yml | 21 +++- .github/workflows/release.yml | 8 +- backend/Dockerfile | 112 +++++++++++++++--- docker-compose.dev.yml | 7 +- docker-compose.yml | 6 +- 9 files changed, 220 insertions(+), 25 deletions(-) create mode 100644 .dockerignore diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index 0d720bc..3584f82 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -5,3 +5,5 @@ {"id":"pl-rust-cache-cold-after-env-change","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":2,"at":"2026-07-31T12:13:54-07:00"},"discovered_by":"gate","note":"Rust Tests wall-clock trended 5m58s (phase 0) -> 10m48s (phase 1) -> 13m50s (phase 2) vs the spec baseline of 311s warm. Root cause found in the phase-2 job log: Swatinem/rust-cache reports \"No cache found.\" It runs with add-rust-environment-hash-key:true, so phase 1 adding CARGO_INCREMENTAL/CARGO_PROFILE_TEST_DEBUG to the workflow env changed the cache key; GitHub also scopes PR-branch caches to their own branch. The develop-scoped cache for the new key only populates once a push-triggered run on develop completes. Expected to self-resolve โ€” but phase 4 (F-5 nextest archive) must NOT compare against these cold numbers when it re-measures warm time-to-green.","status":"open"} {"id":"pl-dockerfile-rust-version-third-source","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":3,"at":"2026-07-31T12:14:55-07:00"},"discovered_by":"reviewer","note":"backend/Dockerfile hardcodes `rust:1.97-slim` (lines ~2 and ~22), so the Rust version still has a THIRD source of truth alongside backend/rust-toolchain.toml and backend/Cargo.toml rust-version. Phase 3 made the CI workflows read the pin from rust-toolchain.toml, but did not touch the Dockerfile โ€” that file belongs to phase 5 (Docker build context), which is already rewriting its COPY/context lines. Worth folding the version single-sourcing into phase 5 (e.g. an ARG defaulted from the toolchain file, or at minimum a comment cross-referencing it) so a future MSRV bump does not leave the image on a stale compiler.","status":"open"} {"id":"pl-bench-job-now-longest-pole","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":4,"at":"2026-07-31T13:21:20-07:00"},"discovered_by":"gate","note":"MEASURED after phase 4 landed. Time-to-test-signal improved hugely: Rust Tests went 12m51s -> 58s (it no longer compiles), and with rust-build (4m39s) running concurrently with clippy (1m33s), the test verdict now arrives in ~5m37s instead of ~16m. BUT total workflow wall-clock is now bound by the new rust-extra-checks job at 11m7s, because `cargo bench --no-run` builds the OPTIMIZED bench profile from a cold cache โ€” a separate full dep-graph build from the archive`s target/debug. Options if PR wall-clock matters more than bench coverage: run benches only on develop/main pushes rather than every PR, cache the release profile separately, or drop `cargo bench --no-run` to a scheduled job. Not actioned in phase 4 because it was not in scope and benches compiling IS real signal.","status":"open"} +{"id":"pl-healthcheck-flag-unhandled","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":5,"at":"2026-07-31T13:21:34-07:00"},"discovered_by":"reviewer","note":"docker-compose.yml healthcheck invokes the binary with `--healthcheck`, but backend/src/main.rs parses only --download-model[s] and ignores unknown flags โ€” so the healthcheck SPAWNS A SECOND SERVER instead of probing the running one, and the container never reports healthy. Pre-existing, not caused by phase 5. Fix: implement a --healthcheck short-circuit in main.rs, or change the compose healthcheck to an HTTP probe against the existing listener. Out of scope for phase 5 (build context), and it needs a main.rs change rather than a Docker change.","status":"open"} +{"id":"pl-phase5-not-ci-verified","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":5,"at":"2026-07-31T13:21:34-07:00"},"discovered_by":"reviewer","note":"Phase 5 fixes the Docker build context, but NO CI job exercises it: neither docker.yml nor release.yml passes `target:`, so buildx builds the LAST stage (development), which never runs cargo build --release. The fix is therefore verified only locally in phase 5. Phase 6 (F-1) is what adds `target: runtime` and makes CI actually compile the image โ€” until phase 6 lands, a regression in the phase-5 work would not be caught by CI. This is inherent to the spec ordering (F-1 depends_on F-2), not a defect, but it means phase 5 should not be read as CI-proven.","status":"open"} diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml index 3a7023a..445ae72 100644 --- a/.autopilot/pipeline.yml +++ b/.autopilot/pipeline.yml @@ -179,7 +179,36 @@ phases: - "grep: 'submodules: true' in .github/workflows/release.yml" - "grep: ruvector in backend/Dockerfile" - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from repo root with the reworked context)" - conventions: "ruvector is a vendored third-party git submodule consumed via path deps โ€” do not edit it, only fix how the build context references it (OPTIMIZATION_SPEC.md preamble)." + - "cmd: test \"$(docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'ldd /app/emailibrium 2>&1 | grep -c \"not found\"')\" = \"0\" (every shared lib resolves inside the image โ€” a successful BUILD does not prove the binary can run)" + conventions: | + ruvector is a vendored third-party git submodule consumed via path deps โ€” do not edit it, + only fix how the build context references it (OPTIMIZATION_SPEC.md preamble). + + DoD STRENGTHENED 2026-07-31 during execution. A build that exits 0 does NOT prove the image + works. This phase produced a clean 179MB image containing a 72MB binary that died instantly + on startup with "version `GLIBC_2.39' not found", because `rust:1.97-slim` now resolves to a + trixie-based image (glibc 2.41 / GCC 13+) while the runtime stage was debian:bookworm-slim + (glibc 2.36 / GCC 12). Shipping that would have reproduced the very defect F-1 exists to fix. + The added runtime check is what catches it, so it is now part of the gate rather than a thing + the implementer happened to try. + + That check is `ldd`, NOT `--help`. OPTIMIZATION_SPEC F-1 suggests `/app/emailibrium --help`, + but this binary parses `std::env::args()` directly with no clap and no --help handler, so it + ignores the flag and proceeds to real startup. `ldd` reporting zero unresolved libraries + tests exactly the defect class โ€” whether the binary's dynamic dependencies exist in the + runtime image โ€” and is independent of app configuration. + + CORRECTION: the `Permission denied (os error 13)` seen while running the image was first + written off as an artifact of bare `docker run` (no volume/config/secrets). That was WRONG, + and Tier-3 review reproduced the real cause: /app/data did not exist in the image, so the + `backend_data:/app/data` mount in docker-compose.yml was created root:root while the + container runs read_only as 1000:1000 โ€” it fails under compose too, not just bare. Fixed by + creating and chown-ing /app/data before the USER directive. + + Direction of the fix matters and is counter-intuitive: ort_sys (ONNX Runtime, via fastembed) + ships PREBUILT C++ objects built with GCC 13+, so a bookworm builder cannot link them at all + (`undefined reference to __cxa_call_terminate`). Trixie is therefore a hard floor โ€” the + runtime had to move UP to match the builder, not the builder down to match the runtime. depends_on: [] touches: [".github/workflows/docker.yml", ".github/workflows/release.yml", "backend/Dockerfile"] adrs: [] diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 47d33ef..169d235 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -7,3 +7,4 @@ {"phase":3,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up (no services)","build/test/test_frontend (deferred to PR CI, the merge authority โ€” workflow-YAML-only diff; and this phase changes HOW CI installs the toolchain, so CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"9f24209","at":"2026-07-31T12:29:51-07:00","summary":"Workflows now read the Rust pin from rust-toolchain.toml (no more @stable drift); lld+RUSTFLAGS scoped to rust-test; HEAVY_RUNNER opt-in on rust-test+docker-images. Corrected a stale DoD demanding 1.96 (real pin 1.97.0 โ€” would have broken the build). Tier-3 caught that HEAVY_RUNNER could silently republish :latest as arm64-only; pinned platforms: linux/amd64. DoD 7/7."} {"phase":4,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test deferred to PR CI (workflow restructure โ€” CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":["qe_court (partial)"],"marker":"6d3296b","at":"2026-07-31T13:07:53-07:00","summary":"nextest archive: rust-build compiles once concurrently with clippy; rust-test runs from artifact. 1177-test parity verified by execution. Court REMANDed 4 charges (arch split, test-signal suppression, version skew, false bench claim) โ€” all fixed. Court PARTIAL: no jury (cognitum unconfigured), referee.ts absent."} {"type":"court","phase":4,"verdict":"SHIP","charges_surviving":0,"overturn_rounds":0,"vendors":2,"record":".autopilot/court/ci-build-optimization/phase-4.md","at":"2026-07-31T13:07:53-07:00","note":"PARTIAL COURT โ€” no independent jury seated (cognitum unconfigured, routed 4/8 roles incl. jury); validateCourtConfig unavailable; overturn round not run. 2 vendors filed charges (Claude + GPT/codex). Author adjudicated. Weaker guarantee than a full court."} +{"phase":5,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test/test_frontend (Rust/JS unaffected; this is a Docker-packaging change verified by building and running the image locally)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"1c1e527","at":"2026-07-31T13:45:50-07:00","summary":"Docker build context moved to repo root so vendored ruvector resolves. 7 defects total, 6 found by actually building/running: no .dockerignore (74GB target), ignored RUST_VERSION arg, dev compose mount, missing pkg-config/libssl-dev/g++, glibc/CXXABI trixie-vs-bookworm mismatch (built clean, binary could not run), /app/data root-owned under compose. Verified: build 0, ldd 0 unresolved, /app/data writable as uid 1000. NOT CI-verified until phase 6 adds target: runtime."} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..41c3bb4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,57 @@ +# Docker build context excludes โ€” REPO ROOT context. +# +# backend/Dockerfile builds from the repo root (it needs both backend/ and the +# vendored ruvector/ submodule, because backend/Cargo.toml has path deps on +# ../ruvector/crates/*). Without this file the daemon would be sent the whole +# tree โ€” backend/target alone is tens of GB locally โ€” and the build would crawl +# or die before the first instruction ran. +# +# Keep this list conservative: excluding something the build genuinely needs +# fails loudly at COPY time, which is the safe direction. + +# Rust build output โ€” by far the largest offender. +**/target/ + +# Node +**/node_modules/ +**/dist/ +**/.turbo/ +**/storybook-static/ + +# VCS + local state +.git/ +.gitignore +**/.DS_Store + +# Local databases and caches (never belong in an image) +**/*.db +**/*.db-shm +**/*.db-wal +**/*.rvf +**/*.rvf.lock +.lycheecache + +# Agent/tooling scratch โ€” irrelevant to the image, and some is machine-local +.claude/ +.claude-flow/ +.agentic-qe/ +.swarm/ +.beads/ +.autopilot/ +.optimizer/ +.agents/ +.codex/ +.husky/ + +# Secrets must never enter a build context. +secrets/ +**/*.pem +**/*.key +.env +.env.* + +# Docs and CI config are not needed to compile or run the service. +docs/ +images/ +.github/ +*.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 15b0c8c..6dd6029 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,20 +11,31 @@ jobs: name: Build Backend Image runs-on: ubuntu-latest steps: + # submodules are REQUIRED: backend/Cargo.toml has path deps on the vendored + # ruvector crates, and the image build COPYs ruvector/ from the context. + # Without this the directory is empty and the release compile fails. - uses: actions/checkout@v7 + with: + submodules: true - uses: docker/setup-buildx-action@v4 - uses: docker/build-push-action@v7 with: - context: ./backend + # Repo root, so ruvector/ is inside the context. See backend/Dockerfile. + context: . + file: backend/Dockerfile push: false tags: emailibrium-backend:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + # Scope the cache per image. Unscoped, the backend and frontend jobs + # share one gha cache entry and evict each other's layers. + cache-from: type=gha,scope=backend + cache-to: type=gha,mode=max,scope=backend build-frontend: name: Build Frontend Image runs-on: ubuntu-latest steps: + # The frontend image is self-contained, so it keeps a frontend/ context and + # needs no submodules. - uses: actions/checkout@v7 - uses: docker/setup-buildx-action@v4 - uses: docker/build-push-action@v7 @@ -32,5 +43,5 @@ jobs: context: ./frontend push: false tags: emailibrium-frontend:${{ github.sha }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=frontend + cache-to: type=gha,mode=max,scope=frontend diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6394530..eaf852b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -159,10 +159,15 @@ jobs: strategy: matrix: include: + # The backend image builds from the REPO ROOT so the vendored ruvector + # submodule (a path dependency of backend/Cargo.toml) is in the context. - app: backend - context: ./backend + context: . + file: backend/Dockerfile + # The frontend image is self-contained. - app: frontend context: ./frontend + file: frontend/Dockerfile steps: - uses: actions/checkout@v7 with: @@ -190,6 +195,7 @@ jobs: - uses: docker/build-push-action@v7 with: context: ${{ matrix.context }} + file: ${{ matrix.file }} push: true tags: ${{ steps.tags.outputs.tags }} # Pin the published architecture. Without this, buildx targets the diff --git a/backend/Dockerfile b/backend/Dockerfile index 290e6d4..bc28ce5 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,25 +1,107 @@ -# Stage 1: Build -FROM rust:1.97-slim AS builder +# BUILD CONTEXT IS THE REPO ROOT โ€” not backend/. +# +# backend/Cargo.toml declares path dependencies on the vendored ruvector +# submodule (../ruvector/crates/ruvector-core, ruvector-gnn). Those live OUTSIDE +# backend/, so a `context: ./backend` build cannot resolve them and the release +# compile fails at manifest load. That failure stayed invisible because CI built +# the last stage (`development`), which never runs cargo build. +# +# Every caller must therefore pass `context: .` + `dockerfile: backend/Dockerfile`: +# docker-compose.yml, .github/workflows/docker.yml and .github/workflows/release.yml +# are all wired that way. Checkouts must also fetch submodules, or ruvector/ is an +# empty directory and the COPY below silently yields nothing. +# +# See also /.dockerignore โ€” with a repo-root context it is load-bearing, not +# cosmetic (backend/target alone is tens of GB on a dev machine). + +# Single source for the Rust version across all stages. docker-compose.yml +# already passed a RUST_VERSION build arg; before this it was silently ignored, +# because no ARG was declared and the tag was hardcoded. +ARG RUST_VERSION=1.97 + +# โ”€โ”€โ”€ Stage 1: Build โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# The Debian release is pinned EXPLICITLY on both the builder and the runtime +# stage, and the two MUST stay on the same release. Getting this wrong does not +# fail the build โ€” it produces an image whose binary dies instantly at startup: +# +# builder trixie + runtime bookworm -> builds clean, then +# "/app/emailibrium: version `GLIBC_2.39' not found" (also GLIBC_2.38, +# CXXABI_1.3.15). Observed; this is how the image looked "fine" in CI. +# +# Trixie is the floor, not a preference: ort_sys (ONNX Runtime, via fastembed) +# ships PREBUILT C++ objects compiled with GCC 13+, so a bookworm builder cannot +# even link them โ€” `undefined reference to __cxa_call_terminate`. The runtime +# therefore has to come up to trixie rather than the builder going down. +# +# Bump these two lines together or not at all. +FROM rust:${RUST_VERSION}-slim-trixie AS builder + +# rust:*-slim ships neither of these, and the release build needs both. This gap +# was latent until now: the release build had never actually run in CI, because +# the workflows built the last stage (`development`), which compiles nothing. +# pkg-config + libssl-dev โ€” openssl-sys, pulled in via async-native-tls. +# g++ (libstdc++) โ€” the final link needs -lstdc++ for the C/C++ deps +# (onig_sys, simsimd, ort/ONNX). +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev g++ \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app -COPY Cargo.toml Cargo.lock ./ -COPY src/ src/ -COPY migrations/ migrations/ + +# The vendored path dependencies. This is the whole point of the repo-root +# context โ€” without it cargo cannot resolve ruvector-core / ruvector-gnn. +COPY ruvector/ ruvector/ + +# backend/ is copied piecewise rather than wholesale so the image never picks up +# local dev state (target/, *.db) or backend/.cargo/config.toml, whose +# rustc-wrapper would also require backend/scripts/ to be present. +COPY backend/rust-toolchain.toml backend/ +COPY backend/Cargo.toml backend/Cargo.lock backend/ +COPY backend/src/ backend/src/ +# benches/ is needed at manifest load: Cargo.toml declares [[bench]] +# vector_benchmarks, and cargo errors if a declared target's file is missing. +COPY backend/benches/ backend/benches/ +COPY backend/migrations/ backend/migrations/ + +WORKDIR /app/backend RUN cargo build --release -# Stage 2: Runtime -FROM debian:bookworm-slim AS runtime -RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/emailibrium /app/emailibrium -COPY --from=builder /app/migrations /app/migrations -COPY entrypoint.sh /app/entrypoint.sh +# โ”€โ”€โ”€ Stage 2: Runtime โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# MUST match the builder's Debian release โ€” see the note on the builder stage. +FROM debian:trixie-slim AS runtime +# The binary dynamically links -lssl -lcrypto -lstdc++, so those runtime libs +# must be present. Declared explicitly rather than relying on whatever the base +# image happens to pull in, so a base-image change fails at build time here +# instead of as a missing-shared-object crash on first start. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libssl3 libstdc++6 \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/backend/target/release/emailibrium /app/emailibrium +COPY --from=builder /app/backend/migrations /app/migrations +COPY backend/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh +# /app/data must exist AND be owned by the runtime uid before USER drops +# privileges. docker-compose.yml mounts backend_data:/app/data; if the mountpoint +# is absent from the image, Docker creates it root:root, and the container โ€” which +# runs read_only as 1000:1000 โ€” then dies with "Permission denied (os error 13)" +# the first time it writes there. +RUN mkdir -p /app/data && chown -R 1000:1000 /app/data WORKDIR /app USER 1000:1000 ENTRYPOINT ["/app/entrypoint.sh"] CMD ["/app/emailibrium"] -# Stage 3: Development (for docker-compose.dev.yml) -FROM rust:1.97-slim AS development -RUN cargo install cargo-watch -WORKDIR /app +# โ”€โ”€โ”€ Stage 3: Development (docker-compose.dev.yml) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Same trixie pin and same build deps as the builder stage โ€” this stage actually +# COMPILES (cargo watch -x run), so bookworm here would hit the identical +# ort_sys/GCC-13 link failure, and without pkg-config/libssl-dev/g++ it cannot +# link at all. +FROM rust:${RUST_VERSION}-slim-trixie AS development +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev g++ \ + && rm -rf /var/lib/apt/lists/* \ + && cargo install cargo-watch +# Matches the compose dev mount: docker-compose.dev.yml binds the repo root at +# /app, so the crate is at /app/backend and ../ruvector resolves correctly. +WORKDIR /app/backend CMD ["cargo", "watch", "-x", "run"] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 69992e0..6eceff6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,8 +10,13 @@ services: APP_ENV: development RUST_LOG: emailibrium=debug,tower_http=debug volumes: - - ./backend:/app + # Mount the REPO ROOT, not just backend/. The dev stage runs cargo watch in + # /app/backend, whose Cargo.toml path-depends on ../ruvector/crates/* โ€” with + # only ./backend mounted those resolve to a nonexistent /ruvector and the + # dev container cannot build at all. + - .:/app - cargo_cache:/usr/local/cargo/registry + working_dir: /app/backend command: cargo watch -x run frontend: diff --git a/docker-compose.yml b/docker-compose.yml index 6b83b42..fea2cc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,10 @@ services: # โ”€โ”€โ”€ Backend API (Rust) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ backend: build: - context: ./backend - dockerfile: Dockerfile + # Repo root: backend/Cargo.toml path-depends on the vendored ruvector + # submodule, which is outside backend/. See backend/Dockerfile. + context: . + dockerfile: backend/Dockerfile target: runtime args: RUST_VERSION: "1.97" From 5a27d5a76ebea7492e3b746e7b39a8587d1f3fe0 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 14:18:17 -0700 Subject: [PATCH 15/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=206=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../discovered/ci-build-optimization.jsonl | 2 ++ .autopilot/pipeline.yml | 24 +++++++++++++++--- .autopilot/runs/ci-build-optimization.jsonl | 2 ++ .github/workflows/docker.yml | 25 ++++++++++++++++++- .github/workflows/release.yml | 5 ++++ 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/.autopilot/discovered/ci-build-optimization.jsonl b/.autopilot/discovered/ci-build-optimization.jsonl index 3584f82..6dc9a15 100644 --- a/.autopilot/discovered/ci-build-optimization.jsonl +++ b/.autopilot/discovered/ci-build-optimization.jsonl @@ -7,3 +7,5 @@ {"id":"pl-bench-job-now-longest-pole","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":4,"at":"2026-07-31T13:21:20-07:00"},"discovered_by":"gate","note":"MEASURED after phase 4 landed. Time-to-test-signal improved hugely: Rust Tests went 12m51s -> 58s (it no longer compiles), and with rust-build (4m39s) running concurrently with clippy (1m33s), the test verdict now arrives in ~5m37s instead of ~16m. BUT total workflow wall-clock is now bound by the new rust-extra-checks job at 11m7s, because `cargo bench --no-run` builds the OPTIMIZED bench profile from a cold cache โ€” a separate full dep-graph build from the archive`s target/debug. Options if PR wall-clock matters more than bench coverage: run benches only on develop/main pushes rather than every PR, cache the release profile separately, or drop `cargo bench --no-run` to a scheduled job. Not actioned in phase 4 because it was not in scope and benches compiling IS real signal.","status":"open"} {"id":"pl-healthcheck-flag-unhandled","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":5,"at":"2026-07-31T13:21:34-07:00"},"discovered_by":"reviewer","note":"docker-compose.yml healthcheck invokes the binary with `--healthcheck`, but backend/src/main.rs parses only --download-model[s] and ignores unknown flags โ€” so the healthcheck SPAWNS A SECOND SERVER instead of probing the running one, and the container never reports healthy. Pre-existing, not caused by phase 5. Fix: implement a --healthcheck short-circuit in main.rs, or change the compose healthcheck to an HTTP probe against the existing listener. Out of scope for phase 5 (build context), and it needs a main.rs change rather than a Docker change.","status":"open"} {"id":"pl-phase5-not-ci-verified","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":5,"at":"2026-07-31T13:21:34-07:00"},"discovered_by":"reviewer","note":"Phase 5 fixes the Docker build context, but NO CI job exercises it: neither docker.yml nor release.yml passes `target:`, so buildx builds the LAST stage (development), which never runs cargo build --release. The fix is therefore verified only locally in phase 5. Phase 6 (F-1) is what adds `target: runtime` and makes CI actually compile the image โ€” until phase 6 lands, a regression in the phase-5 work would not be caught by CI. This is inherent to the spec ordering (F-1 depends_on F-2), not a defect, but it means phase 5 should not be read as CI-proven.","status":"open"} +{"id":"pl-docker-verified-arm64-only","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":6,"at":"2026-07-31T13:52:37-07:00"},"discovered_by":"reviewer","note":"All local Docker verification for phases 5 and 6 (ldd, image contents, nginx, sizes) ran on linux/arm64 (Apple Silicon), but release.yml pins platforms: linux/amd64 โ€” so the ARCH THAT ACTUALLY SHIPS was never built locally. This matters more than usual here: the defects fixed in phase 5 were toolchain/ABI-specific (glibc 2.38/2.39, CXXABI_1.3.15, ort_sys prebuilt C++ objects needing GCC 13+), and ort/ONNX ships different prebuilt binaries per arch. The new pull_request trigger on docker.yml makes CI build on amd64 runners, which closes this โ€” but until that run is green, amd64 correctness is inferred, not demonstrated.","status":"open"} +{"id":"pl-frontend-image-needs-compose-to-run","kind":"parking-lot","origin":{"feature_id":"ci-build-optimization","phase":6,"at":"2026-07-31T13:52:37-07:00"},"discovered_by":"reviewer","note":"The frontend runtime image exits 1 standalone: nginx.conf proxies to upstream `backend`, which does not resolve outside the compose network (host not found in upstream \"backend\"). The phase-6 DoD only asserts nginx is present, which passes regardless. That is acceptable for a packaging check, but it means no test proves the frontend image actually SERVES. A compose-based smoke test (bring up backend+frontend, curl the SPA) would close it. Also note frontend/ has no .dockerignore of its own and the repo-root one does not apply to context: ./frontend.","status":"open"} diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml index 445ae72..c0a9d83 100644 --- a/.autopilot/pipeline.yml +++ b/.autopilot/pipeline.yml @@ -222,9 +222,27 @@ phases: definition_of_done: - "grep: 'target: runtime' in .github/workflows/docker.yml" - "grep: 'target: runtime' in .github/workflows/release.yml" - - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . && docker run --rm emailibrium-backend:spec-check /app/emailibrium --help (exits 0 โ€” confirms the runtime stage ships the compiled binary, not the cargo-watch dev stage)" - - "cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend (exits 0 โ€” confirms the frontend image builds from the nginx runtime stage, not the node dev stage)" - conventions: "`cmd:` lines that invoke cargo must use `--manifest-path backend/Cargo.toml` (or an equivalent path arg) so they run correctly from the repo root (OPTIMIZATION_SPEC.md preamble)." + - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from the repo root)" + - "cmd: docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'test -x /app/emailibrium && test -d /app/migrations && ! command -v cargo-watch' (the backend image is the runtime stage: compiled binary + migrations present, cargo-watch absent)" + - "cmd: docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'test -x /app/emailibrium && ldd /app/emailibrium 2>&1 | grep -q \"=> /\" && ! ldd /app/emailibrium 2>&1 | grep -q \"not found\"' (FAIL-CLOSED: binary must exist AND ldd must actually resolve libs AND report none missing)" + - "cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend (exits 0)" + - "cmd: docker run --rm --entrypoint sh emailibrium-frontend:spec-check -c 'command -v nginx && test -d /usr/share/nginx/html' (the frontend image is the nginx runtime stage, not the node dev server)" + conventions: | + `cmd:` lines that invoke cargo must use `--manifest-path backend/Cargo.toml` (or an + equivalent path arg) so they run correctly from the repo root (OPTIMIZATION_SPEC.md preamble). + + DoD CORRECTED 2026-07-31, same defect as phase 5's: the original line ended in + `/app/emailibrium --help`, but this binary parses std::env::args() directly with no clap and + no --help handler โ€” it ignores the flag and proceeds to real startup, which then fails for + reasons unrelated to image correctness. Replaced with checks that actually discriminate the + runtime stage from the development stage (binary + migrations present, cargo-watch absent; + nginx present on the frontend) plus the `ldd` runnability check. + + This phase is also the FIRST time CI compiles the release image at all. Until `target: + runtime` lands here, both docker.yml and release.yml built the last stage (`development`), + so phase 5's entire Docker fix was verified only locally (pl-phase5-not-ci-verified). Expect + this phase's CI to be the real proving run for phases 5 AND 6 together โ€” and to be much + slower than previous phases, because it is a genuine cold release build. depends_on: [5] touches: [".github/workflows/docker.yml", ".github/workflows/release.yml"] adrs: [] diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 169d235..25251f6 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -8,3 +8,5 @@ {"phase":4,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test deferred to PR CI (workflow restructure โ€” CI is the only meaningful verifier)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":["qe_court (partial)"],"marker":"6d3296b","at":"2026-07-31T13:07:53-07:00","summary":"nextest archive: rust-build compiles once concurrently with clippy; rust-test runs from artifact. 1177-test parity verified by execution. Court REMANDed 4 charges (arch split, test-signal suppression, version skew, false bench claim) โ€” all fixed. Court PARTIAL: no jury (cognitum unconfigured), referee.ts absent."} {"type":"court","phase":4,"verdict":"SHIP","charges_surviving":0,"overturn_rounds":0,"vendors":2,"record":".autopilot/court/ci-build-optimization/phase-4.md","at":"2026-07-31T13:07:53-07:00","note":"PARTIAL COURT โ€” no independent jury seated (cognitum unconfigured, routed 4/8 roles incl. jury); validateCourtConfig unavailable; overturn round not run. 2 vendors filed charges (Claude + GPT/codex). Author adjudicated. Weaker guarantee than a full court."} {"phase":5,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test/test_frontend (Rust/JS unaffected; this is a Docker-packaging change verified by building and running the image locally)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"1c1e527","at":"2026-07-31T13:45:50-07:00","summary":"Docker build context moved to repo root so vendored ruvector resolves. 7 defects total, 6 found by actually building/running: no .dockerignore (74GB target), ignored RUST_VERSION arg, dev compose mount, missing pkg-config/libssl-dev/g++, glibc/CXXABI trixie-vs-bookworm mismatch (built clean, binary could not run), /app/data root-owned under compose. Verified: build 0, ldd 0 unresolved, /app/data writable as uid 1000. NOT CI-verified until phase 6 adds target: runtime."} +{"phase":6,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test/test_frontend (Docker packaging change)","qe-court cross-vendor prosecutor (codex derailed by injected plugin prompt, filed no charges)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"9c9cd22","at":"2026-07-31T14:02:43-07:00","summary":"target: runtime on all 3 build-push steps; proved the defect (frontend without target: = 207MB, no nginx; with = 69.8MB nginx). Tier-3 caught my DoD was FAIL-OPEN (ldd check passed for a missing binary) and that docker.yml had no pull_request trigger so the Docker work was never CI-verified pre-merge โ€” both fixed. Removed duplicate tags trigger."} +{"type":"court","phase":6,"verdict":"REMAND","charges_surviving":0,"overturn_rounds":0,"vendors":1,"record":"(no separate record โ€” partial court, see phase-4.md for the seating problem)","at":"2026-07-31T14:02:43-07:00","note":"PARTIAL COURT, WEAKER THAN PHASE 4: only ONE vendor filed charges. The codex/GPT prosecutor was derailed by an injected RuvNet plugin prompt and never reviewed the diff; cognitum roles remain unconfigured so no jury seated. minDistinctVendors:2 NOT met. All Tier-3 charges were fixed, but this phase did not get genuine cross-vendor adversarial review."} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6dd6029..9b92b70 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,23 @@ name: Docker on: push: branches: [main] - tags: ["v*"] + # Build the images on PRs that touch how they are built. Without this, the + # packaging was only ever exercised AFTER merge to main โ€” which is how the + # published image stayed broken (it shipped the `development` stage) without + # any PR ever going red. Scoped by paths so an ordinary source change does not + # pay for a cold release build; regular Rust/frontend CI already covers compilation. + pull_request: + paths: + - "backend/Dockerfile" + - "frontend/Dockerfile" + - "backend/Cargo.toml" + - "backend/Cargo.lock" + - ".dockerignore" + - "docker-compose*.yml" + - ".github/workflows/docker.yml" + # NOTE: deliberately NOT triggered on tags. release.yml already builds and + # pushes both images on a tag; duplicating it here ran two concurrent cold + # Rust release builds writing to the same gha cache scope. workflow_dispatch: jobs: @@ -23,6 +39,10 @@ jobs: # Repo root, so ruvector/ is inside the context. See backend/Dockerfile. context: . file: backend/Dockerfile + # Without an explicit target, buildx builds the LAST stage, which is + # `development` โ€” a cargo-watch image with no compiled binary. That is + # why this job was a no-op that never caught a broken release build. + target: runtime push: false tags: emailibrium-backend:${{ github.sha }} # Scope the cache per image. Unscoped, the backend and frontend jobs @@ -41,6 +61,9 @@ jobs: - uses: docker/build-push-action@v7 with: context: ./frontend + # Same defect on the frontend: the last stage is `development` (a node + # dev server), not the nginx:alpine runtime image we mean to ship. + target: runtime push: false tags: emailibrium-frontend:${{ github.sha }} cache-from: type=gha,scope=frontend diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eaf852b..c52a19b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -196,6 +196,11 @@ jobs: with: context: ${{ matrix.context }} file: ${{ matrix.file }} + # THE published-artifact fix. Without this, buildx builds each + # Dockerfile's LAST stage โ€” `development` โ€” so every release published + # a cargo-watch / node-dev-server image to ghcr as :latest, with no + # compiled binary, no migrations and no entrypoint. It could not run. + target: runtime push: true tags: ${{ steps.tags.outputs.tags }} # Pin the published architecture. Without this, buildx targets the From a07de31d01c5b1e43a093621e205a76439fa9ebb Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 14:53:53 -0700 Subject: [PATCH 16/24] =?UTF-8?q?feat(autopilot:ci-build-optimization):=20?= =?UTF-8?q?phase=207=20complete=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../court/ci-build-optimization/phase-4.md | 58 +- .autopilot/pipeline.yml | 41 +- .autopilot/profile.yml | 53 +- .autopilot/runs/ci-build-optimization.jsonl | 1 + .github/scripts/generate-changelog.sh | 4 +- .markdownlint-cli2.jsonc | 12 +- .yamllint.yaml | 17 + CLAUDE.md | 18 +- Makefile | 537 -------------- OPTIMIZATION_SPEC.md | 18 +- QUICKSTART.md | 8 +- README.md | 36 +- backend/Makefile | 283 -------- backend/justfile | 353 +++++++++ docs/ADRs/ADR-032-make-to-just-task-runner.md | 81 +++ docs/deployment-guide.md | 28 +- docs/maintainer-guide.md | 164 ++--- docs/oauth-setup-guide.md | 14 +- docs/plan/mcp-maturation.md | 160 ++--- docs/releasing.md | 28 +- docs/setup-guide.md | 38 +- docs/user-guide.md | 2 +- frontend/Makefile | 167 ----- frontend/justfile | 199 ++++++ justfile | 669 ++++++++++++++++++ scripts/setup-docker.sh | 6 +- scripts/setup-prereqs.sh | 4 +- scripts/setup-secrets.sh | 4 +- scripts/setup-validate.sh | 8 +- scripts/setup.sh | 2 +- 30 files changed, 1692 insertions(+), 1321 deletions(-) delete mode 100644 Makefile delete mode 100644 backend/Makefile create mode 100644 backend/justfile create mode 100644 docs/ADRs/ADR-032-make-to-just-task-runner.md delete mode 100644 frontend/Makefile create mode 100644 frontend/justfile create mode 100644 justfile diff --git a/.autopilot/court/ci-build-optimization/phase-4.md b/.autopilot/court/ci-build-optimization/phase-4.md index ff0efac..eedaf2f 100644 --- a/.autopilot/court/ci-build-optimization/phase-4.md +++ b/.autopilot/court/ci-build-optimization/phase-4.md @@ -19,26 +19,26 @@ in this project's skill install (`.claude/skills/qe-court/` contains only `config.json`, `evals`, `schemas`, `scripts`, `SKILL.md`), so the machine-checked invariant validation could not be run at all. -Per the skill's own instruction โ€” *"do not proceed with a degraded panel and do not -silently re-route around it"* โ€” this is recorded as a **partial court**, not a full +Per the skill's own instruction โ€” _"do not proceed with a degraded panel and do not +silently re-route around it"_ โ€” this is recorded as a **partial court**, not a full one. What actually ran is stated plainly below. **This verdict carries less weight than a fully-seated court** and should be read as one strong cross-vendor lens plus the Tier-3 floor, not as the ADR-124 protocol. -| Role | Configured | Actually run | -|---|---|---| -| Defense | claude-code | โ€” (not run) | -| Prosecutor ยท devils-advocate | cognitum-mid | โŒ vendor unavailable | -| Prosecutor ยท brutal-honesty | claude-code | โœ… ran as the Tier-3 reviewer subagent | -| Prosecutor ยท sherlock | cognitum-high | โŒ vendor unavailable | -| Prosecutor ยท security-scanner | cognitum-mid | โŒ vendor unavailable | -| Prosecutor ยท mutation | ollama | โŒ not run | -| Prosecutor ยท codex-review | codex | โœ… ran (`codex exec`, GPT โ€” true cross-vendor) | -| Jury | cognitum-high | โŒ **vendor unavailable โ€” no independent jury seated** | -| Deeper reviewer / overturn | codex | โŒ overturn round not run | +| Role | Configured | Actually run | +| ----------------------------- | ------------- | ------------------------------------------------------ | +| Defense | claude-code | โ€” (not run) | +| Prosecutor ยท devils-advocate | cognitum-mid | โŒ vendor unavailable | +| Prosecutor ยท brutal-honesty | claude-code | โœ… ran as the Tier-3 reviewer subagent | +| Prosecutor ยท sherlock | cognitum-high | โŒ vendor unavailable | +| Prosecutor ยท security-scanner | cognitum-mid | โŒ vendor unavailable | +| Prosecutor ยท mutation | ollama | โŒ not run | +| Prosecutor ยท codex-review | codex | โœ… ran (`codex exec`, GPT โ€” true cross-vendor) | +| Jury | cognitum-high | โŒ **vendor unavailable โ€” no independent jury seated** | +| Deeper reviewer / overturn | codex | โŒ overturn round not run | **Anti-collusion status:** 2 distinct vendors did file charges (Claude + GPT-via-codex), -satisfying `minDistinctVendors: 2`. But `writerIsNeverJuror` could not be *enforced* +satisfying `minDistinctVendors: 2`. But `writerIsNeverJuror` could not be _enforced_ because no jury was seated โ€” the author (Claude) adjudicated the charges. That is the weakness in this record. The human judge is the real backstop. @@ -48,22 +48,22 @@ weakness in this record. The human judge is the real backstop. ### Prosecutor: brutal-honesty / Tier-3 reviewer (Claude) โ€” 2 blockers, 1 false claim -| # | Charge | Status | -|---|---|---| -| 1 | **Arch split.** `rust-build` used `HEAVY_RUNNER` while `rust-test` hardcoded `ubuntu-latest`. A nextest archive is host-bound (pins host target triple, ships host libstd). Setting `HEAVY_RUNNER` to `ubuntu-24.04-arm` โ€” its documented purpose โ€” would emit aarch64 binaries `rust-test` cannot exec. Impossible before the split, when build+run shared a runner. | **REPRODUCED โ†’ FIXED** (both jobs now share `runs-on`) | -| 2 | **Bench gated the artifact.** `cargo bench --no-run` ran before `upload-artifact`, so a bench-only break aborted the job pre-upload and skipped `rust-test`. | **REPRODUCED โ†’ FIXED** | -| 3 | **False claim in my own comment.** I wrote that benches were "nearly free" in `rust-build` because they share its target dir. Verified false: `cargo bench --no-run` builds the *optimized* `bench` profile into `target/release`, a separate full dep-graph build. | **CONFIRMED โ†’ comment removed, bench moved to its own job** | +| # | Charge | Status | +| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | +| 1 | **Arch split.** `rust-build` used `HEAVY_RUNNER` while `rust-test` hardcoded `ubuntu-latest`. A nextest archive is host-bound (pins host target triple, ships host libstd). Setting `HEAVY_RUNNER` to `ubuntu-24.04-arm` โ€” its documented purpose โ€” would emit aarch64 binaries `rust-test` cannot exec. Impossible before the split, when build+run shared a runner. | **REPRODUCED โ†’ FIXED** (both jobs now share `runs-on`) | +| 2 | **Bench gated the artifact.** `cargo bench --no-run` ran before `upload-artifact`, so a bench-only break aborted the job pre-upload and skipped `rust-test`. | **REPRODUCED โ†’ FIXED** | +| 3 | **False claim in my own comment.** I wrote that benches were "nearly free" in `rust-build` because they share its target dir. Verified false: `cargo bench --no-run` builds the _optimized_ `bench` profile into `target/release`, a separate full dep-graph build. | **CONFIRMED โ†’ comment removed, bench moved to its own job** | ### Prosecutor: codex-review (GPT โ€” cross-vendor) โ€” 3 charges -| # | Charge | Status | -|---|---|---| -| 1 | **Test-signal suppression.** Moving `upload-artifact` earlier fixes artifact *availability* but not job *conclusion*. A failing doctest step after the upload still fails `rust-build`; `rust-test` `needs:` it, so GitHub skips `rust-test` and all ~1177 tests go dark. **The upload-first comment was false.** | **REPRODUCED โ†’ FIXED.** `rust-build` now *ends* at upload; doctests moved to independent `rust-extra-checks`. | -| 2 | **Version-skew / supply-chain race.** Both jobs installed `cargo-nextest` unpinned via mutable `@v2`. A release landing between the jobs could mismatch archive producer and consumer; nextest's archiving docs require matching versions. | **VALID โ†’ FIXED.** Both pinned to `cargo-nextest@0.9.140`. | -| 3 | **Merge-gate regression.** Benches moved out of the `Rust Tests` context; if branch protection required only that context, a bench failure would no longer block merge. | **NOT CURRENTLY EXPLOITABLE.** Verified via `gh api`: neither `main` nor `develop` has branch protection, so there are no required contexts to regress. Recorded for whenever protection is added โ€” the new job names (`rust-build`, `rust-extra-checks`) must be included. | +| # | Charge | Status | +| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | **Test-signal suppression.** Moving `upload-artifact` earlier fixes artifact _availability_ but not job _conclusion_. A failing doctest step after the upload still fails `rust-build`; `rust-test` `needs:` it, so GitHub skips `rust-test` and all ~1177 tests go dark. **The upload-first comment was false.** | **REPRODUCED โ†’ FIXED.** `rust-build` now _ends_ at upload; doctests moved to independent `rust-extra-checks`. | +| 2 | **Version-skew / supply-chain race.** Both jobs installed `cargo-nextest` unpinned via mutable `@v2`. A release landing between the jobs could mismatch archive producer and consumer; nextest's archiving docs require matching versions. | **VALID โ†’ FIXED.** Both pinned to `cargo-nextest@0.9.140`. | +| 3 | **Merge-gate regression.** Benches moved out of the `Rust Tests` context; if branch protection required only that context, a bench failure would no longer block merge. | **NOT CURRENTLY EXPLOITABLE.** Verified via `gh api`: neither `main` nor `develop` has branch protection, so there are no required contexts to regress. Recorded for whenever protection is added โ€” the new job names (`rust-build`, `rust-extra-checks`) must be included. | **Notable:** charge 1 from the cross-vendor prosecutor is exactly the kind of finding -the court exists for โ€” the Claude-side reviewer found the *adjacent* bug (ordering) +the court exists for โ€” the Claude-side reviewer found the _adjacent_ bug (ordering) and I applied a fix that looked sufficient but wasn't. A second vendor caught that the fix was incomplete. Single-lens review would have shipped this. @@ -74,7 +74,7 @@ fix was incomplete. Single-lens review would have shipped this. Not run (no blind refuter seated). All charges above were instead **verified directly by reproduction or by reading the authoritative source** before being accepted: -- Charge 3 (merge-gate) was *downgraded* by direct evidence (`gh api` โ†’ 404 "Branch not protected"). +- Charge 3 (merge-gate) was _downgraded_ by direct evidence (`gh api` โ†’ 404 "Branch not protected"). - The bench-profile claim was confirmed by observing `cargo bench --no-run` build the optimized profile. ## Verdict @@ -82,14 +82,14 @@ by reproduction or by reading the authoritative source** before being accepted: **REMAND โ†’ charges fixed โ†’ re-rendered as SHIP (partial court).** Every reproduced charge was fixed in-phase; none was waived. The verdict is marked -*partial* because no independent jury was seated and no overturn round ran, so the +_partial_ because no independent jury was seated and no overturn round ran, so the asymmetric "SHIP must survive escalation" guarantee โ€” the mechanic that makes a court harder to fool than a review โ€” **did not apply here**. ## Evidence backing the delivery - Archive builds: 15 binaries, 106 files, **125 MB**. -- Run-from-archive in a *different directory*: **1177 tests run, 1177 passed, 7 skipped**. +- Run-from-archive in a _different directory_: **1177 tests run, 1177 passed, 7 skipped**. - `cargo test --workspace`: **1177 passed, 7 ignored** โ€” exact parity, no lost tests. - Doctests: 2 present, both `ignored`, **0 executed** either way โ†’ nextest's lack of doctest support costs nothing today, and `rust-extra-checks` now guards future ones. @@ -111,7 +111,7 @@ and only then compiling). **Strongest case AGAINST:** no independent jury and no overturn round โ€” the author adjudicated charges against their own work. Two of the three most serious findings -were caught only *after* an initial fix looked adequate, which is evidence the +were caught only _after_ an initial fix looked adequate, which is evidence the remaining unreviewed surface may hold more. The 125 MB artifact round-trip is new per-run cost that partially offsets the scheduling win, and has not been measured in CI. And the real wall-clock benefit is still **unproven in CI** โ€” the rust-cache has diff --git a/.autopilot/pipeline.yml b/.autopilot/pipeline.yml index c0a9d83..5be39ae 100644 --- a/.autopilot/pipeline.yml +++ b/.autopilot/pipeline.yml @@ -13,11 +13,11 @@ goal: "Fix the broken Docker publish (wrong build stage + missing submodule cont spec: "OPTIMIZATION_SPEC.md" references: - prd: "" - plan: "" + prd: "" + plan: "" adr_dir: "docs/ADRs" ddd_dir: "docs/DDDs" - extra: [] + extra: [] # Branch model. trunk is NEVER merged autonomously โ€” a human always merges the final PR. # Revised 2026-07-31 (user request): each phase branches off `develop`, PRs into `develop`, and @@ -29,7 +29,7 @@ references: # same sitting (2026-07-31) so phase PRs into `develop` actually get CI-checked โ€” without that, every # phase PR would show zero checks and orchestrate's anti-vacuous-green guard would refuse to merge. trunk: main -base: develop +base: develop # Autonomy mode: pr_ci โ€” branch -> PR -> CI -> bounded fix-loop -> squash-merge, no per-phase # human checkpoint. NOTE: OPTIMIZATION_SPEC.md's own preamble recommends `reviewed` for the first @@ -62,7 +62,7 @@ phases: deliverables: - "Remove the unused `ruvector-collections` path dependency (0 source refs; a submodule path-crate compiled on every build for nothing)." - "Remove the unused `encoding_rs` optional dependency, including its `dep:encoding_rs` reference in the `builtin-llm` feature." - - "Add a `[package.metadata.cargo-machete]` `ignored = [\"apalis\", \"apalis-sql\"]` allowlist with a rationale comment for the planned-but-not-yet-wired job queue deps." + - 'Add a `[package.metadata.cargo-machete]` `ignored = ["apalis", "apalis-sql"]` allowlist with a rationale comment for the planned-but-not-yet-wired job queue deps.' definition_of_done: - "cmd: cargo machete backend" - "cmd: cargo build --manifest-path backend/Cargo.toml --features builtin-llm" @@ -113,7 +113,8 @@ phases: - "cmd: test -f frontend/lighthouserc.json" conventions: "This also resolves the frontend half of F-5's redundant-build concern: removing both jobs drops 2 of the 3 full `pnpm install` + `pnpm turbo build` runs per PR (only frontend-quality's build remains)." depends_on: [] - touches: [".github/workflows/ci.yml", ".github/workflows/lighthouse.yml", "frontend/lighthouserc.json"] + touches: + [".github/workflows/ci.yml", ".github/workflows/lighthouse.yml", "frontend/lighthouserc.json"] adrs: [] ddd: [] @@ -148,7 +149,8 @@ phases: build require lld to be installed. It is scoped to the rust-test job, the only job that links (fmt/clippy emit metadata and never link). depends_on: [] - touches: [".github/workflows/ci.yml", ".github/workflows/release.yml", "backend/.cargo/config.toml"] + touches: + [".github/workflows/ci.yml", ".github/workflows/release.yml", "backend/.cargo/config.toml"] adrs: [] ddd: [] @@ -179,7 +181,7 @@ phases: - "grep: 'submodules: true' in .github/workflows/release.yml" - "grep: ruvector in backend/Dockerfile" - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from repo root with the reworked context)" - - "cmd: test \"$(docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'ldd /app/emailibrium 2>&1 | grep -c \"not found\"')\" = \"0\" (every shared lib resolves inside the image โ€” a successful BUILD does not prove the binary can run)" + - 'cmd: test "$(docker run --rm --entrypoint sh emailibrium-backend:spec-check -c ''ldd /app/emailibrium 2>&1 | grep -c "not found"'')" = "0" (every shared lib resolves inside the image โ€” a successful BUILD does not prove the binary can run)' conventions: | ruvector is a vendored third-party git submodule consumed via path deps โ€” do not edit it, only fix how the build context references it (OPTIMIZATION_SPEC.md preamble). @@ -224,7 +226,7 @@ phases: - "grep: 'target: runtime' in .github/workflows/release.yml" - "cmd: docker buildx build --target runtime -f backend/Dockerfile --load -t emailibrium-backend:spec-check . (exits 0 from the repo root)" - "cmd: docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'test -x /app/emailibrium && test -d /app/migrations && ! command -v cargo-watch' (the backend image is the runtime stage: compiled binary + migrations present, cargo-watch absent)" - - "cmd: docker run --rm --entrypoint sh emailibrium-backend:spec-check -c 'test -x /app/emailibrium && ldd /app/emailibrium 2>&1 | grep -q \"=> /\" && ! ldd /app/emailibrium 2>&1 | grep -q \"not found\"' (FAIL-CLOSED: binary must exist AND ldd must actually resolve libs AND report none missing)" + - 'cmd: docker run --rm --entrypoint sh emailibrium-backend:spec-check -c ''test -x /app/emailibrium && ldd /app/emailibrium 2>&1 | grep -q "=> /" && ! ldd /app/emailibrium 2>&1 | grep -q "not found"'' (FAIL-CLOSED: binary must exist AND ldd must actually resolve libs AND report none missing)' - "cmd: docker buildx build --target runtime -f frontend/Dockerfile --load -t emailibrium-frontend:spec-check frontend (exits 0)" - "cmd: docker run --rm --entrypoint sh emailibrium-frontend:spec-check -c 'command -v nginx && test -d /usr/share/nginx/html' (the frontend image is the nginx runtime stage, not the node dev server)" conventions: | @@ -274,6 +276,25 @@ phases: - "prose: every recipe present in the three deleted Makefiles has a behaviorally equivalent `just` recipe โ€” no silent drops. Cite the mapping for any renamed recipe." conventions: "This phase touches nearly every file the other six phases also touch, and profile.yml's commands.* are load-bearing for every future gate run โ€” that's why it's last and depends on all of 0-6, not just file-level `touches` overlap. Preserve exact recipe names where sensible (`just build`, `just test`, `just lint`, `just format-check`, `just audit`, `just ci`) so muscle memory and any external scripts/docs need minimal changes. cargo-machete's `[package.metadata.cargo-machete]` block from phase 0 and the audit-ignore file are untouched by this phase โ€” it's a task-runner swap, not a dependency change. Also a good moment to fix the discovered backend/Makefile audit-swallow bug (pl-mk-audit-swallow, .autopilot/discovered/ci-build-optimization.jsonl) โ€” it disappears for free once backend/Makefile is deleted, but the just recipe must not reintroduce an equivalent swallow." depends_on: [0, 1, 2, 3, 4, 5, 6] - touches: ["Makefile", "justfile", "backend/Makefile", "backend/justfile", "frontend/Makefile", "frontend/justfile", ".autopilot/profile.yml", "README.md", "QUICKSTART.md", "CLAUDE.md", "docs/setup-guide.md", "docs/maintainer-guide.md", "docs/deployment-guide.md", "docs/releasing.md", "docs/user-guide.md", "docs/oauth-setup-guide.md", "docs/ADRs/**"] + touches: + [ + "Makefile", + "justfile", + "backend/Makefile", + "backend/justfile", + "frontend/Makefile", + "frontend/justfile", + ".autopilot/profile.yml", + "README.md", + "QUICKSTART.md", + "CLAUDE.md", + "docs/setup-guide.md", + "docs/maintainer-guide.md", + "docs/deployment-guide.md", + "docs/releasing.md", + "docs/user-guide.md", + "docs/oauth-setup-guide.md", + "docs/ADRs/**", + ] adrs: [] ddd: [] diff --git a/.autopilot/profile.yml b/.autopilot/profile.yml index f0c608e..46a5240 100644 --- a/.autopilot/profile.yml +++ b/.autopilot/profile.yml @@ -10,27 +10,34 @@ # COMMANDS the gate runs. Leave a value empty ("") to skip that check. # These are placeholders the gate template resolves as {{commands.}}. # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# +# MIGRATED to `just` on 2026-07-31 (ADR-032, pipeline phase 7). All three GNU Makefiles +# were deleted in the same change, so these commands would break if left on `make`. +# Recipe names were deliberately preserved across the migration, so each command below +# is the same name it always was โ€” only the runner changed. commands: # Bring up anything the tests need (db, services). Runs once before the gate. Optional. - infra_up: "make docker-up-dev" # postgres, redis, qdrant, backend, frontend (hot-reload) + infra_up: "just docker-up-dev" # postgres, redis, qdrant, backend, frontend (hot-reload) # Fast formatting check (non-mutating). The gate fails if this fails. - format_check: "make format-check" # backend cargo fmt --check + frontend prettier --check + docs + format_check: "just format-check" # backend cargo fmt --check + frontend prettier --check + docs # Static analysis / linters. - lint: "make lint" # backend cargo clippy -D warnings + frontend eslint + docs lint + lint: "just lint" # backend cargo clippy -D warnings + frontend eslint + docs lint # Build everything (the phase must compile/bundle). - build: "make build" # backend cargo build + frontend turbo build + build: "just build" # backend cargo build + frontend turbo build # Primary/unit test suite (fast, no external infra ideally). - # NOTE: frontend/Makefile's `test` and `audit` targets used to swallow failures - # (`... || true` / `... 2>/dev/null || echo ...`), which would have let a broken - # phase pass the gate silently. Fixed directly in frontend/Makefile during detect - # (2026-07-31); the split commands below no longer depend on that fix holding. - test: "make -C backend test && (cd frontend && pnpm turbo test)" + # HISTORY: the old top-level test target could pass while tests failed โ€” frontend/Makefile's + # `test` target was `@$(TURBO) test || true`, and the backend audit target swallowed exit + # codes too. That is exactly why this entry used to bypass the Makefile and call the + # underlying tools directly. The justfiles fix the swallow at the source (ADR-032 + # ยง2), so this can go back to the simple top-level recipe โ€” but only because the + # honest exit code is now verified, not assumed. + test: "just test" # backend cargo test + frontend vitest run (both fail honestly) # Integration tests (may need infra_up). Optional. - test_integration: "make -C backend test-integration" # cargo test --test '*' + test_integration: "just --justfile backend/justfile --working-directory backend test-integration" # Frontend/UI tests, only relevant if the phase touches the UI. Optional. - test_frontend: "cd frontend && pnpm turbo test" # vitest run (non-watch) via turbo + test_frontend: "just --justfile frontend/justfile --working-directory frontend test" # Dependency / vulnerability audit. Optional. - audit: "make -C backend audit && (cd frontend && pnpm audit --prod)" + audit: "just audit" # cargo audit + pnpm audit --prod (both fail honestly now) # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # CONVENTIONS โ€” free-text house style, injected into the runner's 'match the @@ -55,7 +62,7 @@ conventions: | src/services/ai/__tests__/, src/features/settings/hooks/__tests__/, src/features/email/utils/groupBySender.test.ts). Run via Vitest โ€” always `vitest run` (non-watch); `test:watch` is the separate opt-in script. - - E2E via Playwright (apps/web/e2e, `make test-e2e` โ€” not part of the default gate `test`). + - E2E via Playwright (apps/web/e2e, `just test-e2e` โ€” not part of the default gate `test`). docs: architecture decisions in docs/ADRs/ADR-NNN-*.md, domain design in docs/DDDs/DDD-NNN-*.md (start with DDD-000-context-map.md for the bounded-context overview). Rich corpus โ€” worth @@ -65,7 +72,7 @@ conventions: | # CI โ€” the merge authority in pr_ci mode. # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ ci: - provider: github # github is the supported provider today + provider: github # github is the supported provider today # When true, "all required PR checks green" is the ONLY merge gate in pr_ci mode โ€” # the agent does not re-run the local gate before merging because CI already proved it. ci_is_merge_authority: true @@ -78,7 +85,7 @@ ci: # above does run frontend tests via commands.test/test_frontend, so pr_ci phases are still # protected, but the baseโ†’trunk integration PR's CI checks alone won't catch a frontend # regression). Consider adding a frontend test job to ci.yml. - base_coverage: covered # covered | trunk-only | none + base_coverage: covered # covered | trunk-only | none # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # SECURITY INVARIANTS โ€” non-negotiables the gate greps the diff for every phase. @@ -89,7 +96,7 @@ security_invariants: - "No secrets, tokens, or credentials committed or written to logs." - "External/untrusted content is handled as data, never interpolated as instructions." - "OAuth client secrets, encryption keys, and DB credentials stay in config/*.yaml (gitignored - variants) or Docker/Compose secrets โ€” never inlined in source or committed config files." + variants) or Docker/Compose secrets โ€” never inlined in source or committed config files." # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # ACCELERATORS โ€” optional tooling, in two classes that share ONE contract: @@ -105,14 +112,14 @@ accelerators: # โ”€โ”€ execution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # ruflo โ†’ drives comprehension recall + multi-agent swarms + cross-session memory. # Found on PATH globally (v3.32.41); no project-local .ruflo/ directory. - ruflo: { available: true, scope: "global" } + ruflo: { available: true, scope: "global" } # agentic-qe โ†’ drives the measured-quality fleet layered onto the gate (Tier 2/4). # Found on PATH globally (3.13.3) AND initialized in-project (.agentic-qe/). - agentic_qe: { available: true, scope: "project" } + agentic_qe: { available: true, scope: "project" } # qe-court (aqe >= 3.13, ADR-124) โ†’ adversarial review court with >=2-vendor requirement. # Skill footprint present at .claude/skills/qe-court/ (project-scoped). Vendor count = 3: # Claude (this session) + codex CLI on PATH + OPENROUTER_API_KEY in env. - qe_court: { available: true, scope: "project", vendors: 3 } + qe_court: { available: true, scope: "project", vendors: 3 } # beads (bd) โ†’ work-graph projection, disabled by user choice (2026-07-31): `bd` is on PATH # globally (v1.1.0), but `bd init` turned out to auto-commit its scaffolding AND rewrite large # sections of CLAUDE.md/AGENTS.md unprompted โ€” more invasive than the "projection only" framing @@ -120,13 +127,13 @@ accelerators: # authoritative on its own, and orchestrate/run-phase compute the ready-set from that + git # markers directly, never from beads. Revisit deliberately (re-run `bd init`) if that tradeoff # becomes worth it later; nothing about autopilot's correctness depends on it either way. - beads: { available: false, scope: "" } + beads: { available: false, scope: "" } # the /code-review skill/command โ€” the Tier-3 floor, assumed present in Claude Code. - code_review: { available: true, scope: "global" } + code_review: { available: true, scope: "global" } # โ”€โ”€ planning (skill-based; scope: "skill" | "") โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # superpowers โ†’ active in this session's skill registry (brainstorming, TDD, debugging, etc). - superpowers: { available: true, scope: "skill" } + superpowers: { available: true, scope: "skill" } # clarity โ†’ active in this session's skill registry (spec generation from references). - clarity: { available: true, scope: "skill" } + clarity: { available: true, scope: "skill" } # deep_research โ†’ not present in the active skill registry. deep_research: { available: false, scope: "" } diff --git a/.autopilot/runs/ci-build-optimization.jsonl b/.autopilot/runs/ci-build-optimization.jsonl index 25251f6..4c6da13 100644 --- a/.autopilot/runs/ci-build-optimization.jsonl +++ b/.autopilot/runs/ci-build-optimization.jsonl @@ -10,3 +10,4 @@ {"phase":5,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test/test_frontend (Rust/JS unaffected; this is a Docker-packaging change verified by building and running the image locally)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"1c1e527","at":"2026-07-31T13:45:50-07:00","summary":"Docker build context moved to repo root so vendored ruvector resolves. 7 defects total, 6 found by actually building/running: no .dockerignore (74GB target), ignored RUST_VERSION arg, dev compose mount, missing pkg-config/libssl-dev/g++, glibc/CXXABI trixie-vs-bookworm mismatch (built clean, binary could not run), /app/data root-owned under compose. Verified: build 0, ldd 0 unresolved, /app/data writable as uid 1000. NOT CI-verified until phase 6 adds target: runtime."} {"phase":6,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","build/test/test_frontend (Docker packaging change)","qe-court cross-vendor prosecutor (codex derailed by injected plugin prompt, filed no charges)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"9c9cd22","at":"2026-07-31T14:02:43-07:00","summary":"target: runtime on all 3 build-push steps; proved the defect (frontend without target: = 207MB, no nginx; with = 69.8MB nginx). Tier-3 caught my DoD was FAIL-OPEN (ldd check passed for a missing binary) and that docker.yml had no pull_request trigger so the Docker work was never CI-verified pre-merge โ€” both fixed. Removed duplicate tags trigger."} {"type":"court","phase":6,"verdict":"REMAND","charges_surviving":0,"overturn_rounds":0,"vendors":1,"record":"(no separate record โ€” partial court, see phase-4.md for the seating problem)","at":"2026-07-31T14:02:43-07:00","note":"PARTIAL COURT, WEAKER THAN PHASE 4: only ONE vendor filed charges. The codex/GPT prosecutor was derailed by an injected RuvNet plugin prompt and never reviewed the diff; cognitum roles remain unconfigured so no jury seated. minDistinctVendors:2 NOT met. All Tier-3 charges were fixed, but this phase did not get genuine cross-vendor adversarial review."} +{"phase":7,"mode":"pr_ci","verdict":"PASSED","skipped":["infra_up","test_integration (unchanged by a task-runner swap)","qe-court (phase 7 not in risk_phases)"],"failed":[],"ci_attempts":0,"pr":null,"accelerators":[],"marker":"529c934","at":"2026-07-31T14:48:20-07:00","summary":"Make -> just: 110/110 recipe parity, 3 Makefiles deleted, profile.yml+9 docs+ADR-032 migrated. just build/test/lint/format-check all exit 0; just test runs 1177 tests = exact parity with make test. 4th swallowed exit code found (root download-models). Tier-3 caught 2 must-fixes of mine: setup-prereqs still required Make not just, and my .autopilot/ yamllint ignore had disabled key-duplicates on the tracked profile.yml."} diff --git a/.github/scripts/generate-changelog.sh b/.github/scripts/generate-changelog.sh index 50901a3..c8afc1b 100755 --- a/.github/scripts/generate-changelog.sh +++ b/.github/scripts/generate-changelog.sh @@ -85,8 +85,8 @@ CATEGORIES=( echo "" echo "# From source" echo "git checkout v${VERSION}" - echo "make install" - echo "make dev" + echo "just install" + echo "just dev" echo '```' echo "" echo "**Full Changelog**: https://github.com/${REPO}/compare/${PREVIOUS_TAG}...${CURRENT_TAG}" diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index ae9285b..6a04124 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -5,9 +5,17 @@ ".claude/**", ".claude-flow/**", ".agentic-qe/**", + // Same class as the three above: tool-generated, git-ignored agent scaffolding. + // Without these, `just lint` fails on a developer's machine purely because a + // local tool wrote docs there โ€” files CI never sees, so the check was red + // locally and green in CI for the same commit. + ".agents/**", + ".codex/**", + ".swarm/**", + ".beads/**", "**/node_modules/**", "**/target/**", "ruvector/**", - ".claude/worktrees/**" - ] + ".claude/worktrees/**", + ], } diff --git a/.yamllint.yaml b/.yamllint.yaml index 5e79ff3..37b7b1a 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -10,9 +10,26 @@ ignore: | .claude/agents/ rules: + # This repo runs BOTH prettier (`format-check-yaml`) and yamllint (`lint-yaml`) + # over the same files, and their defaults contradict each other on flow + # mappings: prettier emits `{ a: 1 }`, yamllint's default `max-spaces-inside: 0` + # rejects exactly that. With the defaults, any YAML containing an inline mapping + # fails one tool or the other no matter how it is written. Allow one space so + # the two agree; prettier remains the formatter of record. + braces: + max-spaces-inside: 1 line-length: max: 200 allow-non-breakable-inline-mappings: true + # PER-RULE ignore, deliberately NOT a directory-wide one. .autopilot/ manifests + # carry long prose fields by design (machine-checkable Definition-of-Done + # strings, decision rationale), so a column limit fights the format. But + # ignoring the whole directory would also disable `key-duplicates` on the + # TRACKED .autopilot/profile.yml โ€” and a duplicate `commands:` key there would + # silently replace the quality gate's entire command set. Relax only the rule + # that actually conflicts. + ignore: | + .autopilot/ document-start: disable truthy: check-keys: false diff --git a/CLAUDE.md b/CLAUDE.md index d4cbaa2..9125c56 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,18 +17,18 @@ Vector-native, local-first email intelligence: semantic search, clustering, clas | `docs/` | `architecture.md`, `ADRs/`, `DDDs/`, evaluation, setup/oauth guides | โ€” | | `config/`, `secrets/` | Runtime config + dev secrets (never commit secrets) | โ€” | -## Build & Test โ€” Makefile-driven, not npm +## Build & Test โ€” justfile-driven, not npm -The root `package.json` only wires Husky; **do not run `npm build`/`npm test`**. Use `make`: +The root `package.json` only wires Husky; **do not run `npm build`/`npm test`**. Use `just`: ```bash -make ci # format-check + lint + typecheck + test (run before committing) -make test # backend (cargo) + frontend (Vitest) -make build # build everything -make dev # full stack: backend :8080, frontend :3000 -make lint # code + docs (markdownlint, yamllint) -make audit # cargo-audit + npm audit -make help # all targets +just ci # format-check + lint + typecheck + test (run before committing) +just test # backend (cargo) + frontend (Vitest) +just build # build everything +just dev # full stack: backend :8080, frontend :3000 +just lint # code + docs (markdownlint, yamllint) +just audit # cargo-audit + npm audit +just --list # all targets ``` Backend-only: `cd backend && cargo test` / `cargo clippy`. Frontend-only: `cd frontend && pnpm test` / `pnpm lint` / `pnpm typecheck`. diff --git a/Makefile b/Makefile deleted file mode 100644 index aff0b38..0000000 --- a/Makefile +++ /dev/null @@ -1,537 +0,0 @@ -# ============================================================================ -# Emailibrium โ€” Root Makefile -# ============================================================================ -# Delegates to backend/ and frontend/ Makefiles. -# Provides cross-cutting targets for CI, Docker, releases, and docs. -# -# Quick Start: -# make help - Show all available targets -# make install - Install all dependencies -# make dev - Start full stack (native) -# make docker-up-dev - Start full stack (Docker) -# make ci - Run full CI pipeline -# make release VERSION=x.y.z - Tag and release -# ============================================================================ - -# ============================================================================ -# Variables and Configuration -# ============================================================================ - -SHELL := /bin/bash -.DEFAULT_GOAL := help - -BACKEND_DIR := backend -FRONTEND_DIR := frontend - -COMPOSE := docker compose -COMPOSE_DEV := $(COMPOSE) -f docker-compose.yml -f docker-compose.dev.yml - -LYCHEE := $(shell command -v lychee 2>/dev/null || echo "") - -# Colors -BOLD := $(shell tput bold 2>/dev/null || echo '') -GREEN := $(shell tput setaf 2 2>/dev/null || echo '') -YELLOW := $(shell tput setaf 3 2>/dev/null || echo '') -BLUE := $(shell tput setaf 4 2>/dev/null || echo '') -RED := $(shell tput setaf 1 2>/dev/null || echo '') -RESET := $(shell tput sgr0 2>/dev/null || echo '') - -# ============================================================================ -# Default Target -# ============================================================================ - -.PHONY: help -help: - @echo "$(BOLD)$(BLUE)โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—$(RESET)" - @echo "$(BOLD)$(BLUE)โ•‘ Emailibrium Makefile โ•‘$(RESET)" - @echo "$(BOLD)$(BLUE)โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo "" - @echo "$(BOLD)Quick Start:$(RESET)" - @echo " make setup - Guided first-time setup wizard" - @echo " make install - Install all dependencies" - @echo " make dev - Start backend + frontend (native)" - @echo " make dev-llm - Start with built-in LLM (llama.cpp)" - @echo " make models - Show available LLM models" - @echo " make embedding-models - Show available embedding models" - @echo " make download-model - Download a model (MODEL=)" - @echo " make docker-up-dev - Start full stack (Docker)" - @echo " make ci - Run full CI pipeline" - @echo " make test - Run all tests" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Setup & Onboarding โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " setup - Guided first-time setup wizard" - @echo " setup-prereqs - Check all prerequisites" - @echo " setup-secrets - Generate/configure secrets" - @echo " setup-ai - Configure AI providers" - @echo " setup-docker - Set up Docker environment" - @echo " setup-validate - Validate entire setup" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Install & Build โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " install - Install all dependencies (backend + frontend)" - @echo " build - Build everything" - @echo " dev - Start full stack dev servers (native)" - @echo " dev-llm - Start with built-in LLM (llama.cpp)" - @echo " clean - Clean all build artifacts" - @echo " clean-data - Remove all local data (DB, vectors)" - @echo " clean-all - Clean build artifacts + all local data" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• AI & Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " models - Show available LLM models" - @echo " embedding-models - Show available embedding models" - @echo " download-model MODEL=x - Download a specific model" - @echo " download-models - Download AI models (ONNX + GGUF)" - @echo " diagnose - Show AI configuration diagnostics" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Test โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " test - Run all tests (backend + frontend)" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Lint & Format โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " lint - Lint everything (code + docs)" - @echo " format - Format everything (code + docs)" - @echo " format-check - Check formatting (no changes)" - @echo " typecheck - TypeScript type check" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Security & Quality โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " audit - Security audit all dependencies" - @echo " deadcode - Check for dead code" - @echo " ci - Full CI pipeline" - @echo " ci-full - CI + link checking" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Dependency Management โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " upgrade - Upgrade all deps (within semver)" - @echo " outdated - Show outdated deps (no changes)" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Documentation โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " lint-md - Lint Markdown files" - @echo " lint-yaml - Lint YAML files" - @echo " links-check - Check internal links in Markdown" - @echo " links-check-external - Check external links (slow)" - @echo " links-check-all - Check all links" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Docker โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " docker-up - Start production stack" - @echo " docker-up-dev - Start dev stack (hot-reload)" - @echo " docker-down - Stop all containers" - @echo " docker-down-volumes - Stop + remove volumes (DESTROYS DATA)" - @echo " docker-restart - Restart all containers" - @echo " docker-build - Build Docker images" - @echo " docker-build-no-cache - Build images without cache" - @echo " docker-logs - Tail all container logs" - @echo " docker-logs-backend - Tail backend logs" - @echo " docker-logs-frontend - Tail frontend logs" - @echo " docker-ps - Show container status" - @echo " docker-exec-backend - Shell into backend container" - @echo " docker-exec-frontend - Shell into frontend container" - @echo " docker-health - Health check all containers" - @echo " docker-clean - Prune dangling Docker artifacts" - @echo " docker-secrets - Generate dev secrets" - @echo "" - @echo "$(BOLD)$(BLUE)โ•โ•โ• Release โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo " release-check - Pre-release CI validation" - @echo " release-tag VERSION=x.y.z - Create annotated tag" - @echo " release-push - Push latest tag to trigger release" - @echo " release VERSION=x.y.z - Full release (check + tag + push)" - @echo " changelog VERSION=x.y.z - Preview changelog" - @echo "" - @echo " Run '$(BOLD)make -C backend$(RESET)' or '$(BOLD)make -C frontend$(RESET)' for layer-specific targets." - -# ============================================================================ -# Setup & Onboarding -# ============================================================================ - -.PHONY: setup -setup: ## Guided first-time setup wizard - @bash scripts/setup.sh - -.PHONY: setup-prereqs -setup-prereqs: ## Check all prerequisites - @bash scripts/setup-prereqs.sh - -.PHONY: setup-secrets -setup-secrets: ## Generate/configure secrets - @bash scripts/setup-secrets.sh - -.PHONY: setup-ai -setup-ai: ## Configure AI providers - @bash scripts/setup-ai.sh - -.PHONY: setup-docker -setup-docker: ## Set up Docker environment - @bash scripts/setup-docker.sh - -.PHONY: setup-validate -setup-validate: ## Validate entire setup - @bash scripts/setup-validate.sh - -.PHONY: download-models -download-models: ## Download AI models (ONNX embedding + GGUF LLM) - @echo "$(BOLD)$(BLUE)Downloading AI models...$(RESET)" - @echo "$(GREEN)Step 1:$(RESET) ONNX embedding model" - @cd $(BACKEND_DIR) && cargo run -- --download-models 2>/dev/null || echo " $(YELLOW)Backend not built. Run 'make build' first.$(RESET)" - @echo "$(GREEN)Step 2:$(RESET) GGUF LLM model (qwen2.5-0.5b-q4km)" - @cd $(FRONTEND_DIR)/apps/web && npx tsx ../../../scripts/models.ts download --default 2>/dev/null || echo " $(YELLOW)Frontend not installed. Run 'make install' first.$(RESET)" - @echo "$(GREEN)Done.$(RESET) Models cached for offline use." - -.PHONY: diagnose -diagnose: ## Show AI configuration diagnostics - @echo "$(BOLD)$(BLUE)Emailibrium AI Diagnostics$(RESET)" - @echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" - @echo "" - @echo "$(BOLD)Embedding:$(RESET)" - @if [[ -d "$(BACKEND_DIR)/.fastembed_cache" ]]; then \ - echo " Provider: ONNX (all-MiniLM-L6-v2)"; \ - echo " Status: $(GREEN)cached$(RESET) ($$(du -sh $(BACKEND_DIR)/.fastembed_cache 2>/dev/null | cut -f1))"; \ - else \ - echo " Provider: ONNX (all-MiniLM-L6-v2)"; \ - echo " Status: $(YELLOW)not cached (downloads on first use)$(RESET)"; \ - fi - @echo "" - @echo "$(BOLD)Generative (LLM):$(RESET)" - @CACHE="$$HOME/.emailibrium/models/llm"; \ - if [[ -d "$$CACHE" ]] && find "$$CACHE" -name "*.gguf" -print -quit 2>/dev/null | grep -q .; then \ - MODEL=$$(find "$$CACHE" -name "*.gguf" -print -quit 2>/dev/null | xargs basename); \ - SIZE=$$(du -sh "$$CACHE" 2>/dev/null | cut -f1); \ - echo " Provider: builtin ($$MODEL)"; \ - echo " Status: $(GREEN)cached$(RESET) ($$SIZE)"; \ - else \ - echo " Provider: builtin (qwen2.5-0.5b-q4km)"; \ - echo " Status: $(YELLOW)not cached$(RESET)"; \ - echo " Fix: make download-models"; \ - fi - @echo "" - @echo "$(BOLD)Ollama:$(RESET)" - @if command -v ollama &>/dev/null && ollama list &>/dev/null 2>&1; then \ - echo " Status: $(GREEN)running$(RESET)"; \ - elif command -v ollama &>/dev/null; then \ - echo " Status: $(YELLOW)installed but not running$(RESET)"; \ - else \ - echo " Status: not installed (optional)"; \ - fi - @echo "" - @echo "$(BOLD)Cloud APIs:$(RESET)" - @for var in EMAILIBRIUM_OPENAI_API_KEY EMAILIBRIUM_ANTHROPIC_API_KEY EMAILIBRIUM_GEMINI_API_KEY; do \ - name=$$(echo "$$var" | sed 's/EMAILIBRIUM_//;s/_API_KEY//'); \ - if [[ -n "$${!var:-}" ]]; then \ - echo " $$name: $(GREEN)configured$(RESET)"; \ - else \ - echo " $$name: not configured"; \ - fi; \ - done - @echo "" - @echo "$(BOLD)Database:$(RESET)" - @if [[ -f "$(BACKEND_DIR)/emailibrium-dev.db" ]]; then \ - echo " Status: $(GREEN)exists$(RESET) ($$(du -sh $(BACKEND_DIR)/emailibrium-dev.db 2>/dev/null | cut -f1))"; \ - else \ - echo " Status: not created yet (created on first run)"; \ - fi - -# ============================================================================ -# Install & Build -# ============================================================================ - -.PHONY: install -install: ## Install all dependencies - @$(MAKE) -C $(BACKEND_DIR) build - @$(MAKE) -C $(FRONTEND_DIR) install - -.PHONY: build -build: ## Build everything - @$(MAKE) -C $(BACKEND_DIR) build - @$(MAKE) -C $(FRONTEND_DIR) build - -.PHONY: dev -dev: ## Start full stack dev servers (native, loads secrets/dev/ as env vars) - @echo "$(GREEN)Backend: http://localhost:8080 Frontend: http://localhost:3000$(RESET)" - @export EMAILIBRIUM_GOOGLE_CLIENT_ID="$$(cat secrets/dev/google_client_id 2>/dev/null)" \ - EMAILIBRIUM_GOOGLE_CLIENT_SECRET="$$(cat secrets/dev/google_client_secret 2>/dev/null)" \ - EMAILIBRIUM_MICROSOFT_CLIENT_ID="$$(cat secrets/dev/microsoft_client_id 2>/dev/null)" \ - EMAILIBRIUM_MICROSOFT_CLIENT_SECRET="$$(cat secrets/dev/microsoft_client_secret 2>/dev/null)" \ - JWT_SECRET="$$(cat secrets/dev/jwt_secret 2>/dev/null)" \ - EMAILIBRIUM_ENCRYPTION_MASTER_PASSWORD="$$(cat secrets/dev/oauth_encryption_key 2>/dev/null)" \ - RATE_LIMIT_PRESET=development; \ - trap 'kill 0' INT TERM EXIT; \ - $(MAKE) -C $(BACKEND_DIR) dev & \ - $(MAKE) -C $(FRONTEND_DIR) dev & \ - wait - -.PHONY: dev-llm -dev-llm: ## Start full stack with built-in LLM (downloads ~350MB model on first run) - @echo "$(GREEN)Backend (LLM): http://localhost:8080 Frontend: http://localhost:3000$(RESET)" - @export EMAILIBRIUM_GOOGLE_CLIENT_ID="$$(cat secrets/dev/google_client_id 2>/dev/null)" \ - EMAILIBRIUM_GOOGLE_CLIENT_SECRET="$$(cat secrets/dev/google_client_secret 2>/dev/null)" \ - EMAILIBRIUM_MICROSOFT_CLIENT_ID="$$(cat secrets/dev/microsoft_client_id 2>/dev/null)" \ - EMAILIBRIUM_MICROSOFT_CLIENT_SECRET="$$(cat secrets/dev/microsoft_client_secret 2>/dev/null)" \ - JWT_SECRET="$$(cat secrets/dev/jwt_secret 2>/dev/null)" \ - EMAILIBRIUM_ENCRYPTION_MASTER_PASSWORD="$$(cat secrets/dev/oauth_encryption_key 2>/dev/null)" \ - RATE_LIMIT_PRESET=development; \ - trap 'kill 0' INT TERM EXIT; \ - $(MAKE) -C $(BACKEND_DIR) dev-llm & \ - $(MAKE) -C $(FRONTEND_DIR) dev & \ - wait - -.PHONY: clean -clean: ## Clean all build artifacts - @$(MAKE) -C $(BACKEND_DIR) clean - @$(MAKE) -C $(FRONTEND_DIR) clean - -.PHONY: models -models: ## Show available LLM models with hardware recommendations - @$(MAKE) -C $(BACKEND_DIR) models - -.PHONY: embedding-models -embedding-models: ## Show available embedding models - @$(MAKE) -C $(BACKEND_DIR) embedding-models - -.PHONY: download-model -download-model: ## Download a model (e.g., make download-model MODEL=qwen3-8b-q4km) - @$(MAKE) -C $(BACKEND_DIR) download-model MODEL=$(MODEL) - -.PHONY: clean-data -clean-data: ## Remove all local data (DB, vectors) โ€” fresh start - @$(MAKE) -C $(BACKEND_DIR) clean-data - -.PHONY: clean-all -clean-all: ## Clean build artifacts + all local data - @$(MAKE) -C $(BACKEND_DIR) clean-all - @$(MAKE) -C $(FRONTEND_DIR) clean - -# ============================================================================ -# Test -# ============================================================================ - -.PHONY: test -test: ## Run all tests - @$(MAKE) -C $(BACKEND_DIR) test - @$(MAKE) -C $(FRONTEND_DIR) test - -# ============================================================================ -# Lint & Format -# ============================================================================ - -.PHONY: lint -lint: lint-docs ## Lint everything (code + docs) - @$(MAKE) -C $(BACKEND_DIR) lint - @$(MAKE) -C $(FRONTEND_DIR) lint - -.PHONY: format -format: format-docs ## Format everything (code + docs) - @$(MAKE) -C $(BACKEND_DIR) format - @$(MAKE) -C $(FRONTEND_DIR) format - -.PHONY: format-check -format-check: format-check-docs ## Check formatting (no changes) - @$(MAKE) -C $(BACKEND_DIR) format-check - @$(MAKE) -C $(FRONTEND_DIR) format-check - -.PHONY: typecheck -typecheck: ## Type check (frontend) - @$(MAKE) -C $(FRONTEND_DIR) typecheck - -# ============================================================================ -# Security & Quality -# ============================================================================ - -.PHONY: audit -audit: ## Security audit all dependencies - @$(MAKE) -C $(BACKEND_DIR) audit - @$(MAKE) -C $(FRONTEND_DIR) audit - -.PHONY: deadcode -deadcode: ## Check for dead code - @$(MAKE) -C $(BACKEND_DIR) deadcode - @$(MAKE) -C $(FRONTEND_DIR) deadcode - -.PHONY: ci -ci: format-check lint typecheck test ## Full CI pipeline - -.PHONY: ci-full -ci-full: ci links-check ## Full CI + link checking - -# ============================================================================ -# Dependency Management -# ============================================================================ - -.PHONY: upgrade -upgrade: ## Upgrade all dependencies (within semver) - @$(MAKE) -C $(BACKEND_DIR) upgrade - @$(MAKE) -C $(FRONTEND_DIR) upgrade - -.PHONY: outdated -outdated: ## Show outdated dependencies (no changes) - @$(MAKE) -C $(BACKEND_DIR) outdated - @$(MAKE) -C $(FRONTEND_DIR) outdated - -# ============================================================================ -# Documentation (Markdown, YAML, Links) -# ============================================================================ - -.PHONY: lint-md -lint-md: ## Lint Markdown files (strict โ€” fails on errors or missing tool) - @echo "$(GREEN)Linting Markdown...$(RESET)" - @command -v markdownlint-cli2 >/dev/null 2>&1 || { echo "$(RED)markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2$(RESET)"; exit 1; } - @markdownlint-cli2 '**/*.md' '#**/node_modules' '#**/target' '#.claude/worktrees/**' '#ruvector/**' - -.PHONY: lint-yaml -lint-yaml: ## Lint YAML files (strict โ€” fails on errors or missing tool) - @echo "$(GREEN)Linting YAML...$(RESET)" - @command -v yamllint >/dev/null 2>&1 || { echo "$(RED)yamllint not installed. Run: pip install yamllint$(RESET)"; exit 1; } - @find . \( -name node_modules -o -name target -o -name ruvector -o -name .claude -o -name .claude-flow \) -prune -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs -r yamllint -c .yamllint.yaml - -.PHONY: lint-docs -lint-docs: lint-md lint-yaml ## Lint all docs (Markdown + YAML) - -# find with -prune avoids traversing multi-GB Rust target/ and node_modules/ dirs -# (prettier's own glob walker enters all dirs before filtering via .prettierignore) -PRUNE_DIRS := \( -name node_modules -o -name target -o -name ruvector -o -name .claude \ - -o -name .claude-flow -o -name .git -o -name .agentic-qe -o -name .swarm \ - -o -name .git-rewrite -o -name dist -o -name storybook-static -o -name coverage \) -prune - -.PHONY: format-md -format-md: ## Format Markdown files - @find . $(PRUNE_DIRS) -o -name '*.md' -print | xargs npx prettier --write --no-error-on-unmatched-pattern - -.PHONY: format-yaml -format-yaml: ## Format YAML files - @find . $(PRUNE_DIRS) -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs npx prettier --write --no-error-on-unmatched-pattern - -.PHONY: format-docs -format-docs: format-md format-yaml ## Format docs (Markdown + YAML) - -.PHONY: format-check-md -format-check-md: - @find . $(PRUNE_DIRS) -o -name '*.md' -print | xargs npx prettier --check --no-error-on-unmatched-pattern - -.PHONY: format-check-yaml -format-check-yaml: - @find . $(PRUNE_DIRS) -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs npx prettier --check --no-error-on-unmatched-pattern - -.PHONY: format-check-docs -format-check-docs: format-check-md format-check-yaml - -.PHONY: links-check -links-check: ## Check internal links in Markdown - @echo "$(GREEN)Checking local file links...$(RESET)" - @if [ -n "$(LYCHEE)" ]; then \ - $(LYCHEE) --scheme file --include-fragments --config .lychee.toml '**/*.md'; \ - else \ - echo "$(YELLOW)lychee not installed. Run: cargo install lychee$(RESET)"; \ - fi - -.PHONY: links-check-external -links-check-external: ## Check external links (may take minutes) - @echo "$(GREEN)Checking external links...$(RESET)" - @if [ -n "$(LYCHEE)" ]; then \ - $(LYCHEE) --scheme https --scheme http --config .lychee.toml '**/*.md'; \ - else \ - echo "$(YELLOW)lychee not installed. Run: cargo install lychee$(RESET)"; \ - fi - -.PHONY: links-check-all -links-check-all: links-check links-check-external ## Check all links - -# ============================================================================ -# Docker -# ============================================================================ - -.PHONY: docker-up -docker-up: ## Start production stack - @echo "$(GREEN)Starting Emailibrium stack...$(RESET)" - @$(COMPOSE) up -d - @echo "$(GREEN)Backend: http://localhost:8080 Frontend: http://localhost:3000$(RESET)" - -.PHONY: docker-up-dev -docker-up-dev: ## Start dev stack (hot-reload) - @echo "$(GREEN)Starting Emailibrium dev stack...$(RESET)" - @$(COMPOSE_DEV) up -d - @echo "$(GREEN)Backend: http://localhost:8080 Frontend: http://localhost:3000$(RESET)" - -.PHONY: docker-down -docker-down: ## Stop and remove containers - @$(COMPOSE) down - -.PHONY: docker-down-volumes -docker-down-volumes: ## Stop + remove volumes (DESTROYS DATA) - @$(COMPOSE) down -v - -.PHONY: docker-restart -docker-restart: docker-down docker-up ## Restart all containers - -.PHONY: docker-build -docker-build: ## Build Docker images - @$(COMPOSE) build - -.PHONY: docker-build-no-cache -docker-build-no-cache: ## Build images without cache - @$(COMPOSE) build --no-cache - -.PHONY: docker-logs -docker-logs: ## Tail logs from all containers - @$(COMPOSE) logs -f - -.PHONY: docker-logs-backend -docker-logs-backend: ## Tail backend logs - @$(COMPOSE) logs -f backend - -.PHONY: docker-logs-frontend -docker-logs-frontend: ## Tail frontend logs - @$(COMPOSE) logs -f frontend - -.PHONY: docker-ps -docker-ps: ## Show running containers - @$(COMPOSE) ps - -.PHONY: docker-exec-backend -docker-exec-backend: ## Shell into backend container - @$(COMPOSE) exec backend sh - -.PHONY: docker-exec-frontend -docker-exec-frontend: ## Shell into frontend container - @$(COMPOSE) exec frontend sh - -.PHONY: docker-health -docker-health: ## Health check all containers - @$(COMPOSE) ps --format "table {{.Name}}\t{{.Status}}\t{{.Ports}}" - -.PHONY: docker-clean -docker-clean: ## Prune dangling Docker artifacts - @docker system prune -f --filter "label=com.docker.compose.project=emailibrium" 2>/dev/null || true - -.PHONY: docker-secrets -docker-secrets: ## Generate development secrets - @mkdir -p secrets/dev - @openssl rand -base64 32 > secrets/dev/jwt_secret - @openssl rand -base64 32 > secrets/dev/oauth_encryption_key - @echo "postgres://emailibrium:devpass@postgres:5432/emailibrium" > secrets/dev/database_url - @echo "devpass" > secrets/dev/db_password - @chmod 600 secrets/dev/* - @echo "$(GREEN)Secrets generated in secrets/dev/$(RESET)" - -# ============================================================================ -# Release -# ============================================================================ - -.PHONY: release-check -release-check: ci ## Pre-release CI validation - @echo "$(GREEN)Release checks passed. Ready to tag.$(RESET)" - -.PHONY: release-tag -release-tag: ## Tag a release (usage: make release-tag VERSION=0.1.0) - @if [ -z "$(VERSION)" ]; then echo "$(YELLOW)Usage: make release-tag VERSION=0.1.0$(RESET)"; exit 1; fi - @git tag -a "v$(VERSION)" -m "Release v$(VERSION)" - @echo "$(GREEN)Tagged v$(VERSION). Push with: git push origin v$(VERSION)$(RESET)" - -.PHONY: release-push -release-push: ## Push latest tag to trigger release workflow - @TAG=$$(git describe --tags --abbrev=0 2>/dev/null); \ - if [ -z "$$TAG" ]; then echo "$(YELLOW)No tags found.$(RESET)"; exit 1; fi; \ - echo "$(GREEN)Pushing $$TAG to origin...$(RESET)"; \ - git push origin "$$TAG" - -.PHONY: release -release: ## Cut a release (bumps versions, updates CHANGELOG, commits, tags, pushes). Usage: make release VERSION=0.1.0 - @[ -n "$(VERSION)" ] || (echo "usage: make release VERSION=X.Y.Z" >&2; exit 1) - @./scripts/release.sh $(VERSION) - -.PHONY: changelog -changelog: ## Regenerate CHANGELOG.md from git history using git-cliff - git-cliff --output CHANGELOG.md diff --git a/OPTIMIZATION_SPEC.md b/OPTIMIZATION_SPEC.md index 063383a..41663aa 100644 --- a/OPTIMIZATION_SPEC.md +++ b/OPTIMIZATION_SPEC.md @@ -125,8 +125,8 @@ `taiki-e/install-action`; add `.config/nextest.toml`. Each of clippy and test currently recompiles the full crate **and** the heavy ruvector path crates in a separate job/cache. - **definition_of_done:** - - grep: nextest archive in .github/workflows/** - - grep: archive-file in .github/workflows/** + - grep: nextest archive in .github/workflows/\*\* + - grep: archive-file in .github/workflows/\*\* - cmd: cargo test --manifest-path backend/Cargo.toml - **validation:** CI green; the test job no longer compiles the crate from scratch. **Removing the separate compile may expose a latent flaky test โ€” frame as exposed, not caused**, and fix @@ -157,7 +157,7 @@ - **deliverable:** Replace with `taiki-e/install-action` (prebuilt `cargo-audit`). - **definition_of_done:** - grep: taiki-e/install-action in .github/workflows/ci.yml - - grep:absent: cargo install --locked cargo-audit in .github/workflows/** + - grep:absent: cargo install --locked cargo-audit in .github/workflows/\*\* - **validation:** `rust-audit` job still runs `cargo audit` and stays green; job wall-clock drops. - **est_impact:** deterministic: removes a from-source tool compile per audit run. - **risk:** low @@ -171,8 +171,8 @@ the action to that version (or have it read the toolchain file) instead of installing bare `@stable`, which silently drifts and is overridden by the file inside `backend/` anyway. - **definition_of_done:** - - grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/** - - grep: 1.96 in .github/workflows/** + - grep:absent: dtolnay/rust-toolchain@stable in .github/workflows/\*\* + - grep: 1.96 in .github/workflows/\*\* - grep: rust-toolchain.toml in backend - **validation:** CI green; `rustc --version` in CI matches the pinned MSRV. - **est_impact:** correctness/maintenance โ€” no fmt/clippy version drift between CI and local. @@ -186,8 +186,8 @@ - **deliverable:** Install `lld` in the Rust jobs and set `-Clink-arg=-fuse-ld=lld` (via `RUSTFLAGS` or `backend/.cargo/config.toml`) to cut link time on incremental rebuilds. - **definition_of_done:** - - grep: fuse-ld=lld in .github/workflows/** backend/.cargo/config.toml - - grep: lld in .github/workflows/** + - grep: fuse-ld=lld in .github/workflows/\*\* backend/.cargo/config.toml + - grep: lld in .github/workflows/\*\* - **validation:** CI green; link phase faster on warm runs. - **est_impact:** medium/low โ€” link-time reduction on incremental builds. - **risk:** low @@ -202,7 +202,7 @@ to the free `ubuntu-24.04-arm`; the default keeps every other context safe. A nonexistent label queues forever, so this parameterization is what makes an upgrade safe. - **definition_of_done:** - - grep: vars.HEAVY_RUNNER in .github/workflows/** + - grep: vars.HEAVY_RUNNER in .github/workflows/\*\* - **validation:** CI green with the var unset (falls back to `ubuntu-latest`). - **est_impact:** low/medium โ€” opt-in bigger/native runner with a safe default. - **risk:** low @@ -225,7 +225,7 @@ ## N/A โ€” out of scope (do not plan) - **D1 native arm64 split โ€” N/A:** no QEMU/multi-arch build exists; Docker is single-arch amd64. - (The PUBLIC repo *could* use the free `ubuntu-24.04-arm`, but there is no arm64 build to accelerate. + (The PUBLIC repo _could_ use the free `ubuntu-24.04-arm`, but there is no arm64 build to accelerate. Revisit only if multi-arch publishing is added โ€” F-10 leaves the door open safely.) - **Larger x64 runners โ€” N/A:** personal account; Team/Enterprise only. - **A2 (sccache+Swatinem), A5 (double test run), B1 (disk hacks), C1 (concurrency) โ€” N/A:** not present diff --git a/QUICKSTART.md b/QUICKSTART.md index e94e944..2a36d7f 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -47,13 +47,13 @@ account's security settings. The onboarding screen includes presets for their se git clone https://github.com/pacphi/emailibrium.git cd emailibrium -make setup # interactive wizard โ€” SKIP the OAuth credential prompts (press Enter) -make install -make dev # โ†’ Backend: http://localhost:8080 Frontend: http://localhost:3000 +just setup # interactive wizard โ€” SKIP the OAuth credential prompts (press Enter) +just install +just dev # โ†’ Backend: http://localhost:8080 Frontend: http://localhost:3000 ``` OAuth is optional โ€” pressing Enter at the Google/Microsoft prompts writes placeholders and the -backend still boots normally. (Docker alternative: `make setup-secrets` then `make docker-up-dev`.) +backend still boots normally. (Docker alternative: `just setup-secrets` then `just docker-up-dev`.) ### 3. Connect your mailbox (โ‰ˆ2 min) diff --git a/README.md b/README.md index a29435b..f49acda 100644 --- a/README.md +++ b/README.md @@ -101,16 +101,16 @@ git clone https://github.com/pacphi/emailibrium.git cd emailibrium # Guided setup (recommended for first time) -make setup # interactive wizard: prerequisites, secrets, AI, Docker +just setup # interactive wizard: prerequisites, secrets, AI, Docker # Option A: Native -make install -make dev +just install +just dev # โ†’ Backend: http://localhost:8080 Frontend: http://localhost:3000 # Option B: Docker -make setup-secrets # generate dev secrets (first time only) -make docker-up-dev # start with hot-reload +just setup-secrets # generate dev secrets (first time only) +just docker-up-dev # start with hot-reload ``` **Prerequisites:** Rust 1.97+, Node.js 26 (LTS)+, pnpm 11.5+ โ€” or just Docker. See [Setup Guide](docs/setup-guide.md) for details. @@ -158,13 +158,13 @@ Axum process; there is no second port and no separate daemon. 15 read-only tools, grouped by what they touch: -| Area | Tools | -| --------- | ------------------------------------------------------------------------------------------------------------ | -| Email | `search_emails`, `get_email`, `list_recent_emails`, `count_emails`, `get_email_thread`, `find_similar_emails`, `list_attachments` | -| Insights | `get_insights`, `list_subscriptions`, `list_clusters`, `get_learning_metrics` | -| Accounts | `list_accounts`, `get_sync_status` | -| Rules | `list_rules` | -| Cleanup | `preview_cleanup_plan` | +| Area | Tools | +| -------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Email | `search_emails`, `get_email`, `list_recent_emails`, `count_emails`, `get_email_thread`, `find_similar_emails`, `list_attachments` | +| Insights | `get_insights`, `list_subscriptions`, `list_clusters`, `get_learning_metrics` | +| Accounts | `list_accounts`, `get_sync_status` | +| Rules | `list_rules` | +| Cleanup | `preview_cleanup_plan` | Alongside the tools, three resources (`email://{id}`, `thread://{key}`, `insights://summary`) expose stable read-only views, and two prompts (`triage-inbox`, `weekly-report`) package the @@ -235,12 +235,12 @@ See all DDDs in [docs/DDDs](https://github.com/pacphi/emailibrium/tree/main/docs ## ๐Ÿ› ๏ธ Development ```bash -make help # see all available targets -make ci # format-check + lint + typecheck + test -make test # backend (Rust) + frontend (Vitest) -make docker-up-dev # full stack with hot-reload -make upgrade # upgrade all dependencies -make outdated # check what's stale +just --list # see all available targets +just ci # format-check + lint + typecheck + test +just test # backend (Rust) + frontend (Vitest) +just docker-up-dev # full stack with hot-reload +just upgrade # upgrade all dependencies +just outdated # check what's stale ``` See the [Maintainer Guide](docs/maintainer-guide.md) for the full developer experience. diff --git a/backend/Makefile b/backend/Makefile deleted file mode 100644 index cffe6da..0000000 --- a/backend/Makefile +++ /dev/null @@ -1,283 +0,0 @@ -# ============================================================================ -# Emailibrium Backend โ€” Rust (Cargo) -# ============================================================================ -# -# Quick Start: -# make build - Build the backend -# make test - Run all tests -# make dev - Start dev server -# make bench - Run benchmarks -# ============================================================================ - -SHELL := /bin/bash -CARGO := cargo -export CARGO_INCREMENTAL := 1 - -BOLD := $(shell tput bold 2>/dev/null || echo '') -GREEN := $(shell tput setaf 2 2>/dev/null || echo '') -YELLOW := $(shell tput setaf 3 2>/dev/null || echo '') -BLUE := $(shell tput setaf 4 2>/dev/null || echo '') -RED := $(shell tput setaf 1 2>/dev/null || echo '') -RESET := $(shell tput sgr0 2>/dev/null || echo '') - -.DEFAULT_GOAL := help - -# ============================================================================ -# Default Target -# ============================================================================ - -.PHONY: help -help: - @echo "$(BOLD)$(BLUE)โ•โ•โ• Emailibrium Backend (Rust) โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo "" - @echo "$(BOLD)Build & Run$(RESET)" - @echo " build - Build backend (debug)" - @echo " build-release - Build backend (release, optimized)" - @echo " dev - Start dev server" - @echo " clean - Clean build artifacts" - @echo "" - @echo "$(BOLD)Models$(RESET)" - @echo " models - Show available LLM models" - @echo " embedding-models - Show available embedding models" - @echo " download-model - Download a model (MODEL=)" - @echo "" - @echo "$(BOLD)Test$(RESET)" - @echo " test - Run all tests" - @echo " test-lib - Run library tests only" - @echo " test-integration - Run integration tests only" - @echo " bench - Run benchmarks" - @echo "" - @echo "$(BOLD)Lint & Format$(RESET)" - @echo " lint - Run clippy" - @echo " format - Format code (cargo fmt)" - @echo " format-check - Check formatting" - @echo " deadcode - Check for dead code" - @echo "" - @echo "$(BOLD)Security$(RESET)" - @echo " audit - Security audit (cargo-audit)" - @echo "" - @echo "$(BOLD)Dependencies$(RESET)" - @echo " upgrade - Upgrade deps (within semver)" - @echo " outdated - Show outdated crates" - @echo "" - @echo "$(BOLD)Documentation$(RESET)" - @echo " lint-docs - Lint Markdown + YAML in backend/" - @echo " format-docs - Format Markdown + YAML in backend/" - -# ============================================================================ -# Build & Run -# ============================================================================ - -.PHONY: build -build: ## Build backend (debug) - @$(CARGO) build -p emailibrium - -.PHONY: build-release -build-release: ## Build backend (release, optimized) - @$(CARGO) build --release -p emailibrium - -.PHONY: dev -dev: ## Start dev server - @RATE_LIMIT_PRESET=development $(CARGO) run - -.PHONY: check-llm-deps -check-llm-deps: ## Check that cmake is installed (required for llama.cpp compilation) - @command -v cmake >/dev/null 2>&1 || { \ - echo "$(YELLOW)ERROR: cmake is required to build the built-in LLM.$(RESET)"; \ - echo " Install with: brew install cmake"; \ - exit 1; \ - } - @echo "$(GREEN)cmake found: $$(cmake --version | head -1)$(RESET)" - -.PHONY: dev-llm -dev-llm: check-llm-deps ## Start dev server with built-in LLM (downloads model on first run) - @$(CARGO) run --features builtin-llm - -.PHONY: build-llm -build-llm: check-llm-deps ## Build backend with built-in LLM support - @$(CARGO) build --features builtin-llm - -.PHONY: clean -clean: ## Clean build artifacts - @$(CARGO) clean - -.PHONY: models -models: ## Show available LLM models (reads from config/models-llm.yaml) - @echo "$(BOLD)$(BLUE)โ•โ•โ• Available LLM Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo "" - @RAM=$$(sysctl -n hw.memsize 2>/dev/null || echo $$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $$2 * 1024}') 2>/dev/null || echo 8589934592); \ - RAM_GB=$$(( $$RAM / 1073741824 )); \ - AVAIL_MB=$$(( $$RAM / 1048576 - 4096 )); \ - echo " $(BOLD)System:$(RESET) $${RAM_GB}GB RAM ($$(( $$AVAIL_MB / 1024 ))GB available for models)"; \ - echo ""; \ - awk -v avail="$$AVAIL_MB" -v G="$$(tput setaf 2 2>/dev/null)" -v Y="$$(tput setaf 3 2>/dev/null)" -v B="$$(tput bold 2>/dev/null)" -v R="$$(tput sgr0 2>/dev/null)" ' \ - /^ [a-z][a-z_]*:$$/ { \ - prov=$$1; gsub(/:/, "", prov); \ - label = (prov == "builtin") ? "builtin (onnx)" : prov; \ - if (n++) printf "\n"; \ - printf " %s%s%s\n", B, label, R; \ - printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; \ - printf " %s%-30s %-10s %-8s %-8s %7s %s%s\n", B, "MODEL ID", "PARAMS", "DISK", "RAM", "CTX", "QUALITY", R; \ - printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; \ - } \ - /^ - id:/ { id=$$3 } \ - /^ name:/ { $$1=""; name=$$0; gsub(/^ *"?|"? *$$/,"",name) } \ - /^ params:/ { params=$$2; gsub(/"/,"",params) } \ - /^ context_size:/ { ctx=$$2 } \ - /^ disk_mb:/ { disk=$$2 } \ - /^ min_ram_mb:/ { ram=$$2 } \ - /^ quality:/ { q=$$2; \ - if (ram+0 <= avail+0 || ram+0 == 0) c=G; else c=Y; \ - ds = (disk+0 >= 1000) ? sprintf("%.1fGB", disk/1000) : sprintf("%dMB", disk); \ - rs = (ram+0 >= 1000) ? sprintf("%.1fGB", ram/1000) : sprintf("%dMB", ram); \ - printf " %s%-30s %-10s %-8s %-8s %7d %s%s\n", c, id, params, ds, rs, ctx, q, R; \ - id=""; name=""; params=""; ctx=0; disk=0; ram=0; q="" \ - }' ../config/models-llm.yaml 2>/dev/null || echo " $(YELLOW)Cannot read config/models-llm.yaml$(RESET)"; \ - echo ""; \ - echo " $(GREEN)Green$(RESET) = fits your hardware $(YELLOW)Yellow$(RESET) = may not fit"; \ - echo ""; \ - echo " $(BOLD)Usage:$(RESET)"; \ - echo " make download-model MODEL= Pre-download a model"; \ - echo " Edit config/models-llm.yaml Add or modify models"; \ - echo "" - -.PHONY: embedding-models -embedding-models: ## Show available embedding models (reads from config/models-embedding.yaml) - @echo "$(BOLD)$(BLUE)โ•โ•โ• Available Embedding Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo "" - @awk -v G="$$(tput setaf 2 2>/dev/null)" -v B="$$(tput bold 2>/dev/null)" -v R="$$(tput sgr0 2>/dev/null)" ' \ - /^ [a-z][a-z_]*:$$/ { \ - prov=$$1; gsub(/:/, "", prov); \ - label = (prov == "onnx") ? "builtin (onnx)" : prov; \ - if (n++) printf "\n"; \ - printf " %s%s%s\n", B, label, R; \ - printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; \ - printf " %s%-30s %-6s %-10s %-8s %-8s %s%s\n", B, "MODEL ID", "DIMS", "MAX TOKENS", "DISK", "RAM", "QUALITY", R; \ - printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; \ - } \ - /^ - id:/ { id=$$3 } \ - /^ dimensions:/ { dims=$$2 } \ - /^ max_tokens:/ { maxtok=$$2 } \ - /^ disk_mb:/ { disk=$$2 } \ - /^ min_ram_mb:/ { ram=$$2 } \ - /^ quality:/ { q=$$2; \ - ds = (disk+0 >= 1000) ? sprintf("%.1fGB", disk/1000) : sprintf("%dMB", disk); \ - rs = (ram+0 >= 1000) ? sprintf("%.1fGB", ram/1000) : sprintf("%dMB", ram); \ - printf " %s%-30s %-6d %-10d %-8s %-8s %s%s\n", G, id, dims, maxtok, ds, rs, q, R; \ - id=""; dims=0; maxtok=0; disk=0; ram=0; q="" \ - }' ../config/models-embedding.yaml 2>/dev/null || echo " $(YELLOW)Cannot read config/models-embedding.yaml$(RESET)"; \ - echo ""; \ - echo " $(BOLD)Usage:$(RESET)"; \ - echo " Edit config/models-embedding.yaml Add or modify embedding models"; \ - echo "" - -.PHONY: download-model -download-model: ## Download a model by ID (e.g., make download-model MODEL=qwen3-8b-q4km) - @if [ -z "$(MODEL)" ]; then \ - echo "$(BOLD)Usage:$(RESET) make download-model MODEL="; \ - echo ""; \ - echo "$(BOLD)Examples:$(RESET)"; \ - echo " make download-model MODEL=qwen3-8b-q4km # GGUF via Hugging Face"; \ - echo " make download-model MODEL=qwen3:8b # via Ollama"; \ - echo ""; \ - echo "Run '$(BOLD)make models$(RESET)' to see available models."; \ - exit 1; \ - fi - @$(CARGO) run --features builtin-llm -- --download-model $(MODEL) - -.PHONY: clean-data -clean-data: ## Remove all local data (DB, vectors, sidecars) โ€” fresh start - @echo "$(YELLOW)Removing local data...$(RESET)" - @rm -f emailibrium.db emailibrium.db-wal emailibrium.db-shm - @rm -f emailibrium-dev.db "emailibrium 2.db" - @rm -f data/emailibrium.db - @rm -rf data/vectors/*/vectors.db data/vectors/*/documents.json - @echo "$(GREEN)Local data removed. Next 'make dev' will start fresh.$(RESET)" - @echo " $(YELLOW)Note: .fastembed_cache/ (ONNX models) preserved to avoid re-download.$(RESET)" - @echo " To also remove cached models: rm -rf .fastembed_cache/" - -.PHONY: clean-all -clean-all: clean clean-data ## Clean build artifacts + all local data - @echo "$(GREEN)Full clean complete.$(RESET)" - -# ============================================================================ -# Test -# ============================================================================ - -.PHONY: test -test: ## Run all tests (emailibrium only) - @$(CARGO) test -p emailibrium - -.PHONY: test-lib -test-lib: ## Run library tests only - @$(CARGO) test --lib - -.PHONY: test-integration -test-integration: ## Run integration tests only - @$(CARGO) test --test '*' - -.PHONY: bench -bench: ## Run benchmarks - @$(CARGO) bench - -# ============================================================================ -# Lint & Format -# ============================================================================ - -.PHONY: lint -lint: ## Run clippy (strict โ€” fails on warnings; dead-code allowed) - @$(CARGO) clippy -p emailibrium --all-targets -- \ - -D warnings \ - -A dead_code -A unused_variables -A unused_imports -A unused_mut - -.PHONY: format -format: ## Format code - @$(CARGO) fmt - -.PHONY: format-check -format-check: ## Check formatting - @$(CARGO) fmt -- --check - -.PHONY: deadcode -deadcode: ## Check for dead code - @$(CARGO) clippy -- -W dead_code 2>&1 | grep "warning.*never" | sort | uniq -c | sort -rn - -# ============================================================================ -# Security -# ============================================================================ - -.PHONY: audit -audit: ## Security audit dependencies - @$(CARGO) audit 2>/dev/null || echo "$(YELLOW)cargo-audit not installed. Run: cargo install cargo-audit$(RESET)" - -# ============================================================================ -# Dependency Management -# ============================================================================ - -.PHONY: upgrade -upgrade: ## Upgrade dependencies (within semver) - @$(CARGO) update - @echo "$(GREEN)Cargo.lock updated.$(RESET)" - -.PHONY: outdated -outdated: ## Show outdated crates - @$(CARGO) outdated --root-deps-only 2>/dev/null || echo "$(YELLOW)Install with: cargo install cargo-outdated$(RESET)" - -# ============================================================================ -# Documentation -# ============================================================================ - -.PHONY: lint-docs -lint-docs: ## Lint Markdown + YAML in backend/ (strict) - @command -v markdownlint-cli2 >/dev/null 2>&1 || { echo "$(RED)markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2$(RESET)"; exit 1; } - @command -v yamllint >/dev/null 2>&1 || { echo "$(RED)yamllint not installed. Run: pip install yamllint$(RESET)"; exit 1; } - @markdownlint-cli2 '**/*.md' '#target' - @yamllint -c ../.yamllint.yaml . - -.PHONY: format-docs -format-docs: ## Format Markdown + YAML in backend/ - @npx --no-install prettier --write '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc - -.PHONY: format-check-docs -format-check-docs: ## Check Markdown + YAML formatting - @npx --no-install prettier --check '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc diff --git a/backend/justfile b/backend/justfile new file mode 100644 index 0000000..d9740d8 --- /dev/null +++ b/backend/justfile @@ -0,0 +1,353 @@ +# ============================================================================ +# Emailibrium Backend โ€” Rust (Cargo) +# ============================================================================ +# +# Quick Start: +# just build - Build the backend +# just test - Run all tests +# just dev - Start dev server +# just bench - Run benchmarks +# +# just --list - Show every recipe, grouped by category +# ============================================================================ + +set shell := ["bash", "-cu"] + +CARGO := "cargo" +export CARGO_INCREMENTAL := "1" + +# Model id for `download-model`. Also settable as `just MODEL= download-model`. +MODEL := "" + +BOLD := `tput bold 2>/dev/null || echo ''` +GREEN := `tput setaf 2 2>/dev/null || echo ''` +YELLOW := `tput setaf 3 2>/dev/null || echo ''` +BLUE := `tput setaf 4 2>/dev/null || echo ''` +RED := `tput setaf 1 2>/dev/null || echo ''` +RESET := `tput sgr0 2>/dev/null || echo ''` + +# ============================================================================ +# Default Recipe (just runs the first recipe when given no arguments) +# ============================================================================ + +# Show this help +[group('Help')] +@help: + echo "{{BOLD}}{{BLUE}}โ•โ•โ• Emailibrium Backend (Rust) โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}}" + echo "" + echo "{{BOLD}}Build & Run{{RESET}}" + echo " build - Build backend (debug)" + echo " build-release - Build backend (release, optimized)" + echo " dev - Start dev server" + echo " clean - Clean build artifacts" + echo "" + echo "{{BOLD}}Models{{RESET}}" + echo " models - Show available LLM models" + echo " embedding-models - Show available embedding models" + echo " download-model - Download a model (MODEL=)" + echo "" + echo "{{BOLD}}Test{{RESET}}" + echo " test - Run all tests" + echo " test-lib - Run library tests only" + echo " test-integration - Run integration tests only" + echo " bench - Run benchmarks" + echo "" + echo "{{BOLD}}Lint & Format{{RESET}}" + echo " lint - Run clippy" + echo " format - Format code (cargo fmt)" + echo " format-check - Check formatting" + echo " deadcode - Check for dead code" + echo "" + echo "{{BOLD}}Security{{RESET}}" + echo " audit - Security audit (cargo-audit)" + echo "" + echo "{{BOLD}}Dependencies{{RESET}}" + echo " upgrade - Upgrade deps (within semver)" + echo " outdated - Show outdated crates" + echo "" + echo "{{BOLD}}Documentation{{RESET}}" + echo " lint-docs - Lint Markdown + YAML in backend/" + echo " format-docs - Format Markdown + YAML in backend/" + +# ============================================================================ +# Build & Run +# ============================================================================ + +# Build backend (debug) +[group('Build & Run')] +@build: + {{CARGO}} build -p emailibrium + +# Build backend (release, optimized) +[group('Build & Run')] +@build-release: + {{CARGO}} build --release -p emailibrium + +# Start dev server +[group('Build & Run')] +@dev: + RATE_LIMIT_PRESET=development {{CARGO}} run + +# Check that cmake is installed (required for llama.cpp compilation) +[group('Build & Run')] +check-llm-deps: + #!/usr/bin/env bash + set -euo pipefail + if ! command -v cmake >/dev/null 2>&1; then + echo "{{YELLOW}}ERROR: cmake is required to build the built-in LLM.{{RESET}}" >&2 + echo " Install with: brew install cmake" >&2 + exit 1 + fi + echo "{{GREEN}}cmake found: $(cmake --version | head -1){{RESET}}" + +# Start dev server with built-in LLM (downloads model on first run) +[group('Build & Run')] +@dev-llm: check-llm-deps + {{CARGO}} run --features builtin-llm + +# Build backend with built-in LLM support +[group('Build & Run')] +@build-llm: check-llm-deps + {{CARGO}} build --features builtin-llm + +# Clean build artifacts +[group('Build & Run')] +@clean: + {{CARGO}} clean + +# Remove all local data (DB, vectors, sidecars) โ€” fresh start +[group('Build & Run')] +@clean-data: + echo "{{YELLOW}}Removing local data...{{RESET}}" + rm -f emailibrium.db emailibrium.db-wal emailibrium.db-shm + rm -f emailibrium-dev.db "emailibrium 2.db" + rm -f data/emailibrium.db + rm -rf data/vectors/*/vectors.db data/vectors/*/documents.json + echo "{{GREEN}}Local data removed. Next 'just dev' will start fresh.{{RESET}}" + echo " {{YELLOW}}Note: .fastembed_cache/ (ONNX models) preserved to avoid re-download.{{RESET}}" + echo " To also remove cached models: rm -rf .fastembed_cache/" + +# Clean build artifacts + all local data +[group('Build & Run')] +@clean-all: clean clean-data + echo "{{GREEN}}Full clean complete.{{RESET}}" + +# ============================================================================ +# Models +# ============================================================================ + +# Show available LLM models (reads from config/models-llm.yaml) +[group('Models')] +models: + #!/usr/bin/env bash + set -euo pipefail + echo "{{BOLD}}{{BLUE}}โ•โ•โ• Available LLM Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}}" + echo "" + RAM="$(sysctl -n hw.memsize 2>/dev/null || echo "$(grep MemTotal /proc/meminfo 2>/dev/null | awk '{print $2 * 1024}')" || echo 8589934592)" + case "$RAM" in ''|*[!0-9]*) RAM=8589934592 ;; esac + RAM_GB=$(( RAM / 1073741824 )) + AVAIL_MB=$(( RAM / 1048576 - 4096 )) + echo " {{BOLD}}System:{{RESET}} ${RAM_GB}GB RAM ($(( AVAIL_MB / 1024 ))GB available for models)" + echo "" + awk -v avail="$AVAIL_MB" -v G='{{GREEN}}' -v Y='{{YELLOW}}' -v B='{{BOLD}}' -v R='{{RESET}}' ' + /^ [a-z][a-z_]*:$/ { + prov=$1; gsub(/:/, "", prov); + label = (prov == "builtin") ? "builtin (onnx)" : prov; + if (n++) printf "\n"; + printf " %s%s%s\n", B, label, R; + printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; + printf " %s%-30s %-10s %-8s %-8s %7s %s%s\n", B, "MODEL ID", "PARAMS", "DISK", "RAM", "CTX", "QUALITY", R; + printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; + } + /^ - id:/ { id=$3 } + /^ name:/ { $1=""; name=$0; gsub(/^ *"?|"? *$/,"",name) } + /^ params:/ { params=$2; gsub(/"/,"",params) } + /^ context_size:/ { ctx=$2 } + /^ disk_mb:/ { disk=$2 } + /^ min_ram_mb:/ { ram=$2 } + /^ quality:/ { q=$2; + if (ram+0 <= avail+0 || ram+0 == 0) c=G; else c=Y; + ds = (disk+0 >= 1000) ? sprintf("%.1fGB", disk/1000) : sprintf("%dMB", disk); + rs = (ram+0 >= 1000) ? sprintf("%.1fGB", ram/1000) : sprintf("%dMB", ram); + printf " %s%-30s %-10s %-8s %-8s %7d %s%s\n", c, id, params, ds, rs, ctx, q, R; + id=""; name=""; params=""; ctx=0; disk=0; ram=0; q="" + }' ../config/models-llm.yaml 2>/dev/null || echo " {{YELLOW}}Cannot read config/models-llm.yaml{{RESET}}" + echo "" + echo " {{GREEN}}Green{{RESET}} = fits your hardware {{YELLOW}}Yellow{{RESET}} = may not fit" + echo "" + echo " {{BOLD}}Usage:{{RESET}}" + echo " just download-model MODEL= Pre-download a model" + echo " Edit config/models-llm.yaml Add or modify models" + echo "" + +# Show available embedding models (reads from config/models-embedding.yaml) +[group('Models')] +embedding-models: + #!/usr/bin/env bash + set -euo pipefail + echo "{{BOLD}}{{BLUE}}โ•โ•โ• Available Embedding Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}}" + echo "" + awk -v G='{{GREEN}}' -v B='{{BOLD}}' -v R='{{RESET}}' ' + /^ [a-z][a-z_]*:$/ { + prov=$1; gsub(/:/, "", prov); + label = (prov == "onnx") ? "builtin (onnx)" : prov; + if (n++) printf "\n"; + printf " %s%s%s\n", B, label, R; + printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; + printf " %s%-30s %-6s %-10s %-8s %-8s %s%s\n", B, "MODEL ID", "DIMS", "MAX TOKENS", "DISK", "RAM", "QUALITY", R; + printf " โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€\n"; + } + /^ - id:/ { id=$3 } + /^ dimensions:/ { dims=$2 } + /^ max_tokens:/ { maxtok=$2 } + /^ disk_mb:/ { disk=$2 } + /^ min_ram_mb:/ { ram=$2 } + /^ quality:/ { q=$2; + ds = (disk+0 >= 1000) ? sprintf("%.1fGB", disk/1000) : sprintf("%dMB", disk); + rs = (ram+0 >= 1000) ? sprintf("%.1fGB", ram/1000) : sprintf("%dMB", ram); + printf " %s%-30s %-6d %-10d %-8s %-8s %s%s\n", G, id, dims, maxtok, ds, rs, q, R; + id=""; dims=0; maxtok=0; disk=0; ram=0; q="" + }' ../config/models-embedding.yaml 2>/dev/null || echo " {{YELLOW}}Cannot read config/models-embedding.yaml{{RESET}}" + echo "" + echo " {{BOLD}}Usage:{{RESET}}" + echo " Edit config/models-embedding.yaml Add or modify embedding models" + echo "" + +# Download a model by ID (e.g. `just download-model MODEL=qwen3-8b-q4km`) +[group('Models')] +download-model model=MODEL: + #!/usr/bin/env bash + set -euo pipefail + # Accepts a bare id (`just download-model qwen3-8b-q4km`), the Makefile-style + # `MODEL=` form, and the just-native `just MODEL= download-model`. + model="{{model}}" + model="${model#MODEL=}" + if [ -z "$model" ]; then + echo "{{BOLD}}Usage:{{RESET}} just download-model MODEL=" >&2 + echo "" >&2 + echo "{{BOLD}}Examples:{{RESET}}" >&2 + echo " just download-model MODEL=qwen3-8b-q4km # GGUF via Hugging Face" >&2 + echo " just download-model MODEL=qwen3:8b # via Ollama" >&2 + echo "" >&2 + echo "Run '{{BOLD}}just models{{RESET}}' to see available models." >&2 + exit 1 + fi + {{CARGO}} run --features builtin-llm -- --download-model "$model" + +# ============================================================================ +# Test +# ============================================================================ + +# Run all tests (emailibrium only) +[group('Test')] +@test: + {{CARGO}} test -p emailibrium + +# Run library tests only +[group('Test')] +@test-lib: + {{CARGO}} test --lib + +# Run integration tests only +[group('Test')] +@test-integration: + {{CARGO}} test --test '*' + +# Run benchmarks +[group('Test')] +@bench: + {{CARGO}} bench + +# ============================================================================ +# Lint & Format +# ============================================================================ + +# Run clippy (strict โ€” fails on warnings; dead-code allowed) +[group('Lint & Format')] +@lint: + {{CARGO}} clippy -p emailibrium --all-targets -- \ + -D warnings \ + -A dead_code -A unused_variables -A unused_imports -A unused_mut + +# Format code +[group('Lint & Format')] +@format: + {{CARGO}} fmt + +# Check formatting +[group('Lint & Format')] +@format-check: + {{CARGO}} fmt -- --check + +# NOTE: deliberately linewise (no `pipefail`) so an empty grep result โ€” i.e. no +# dead code found โ€” exits 0, exactly as the Makefile did. +# Check for dead code +[group('Lint & Format')] +@deadcode: + {{CARGO}} clippy -- -W dead_code 2>&1 | grep "warning.*never" | sort | uniq -c | sort -rn + +# ============================================================================ +# Security +# ============================================================================ + +# Security audit dependencies (fails on findings) +[group('Security')] +audit: + #!/usr/bin/env bash + set -euo pipefail + if ! command -v cargo-audit >/dev/null 2>&1; then + echo "{{RED}}cargo-audit not installed. Run: cargo install cargo-audit{{RESET}}" >&2 + exit 1 + fi + {{CARGO}} audit + +# ============================================================================ +# Dependency Management +# ============================================================================ + +# Upgrade dependencies (within semver) +[group('Dependencies')] +@upgrade: + {{CARGO}} update + echo "{{GREEN}}Cargo.lock updated.{{RESET}}" + +# Show outdated crates +[group('Dependencies')] +outdated: + #!/usr/bin/env bash + set -euo pipefail + if ! command -v cargo-outdated >/dev/null 2>&1; then + echo "{{RED}}cargo-outdated not installed. Run: cargo install cargo-outdated{{RESET}}" >&2 + exit 1 + fi + {{CARGO}} outdated --root-deps-only + +# ============================================================================ +# Documentation +# ============================================================================ + +# Lint Markdown + YAML in backend/ (strict) +[group('Documentation')] +lint-docs: + #!/usr/bin/env bash + set -euo pipefail + if ! command -v markdownlint-cli2 >/dev/null 2>&1; then + echo "{{RED}}markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2{{RESET}}" >&2 + exit 1 + fi + if ! command -v yamllint >/dev/null 2>&1; then + echo "{{RED}}yamllint not installed. Run: pip install yamllint{{RESET}}" >&2 + exit 1 + fi + markdownlint-cli2 '**/*.md' '#target' + yamllint -c ../.yamllint.yaml . + +# Format Markdown + YAML in backend/ +[group('Documentation')] +@format-docs: + npx --no-install prettier --write '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc + +# Check Markdown + YAML formatting +[group('Documentation')] +@format-check-docs: + npx --no-install prettier --check '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc diff --git a/docs/ADRs/ADR-032-make-to-just-task-runner.md b/docs/ADRs/ADR-032-make-to-just-task-runner.md new file mode 100644 index 0000000..f701c0f --- /dev/null +++ b/docs/ADRs/ADR-032-make-to-just-task-runner.md @@ -0,0 +1,81 @@ +# ADR-032: Replace Make with Just as the Task Runner + +- **Status:** Accepted +- **Date:** 2026-07-31 +- **Deciders:** Chris Phillipson +- **Context:** The repository drove every developer and CI workflow through three GNU Makefiles (root, `backend/`, `frontend/` โ€” 110 targets, ~987 lines combined). Make is a _build system_ being used purely as a _task runner_: none of these targets declare real file dependencies, every one is `.PHONY`, and nothing relies on Make's timestamp-based rebuild logic. That mismatch cost real correctness, not just elegance โ€” see ยง2. + +--- + +## 1. Problem Statement + +Make is doing a job it was not designed for here, and the mismatch had already produced shipping bugs. + +- **Everything is `.PHONY`.** All 110 targets are task aliases; not one expresses a fileโ†’file dependency. Make's core value โ€” incremental rebuild from timestamps โ€” is unused, while its footguns remain. +- **Silent error swallowing was endemic and invisible.** Four targets masked failure exit codes. All were found while wiring an automated quality gate โ€” the gate is only as trustworthy as the commands it runs, and several of the commands it depended on could never fail: + - `frontend/Makefile: test: @$(TURBO) test || true` โ€” reported success while Vitest failed. + - `frontend/Makefile: audit: @$(PNPM) audit --prod 2>/dev/null || echo "..."` โ€” a real vulnerability finding exited 0. + - `backend/Makefile: audit: @$(CARGO) audit 2>/dev/null || echo "..."` โ€” same defect. + - `Makefile: download-models` โ€” `cargo run ... 2>/dev/null || echo "Backend not built"`, exiting 0 on a genuine failure. + + **Timeline, for accuracy:** the two `frontend/Makefile` instances were fixed in place _before_ this migration (they were blocking the gate and could not wait). The `backend/Makefile: audit` swallow โ€” tracked as `pl-mk-audit-swallow` โ€” and the root `download-models` swallow are fixed _by_ this migration. So at the moment the Makefiles were deleted, two of the four were already patched; the pattern is listed in full because the recurrence across three separate files over time is the actual argument, not any single instance. + +- **Shell-per-line semantics are implicit.** Make runs each recipe line in its own shell unless `.ONESHELL` is set, which is easy to violate accidentally in multi-line logic. +- **Recursive `$(MAKE) -C` delegation** obscures which sub-command actually failed. +- **Tab-vs-space significance** remains a recurring paper cut with no upside. + +## 2. Decision + +**Adopt [`just`](https://github.com/casey/just) as the sole task runner. Delete all three Makefiles.** + +Three justfiles mirror the previous structure โ€” root `justfile` delegating to `backend/justfile` and `frontend/justfile` โ€” and recipe names are preserved verbatim (`build`, `test`, `lint`, `format-check`, `audit`, `ci`, `docker-up`, โ€ฆ) so existing muscle memory, documentation, and any external scripts need minimal change. + +### Why `just` specifically + +| Property | Make | Just | +| ------------------ | -------------------------------------- | ---------------------------------------------------- | +| Purpose | build system (file DAG) | task runner | +| Phony declarations | required for every target | not a concept | +| Recipe listing | hand-written `help` target that drifts | `just --list`, generated from doc comments | +| Multi-line shell | per-line shells unless `.ONESHELL` | explicit shebang recipes with `set -euo pipefail` | +| Arguments | awkward (`$(filter-out ...)`) | first-class recipe parameters | +| Leading whitespace | tabs significant | insignificant | +| Failure semantics | easy to mask accidentally | same shell rules, but recipes are short and explicit | + +### What changed beyond a mechanical port + +- **The swallowed exit codes are fixed** (see the ยง1 timeline for which were patched before vs by this change). `just test`, `just audit` and `just download-models` now fail when the underlying tool fails. `backend/justfile`'s `outdated` got the same treatment, since it carried an identical swallow. +- **Multi-line recipes are explicit.** Anything with `if`/`for`/shared variables became a shebang recipe with `set -euo pipefail`, rather than relying on implicit per-line shells. +- **Deliberate `|| true` survives, with a comment.** A small number of genuinely informational recipes (`outdated`, docker prune-style cleanup) keep a tolerant exit and now say _why_ in a comment, so the distinction between "tolerant on purpose" and "accidentally swallowed" is legible. + +## 3. Consequences + +**Positive** + +- The quality gate's commands are honest: a failing test or a real vulnerability now fails the command. +- `just --list` is generated from the recipes themselves, so it cannot drift from reality the way a hand-maintained `help` target does. +- Recipe intent is clearer: no `.PHONY` noise, no tab sensitivity, explicit shell semantics. + +**Negative / costs** + +- **`just` becomes a required developer dependency.** Make is preinstalled nearly everywhere; `just` is not. Install: `brew install just`, `cargo install just`, `mise use -g just`, or see the project README. +- Existing muscle memory needs a one-word change (`make test` โ†’ `just test`). Mitigated by keeping every recipe name identical. +- Anyone with a personal script wrapping `make ` must update it. + +**Neutral** + +- CI is unaffected in substance: no GitHub Actions workflow invoked `make` directly โ€” the workflows call `cargo`/`pnpm` themselves. The task runner was only ever a developer and local-gate surface. + +## 4. Alternatives Considered + +- **Keep Make, just fix the swallowed exit codes.** Cheapest option, and it would have addressed the immediate correctness bugs. Rejected because it leaves the structural mismatch in place: the next multi-line recipe or `.PHONY` omission reintroduces the same class of problem, and the hand-written `help` target keeps drifting. +- **`cargo-make`.** Rust-native and capable, but TOML-configured and Rust-centric; this repo's task surface is half frontend (pnpm/Turborepo), so a language-neutral runner fits better. +- **npm scripts as the top-level entry point.** Would put the Rust backend behind a Node tool and require Node to run backend tasks. Rejected. +- **Shell scripts in `scripts/`.** Maximum portability, zero new dependency โ€” but loses discoverability (`just --list`), argument handling, and grouping, and the repo already uses `scripts/` for genuine scripts rather than task aliases. + +## 5. References + +- `justfile`, `backend/justfile`, `frontend/justfile` โ€” the implementation. +- `.autopilot/profile.yml` โ€” the automated quality gate's command set, migrated from `make` to `just` in the same change so the gate never points at deleted targets. +- Parking-lot item `pl-mk-audit-swallow` (`.autopilot/discovered/ci-build-optimization.jsonl`) โ€” the backend audit swallow this ADR's change resolves. +- [just manual](https://just.systems/man/en/) diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 0d58be2..b0327ca 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -18,11 +18,11 @@ git clone https://github.com/pacphi/emailibrium.git cd emailibrium # 2. Guided first-time setup (recommended) -make setup # interactive wizard: prereqs, secrets, AI providers, Docker +just setup # interactive wizard: prereqs, secrets, AI providers, Docker # 3. Or skip the wizard and go directly: -make install # install all dependencies -make dev # start backend + frontend dev servers +just install # install all dependencies +just dev # start backend + frontend dev servers # 4. Open the application open http://localhost:3000 @@ -30,7 +30,7 @@ open http://localhost:3000 The backend runs on `http://localhost:8080` and the frontend on `http://localhost:3000`. -> **First time?** Run `make setup` for a guided walkthrough that checks prerequisites, generates secrets, configures AI providers, and validates your environment. See [Setup Guide](setup-guide.md) for the full reference. +> **First time?** Run `just setup` for a guided walkthrough that checks prerequisites, generates secrets, configures AI providers, and validates your environment. See [Setup Guide](setup-guide.md) for the full reference. ## Configuration @@ -49,7 +49,7 @@ For production deployments, pre-download models during the build phase to avoid ```bash # During Docker build or deployment setup: -make download-models +just download-models # Or in Dockerfile: RUN cargo run --release -- --download-models @@ -379,15 +379,15 @@ Use this endpoint for load balancer health checks and uptime monitoring. | Command | Description | | -------------- | ------------------------------------------------------ | -| `make install` | Install all dependencies | -| `make build` | Build backend and frontend | -| `make test` | Run all tests | -| `make lint` | Lint all code | -| `make format` | Format all code | -| `make ci` | Full CI pipeline (format-check, lint, typecheck, test) | -| `make dev` | Start dev servers | -| `make clean` | Clean build artifacts | -| `make audit` | Security audit dependencies | +| `just install` | Install all dependencies | +| `just build` | Build backend and frontend | +| `just test` | Run all tests | +| `just lint` | Lint all code | +| `just format` | Format all code | +| `just ci` | Full CI pipeline (format-check, lint, typecheck, test) | +| `just dev` | Start dev servers | +| `just clean` | Clean build artifacts | +| `just audit` | Security audit dependencies | ## Troubleshooting diff --git a/docs/maintainer-guide.md b/docs/maintainer-guide.md index 10621df..24a3e45 100644 --- a/docs/maintainer-guide.md +++ b/docs/maintainer-guide.md @@ -33,7 +33,7 @@ emailibrium/ tests/ Integration tests (search, classification, clustering, security) benches/ Criterion benchmarks (vector_benchmarks) Cargo.toml Rust dependencies (edition 2021, MSRV 1.97) - Makefile Backend-specific make targets + justfile Backend-specific just recipes Dockerfile Multi-stage Rust build frontend/ React TypeScript monorepo (pnpm workspaces + Turborepo) @@ -45,7 +45,7 @@ emailibrium/ api/ API client and hooks (@emailibrium/api) ui/ Shared component library (@emailibrium/ui) core/ Business logic utilities (@emailibrium/core) - Makefile Frontend-specific make targets + justfile Frontend-specific just recipes nginx.conf Production reverse-proxy config Dockerfile Multi-stage Node build @@ -64,7 +64,7 @@ emailibrium/ docker-compose.yml Production container orchestration docker-compose.dev.yml Dev overlay (hot-reload, debug ports) - Makefile Root Makefile (delegates to backend/ and frontend/) + justfile Root justfile (delegates to backend/ and frontend/) config.yaml Base configuration defaults secrets/ Generated development secrets (gitignored) CLAUDE.md AI assistant configuration @@ -90,13 +90,13 @@ emailibrium/ ```bash # Clone and install all dependencies -make install +just install # Generate development secrets -make docker-secrets +just docker-secrets # Start both servers in dev mode -make dev +just dev # Backend: http://localhost:8080 # Frontend: http://localhost:3000 ``` @@ -106,19 +106,19 @@ make dev **Native development** (recommended for fast iteration): ```bash -make dev # Start backend + frontend with hot-reload -make test # Run all tests (backend + frontend) -make lint # Lint everything (Rust + TypeScript + Markdown + YAML) -make ci # Full CI pipeline: format-check, lint, typecheck, test +just dev # Start backend + frontend with hot-reload +just test # Run all tests (backend + frontend) +just lint # Lint everything (Rust + TypeScript + Markdown + YAML) +just ci # Full CI pipeline: format-check, lint, typecheck, test ``` **Docker development** (matches production environment): ```bash -make docker-up-dev # Start with hot-reload via docker-compose.dev.yml -make docker-logs # Tail all container logs -make docker-health # Check container status -make docker-down # Stop everything +just docker-up-dev # Start with hot-reload via docker-compose.dev.yml +just docker-logs # Tail all container logs +just docker-health # Check container status +just docker-down # Stop everything ``` ### Backend Development @@ -138,7 +138,7 @@ The backend is organized into four module groups under `backend/src/`: 2. Register the route in `api/mod.rs`. 3. If it needs new vector operations, add them to `vectors/mod.rs` (the `VectorService` facade). 4. Write an integration test in `backend/tests/`. -5. Run `make -C backend test` to verify. +5. Run `cd backend && just test` to verify. **Adding a new vector module:** @@ -176,7 +176,7 @@ Two invariants worth stating explicitly, because both are easy to violate by acc them as missing fields: - `list_subscriptions` and `preview_cleanup_plan` never return raw `List-Unsubscribe` / `List-Unsubscribe-Post` header values โ€” one-click URLs embed per-recipient tokens and act - as capability URLs. Callers get `has_unsubscribe` and the unsubscribe *method* instead. + as capability URLs. Callers get `has_unsubscribe` and the unsubscribe _method_ instead. - `list_attachments` returns metadata only, never `storage_path` (a filesystem path) or `provider_attachment_id`. - `get_sync_status` omits the `history_id` and `next_page_token` sync cursors โ€” opaque @@ -188,7 +188,7 @@ belong to the shared dispatch path, not to individual handlers. **Running benchmarks:** ```bash -make -C backend bench # Run Criterion benchmarks +cd backend && just bench # Run Criterion benchmarks ``` ### Frontend Development @@ -225,26 +225,26 @@ The frontend follows a **feature-sliced design** pattern. Features are self-cont | Layer | Tool | Location | Run Command | | ------------------- | ------------------------ | --------------------------------- | --------------------------------------- | -| Backend unit | `#[cfg(test)]` modules | In each `.rs` file | `make -C backend test` | -| Backend integration | `#[test]` functions | `backend/tests/*.rs` | `make -C backend test` | -| Backend benchmarks | Criterion | `backend/benches/` | `make -C backend bench` | -| Frontend unit | Vitest + Testing Library | Co-located with source | `make -C frontend test` | +| Backend unit | `#[cfg(test)]` modules | In each `.rs` file | `cd backend && just test` | +| Backend integration | `#[test]` functions | `backend/tests/*.rs` | `cd backend && just test` | +| Backend benchmarks | Criterion | `backend/benches/` | `cd backend && just bench` | +| Frontend unit | Vitest + Testing Library | Co-located with source | `cd frontend && just test` | | Frontend E2E | Playwright | `frontend/apps/web/e2e/` | `cd frontend/apps/web && pnpm test:e2e` | -| Security audit | Custom test suite | `backend/tests/security_audit.rs` | `make -C backend test` | +| Security audit | Custom test suite | `backend/tests/security_audit.rs` | `cd backend && just test` | ### Code Style - **Rust:** `cargo fmt` for formatting, `cargo clippy` for lints. Both run in CI. - **TypeScript:** Prettier for formatting, ESLint for lints. Config lives in the frontend workspace root. -- **Markdown and YAML:** Prettier + markdownlint-cli2 + yamllint. Run `make lint-docs`. +- **Markdown and YAML:** Prettier + markdownlint-cli2 + yamllint. Run `just lint-docs`. - **Commits:** Use conventional commit messages (`feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`). ### Dependency Management ```bash -make outdated # Show outdated deps across both stacks (no changes) -make upgrade # Upgrade within semver ranges -make audit # Run cargo audit + pnpm audit +just outdated # Show outdated deps across both stacks (no changes) +just upgrade # Upgrade within semver ranges +just audit # Run cargo audit + pnpm audit ``` Pin exact versions for security-critical crates (`aes-gcm`, `argon2`, `zeroize`). Use semver ranges for everything else. @@ -312,12 +312,12 @@ Target: **WCAG 2.1 AA** compliance. ### Docker Deployment ```bash -make docker-build # Build all images -make docker-up # Start production stack -make docker-health # Check container health -make docker-logs # Tail all logs -make docker-down # Stop everything -make docker-down-volumes # Stop and destroy data (CAUTION) +just docker-build # Build all images +just docker-up # Start production stack +just docker-health # Check container health +just docker-logs # Tail all logs +just docker-down # Stop everything +just docker-down-volumes # Stop and destroy data (CAUTION) ``` ### Configuration @@ -331,7 +331,7 @@ Emailibrium uses a **layered configuration** system via figment. Later layers ov > Neither YAML file exists in a fresh checkout. `figment`'s `Yaml::file` is a no-op on a > missing path, so today configuration resolves from compile-time defaults, `EMAILIBRIUM_*` > env vars, and the `config/app.yaml` path-override pass in `apply_yaml_path_defaults`. -> Note that `config/app.yaml` is a *different* file from the `config.yaml` above and is read +> Note that `config/app.yaml` is a _different_ file from the `config.yaml` above and is read > by a different code path -- a key placed in the wrong one is silently read by nobody. > > Env vars map onto nested keys by splitting on `_`: `EMAILIBRIUM_MCP_MODE` sets `mcp.mode`. @@ -347,7 +347,7 @@ For the complete key reference, see [configuration-reference.md](./configuration ### Secrets Management - Use **file-based secrets** via Docker secrets in production. Never use environment variables for sensitive values in production. -- Generate development secrets: `make docker-secrets` (creates `secrets/dev/`). +- Generate development secrets: `just docker-secrets` (creates `secrets/dev/`). - Sensitive keys that must never appear in config files: `encryption.master_password`, `database_url` (production). ### Health Checks @@ -438,7 +438,7 @@ Vector embeddings are derived data, but partial text recovery is theoretically p ### Audit and Testing ```bash -make audit # cargo audit + pnpm audit +just audit # cargo audit + pnpm audit ``` The security test suite at `backend/tests/security_audit.rs` validates: @@ -449,7 +449,7 @@ The security test suite at `backend/tests/security_audit.rs` validates: - Embedding invertibility resistance - CSP and CORS header correctness -**Dependency auditing:** Run `make audit` regularly. Enable GitHub Dependabot for automated vulnerability alerts. +**Dependency auditing:** Run `just audit` regularly. Enable GitHub Dependabot for automated vulnerability alerts. ### ADR References @@ -538,11 +538,11 @@ Located in `docs/evaluation/`. Metrics include: Before requesting review, verify: -- [ ] `make ci` passes (format-check, lint, typecheck, test) +- [ ] `just ci` passes (format-check, lint, typecheck, test) - [ ] No new `cargo clippy` warnings -- [ ] TypeScript compiles cleanly (`make typecheck`) +- [ ] TypeScript compiles cleanly (`just typecheck`) - [ ] New features include tests -- [ ] Security-sensitive changes include `make audit` +- [ ] Security-sensitive changes include `just audit` ### ADR Process @@ -576,7 +576,7 @@ cargo install git-cliff # provides changelog generation ### Happy Path ```bash -make release VERSION=0.1.0 +just release VERSION=0.1.0 ``` This runs `scripts/release.sh`, which: @@ -622,73 +622,73 @@ git push origin main | Command | Description | | -------------- | ----------------------------------------------------------- | -| `make install` | Install all dependencies (backend build + frontend install) | -| `make build` | Build everything (backend release + frontend production) | -| `make clean` | Remove all build artifacts | +| `just install` | Install all dependencies (backend build + frontend install) | +| `just build` | Build everything (backend release + frontend production) | +| `just clean` | Remove all build artifacts | ### Development | Command | Description | | -------------------- | ------------------------------------------------------ | -| `make dev` | Start backend and frontend dev servers with hot-reload | -| `make docker-up-dev` | Start dev stack in Docker with hot-reload | +| `just dev` | Start backend and frontend dev servers with hot-reload | +| `just docker-up-dev` | Start dev stack in Docker with hot-reload | ### Testing -| Command | Description | -| ----------------------- | ----------------------------------------------------- | -| `make test` | Run all tests (backend + frontend) | -| `make ci` | Full CI pipeline: format-check, lint, typecheck, test | -| `make ci-full` | CI + link checking | -| `make -C backend bench` | Run Criterion benchmarks | +| Command | Description | +| -------------------------- | ----------------------------------------------------- | +| `just test` | Run all tests (backend + frontend) | +| `just ci` | Full CI pipeline: format-check, lint, typecheck, test | +| `just ci-full` | CI + link checking | +| `cd backend && just bench` | Run Criterion benchmarks | ### Code Quality | Command | Description | | ------------------- | -------------------------------- | -| `make lint` | Lint all code and docs | -| `make format` | Auto-format all code and docs | -| `make format-check` | Check formatting without changes | -| `make typecheck` | TypeScript type checking | -| `make deadcode` | Detect unused code | -| `make audit` | Security audit all dependencies | +| `just lint` | Lint all code and docs | +| `just format` | Auto-format all code and docs | +| `just format-check` | Check formatting without changes | +| `just typecheck` | TypeScript type checking | +| `just deadcode` | Detect unused code | +| `just audit` | Security audit all dependencies | ### Docker | Command | Description | | ---------------------------- | --------------------------------------------- | -| `make docker-build` | Build all Docker images | -| `make docker-build-no-cache` | Build images without cache | -| `make docker-up` | Start production stack | -| `make docker-up-dev` | Start dev stack with hot-reload | -| `make docker-down` | Stop all containers | -| `make docker-down-volumes` | Stop and remove volumes (destroys data) | -| `make docker-restart` | Restart all containers | -| `make docker-health` | Show container health status | -| `make docker-logs` | Tail logs from all containers | -| `make docker-logs-backend` | Tail backend logs only | -| `make docker-logs-frontend` | Tail frontend logs only | -| `make docker-ps` | Show running containers | -| `make docker-exec-backend` | Shell into backend container | -| `make docker-exec-frontend` | Shell into frontend container | -| `make docker-secrets` | Generate development secrets | -| `make docker-clean` | Remove stopped containers and dangling images | +| `just docker-build` | Build all Docker images | +| `just docker-build-no-cache` | Build images without cache | +| `just docker-up` | Start production stack | +| `just docker-up-dev` | Start dev stack with hot-reload | +| `just docker-down` | Stop all containers | +| `just docker-down-volumes` | Stop and remove volumes (destroys data) | +| `just docker-restart` | Restart all containers | +| `just docker-health` | Show container health status | +| `just docker-logs` | Tail logs from all containers | +| `just docker-logs-backend` | Tail backend logs only | +| `just docker-logs-frontend` | Tail frontend logs only | +| `just docker-ps` | Show running containers | +| `just docker-exec-backend` | Shell into backend container | +| `just docker-exec-frontend` | Shell into frontend container | +| `just docker-secrets` | Generate development secrets | +| `just docker-clean` | Remove stopped containers and dangling images | ### Dependencies | Command | Description | | --------------- | --------------------------------------- | -| `make outdated` | Show outdated dependencies (no changes) | -| `make upgrade` | Upgrade within semver ranges | +| `just outdated` | Show outdated dependencies (no changes) | +| `just upgrade` | Upgrade within semver ranges | ### Documentation | Command | Description | | --------------------------- | ----------------------------- | -| `make lint-md` | Lint Markdown files | -| `make lint-yaml` | Lint YAML files | -| `make format-md` | Format Markdown with Prettier | -| `make format-yaml` | Format YAML with Prettier | -| `make links-check` | Check internal file links | -| `make links-check-external` | Check external HTTP links | -| `make links-check-all` | Check all links | +| `just lint-md` | Lint Markdown files | +| `just lint-yaml` | Lint YAML files | +| `just format-md` | Format Markdown with Prettier | +| `just format-yaml` | Format YAML with Prettier | +| `just links-check` | Check internal file links | +| `just links-check-external` | Check external HTTP links | +| `just links-check-all` | Check all links | diff --git a/docs/oauth-setup-guide.md b/docs/oauth-setup-guide.md index 28c4203..f2561de 100644 --- a/docs/oauth-setup-guide.md +++ b/docs/oauth-setup-guide.md @@ -9,7 +9,7 @@ Configure Gmail (Google) and Outlook (Microsoft) OAuth for Emailibrium. ## Prerequisites - A Google Cloud project and/or Microsoft Entra (Azure AD) app registration -- `make setup` completed (writes credentials to `secrets/dev/`) +- `just setup` completed (writes credentials to `secrets/dev/`) - Backend running on `http://localhost:8080` (default) --- @@ -53,7 +53,7 @@ Configure Gmail (Google) and Outlook (Microsoft) OAuth for Emailibrium. ### 4. Store Credentials -Run `make setup-secrets` or manually create the files: +Run `just setup-secrets` or manually create the files: ```bash echo "YOUR_CLIENT_ID" > secrets/dev/google_client_id @@ -61,11 +61,11 @@ echo "YOUR_CLIENT_SECRET" > secrets/dev/google_client_secret chmod 600 secrets/dev/google_client_id secrets/dev/google_client_secret ``` -The `make dev` target automatically exports these as `EMAILIBRIUM_GOOGLE_CLIENT_ID` and `EMAILIBRIUM_GOOGLE_CLIENT_SECRET` environment variables. +The `just dev` target automatically exports these as `EMAILIBRIUM_GOOGLE_CLIENT_ID` and `EMAILIBRIUM_GOOGLE_CLIENT_SECRET` environment variables. ### 5. Test -1. Start the app: `make dev` +1. Start the app: `just dev` 2. Navigate to `http://localhost:3000/onboarding` 3. Click **Gmail** > authenticate with a test user account 4. You should be redirected back to the app after consent @@ -191,7 +191,7 @@ chmod 600 secrets/dev/microsoft_client_id secrets/dev/microsoft_client_secret ### 7. Test -1. Start the app: `make dev` +1. Start the app: `just dev` 2. Navigate to `http://localhost:3000/onboarding` 3. Click **Outlook** > authenticate with a Microsoft account 4. Grant the requested permissions when prompted @@ -237,7 +237,7 @@ oauth: | `EMAILIBRIUM_MICROSOFT_CLIENT_SECRET` | `secrets/dev/microsoft_client_secret` | Microsoft Client Secret Value | | `EMAILIBRIUM_ENCRYPTION_MASTER_PASSWORD` | `secrets/dev/oauth_encryption_key` | AES-256-GCM key for token encryption at rest | -These are loaded automatically by `make dev` from the `secrets/dev/` directory. +These are loaded automatically by `just dev` from the `secrets/dev/` directory. ### API Endpoints @@ -269,7 +269,7 @@ Your GCP project's OAuth consent screen is set to "Internal" (Workspace org only The backend can't find the OAuth credentials in environment variables. -**Fix**: Either run `make setup-secrets` to store credentials, or ensure `make dev` is used to start the server (it exports secrets from `secrets/dev/`). Running `cargo run` directly won't load secrets. +**Fix**: Either run `just setup-secrets` to store credentials, or ensure `just dev` is used to start the server (it exports secrets from `secrets/dev/`). Running `cargo run` directly won't load secrets. ### Token expired / refresh failed diff --git a/docs/plan/mcp-maturation.md b/docs/plan/mcp-maturation.md index 3d7f3cd..a159d43 100644 --- a/docs/plan/mcp-maturation.md +++ b/docs/plan/mcp-maturation.md @@ -85,7 +85,7 @@ MCP and chat paths cannot disagree about which tools exist. #### Gotchas found in the source -**`#[tool_handler]` defaults to a *static* router call.** Its default `router` expression is +**`#[tool_handler]` defaults to a _static_ router call.** Its default `router` expression is `Self::tool_router()` (`rmcp-macros-1.7.0/src/tool_handler.rs:20-25`), and the macro expands to `#router.call(tcc)`, `#router.list_all()`, `#router.get(name)`. It therefore **rebuilds the router on every single request** and never reads the struct field. This explains the @@ -196,10 +196,10 @@ today. > > They are genuinely different types serving different layers, not accidental duplicates: > -> | | `api::ingestion` | `vectors::ingestion` | -> | --- | --- | --- | -> | `IngestionPhase` | 6 variants, `Serialize + Deserialize` | 7 variants (adds `Backfilling`), `Serialize` only | -> | `IngestionProgress` | same 9 fields, but `phase: IngestionPhase` | `phase: String` | +> | | `api::ingestion` | `vectors::ingestion` | +> | ------------------- | ------------------------------------------ | ------------------------------------------------- | +> | `IngestionPhase` | 6 variants, `Serialize + Deserialize` | 7 variants (adds `Backfilling`), `Serialize` only | +> | `IngestionProgress` | same 9 fields, but `phase: IngestionPhase` | `phase: String` | > > `vectors::IngestionProgress` is the pipeline's own already-stringified snapshot; > `api::IngestionProgress` is the typed SSE broadcast payload. @@ -234,7 +234,7 @@ today. Three findings decide it: - **`vectors::ingestion` already exists** (`vectors/mod.rs:26`, `pub mod ingestion;`) and - already owns `IngestionPipeline` (`vectors/mod.rs:84`) โ€” the component that *produces* these + already owns `IngestionPipeline` (`vectors/mod.rs:84`) โ€” the component that _produces_ these progress events. Co-locating the progress types with their producer is the cohesive placement; `events` is the generic `EventBus`, and putting an ingestion-specific DTO there would mix a domain type into a general-purpose bus. @@ -560,10 +560,10 @@ declared defaults**, which is the desired failure mode. For each spec in `all_specs()`, look up `cfg.tools.get(spec.name)`: -| Field | Resolution | -| ----------------------- | -------------------------------------------------------------------------------------- | -| `enabled` | `override.enabled` ?? `true` | -| `requires_confirmation` | `override.requires_confirmation` ?? `spec.default_requires_confirmation` | +| Field | Resolution | +| ----------------------- | --------------------------------------------------------------------------------------- | +| `enabled` | `override.enabled` ?? `true` | +| `requires_confirmation` | `override.requires_confirmation` ?? `spec.default_requires_confirmation` | | `rate_limit_per_minute` | `override.rate_limit_per_minute` ?? `spec.default_rate_limit_per_minute` ?? `defaults.` | Keys in `cfg.tools` with no matching spec are handled in two classes: @@ -577,7 +577,7 @@ const DEFERRED_TOOLS: &[&str] = &["send_email", "delete_email", "create_rule"]; - name in `DEFERRED_TOOLS` โ†’ `debug!` "deferred to A4" - otherwise โ†’ `warn!(tool = %name, "tools.yaml references unknown tool; ignoring")` -Unknown *fields* inside an entry are ignored (no `deny_unknown_fields`) โ€” a hard startup +Unknown _fields_ inside an entry are ignored (no `deny_unknown_fields`) โ€” a hard startup failure on a stray YAML key is too hostile for an operator-edited file. Startup emits one summary line: `N tools registered, M disabled by config`. @@ -600,7 +600,7 @@ through `AppState` fields that are **already public**, so โ€” with one exception changes are required. > **Visibility:** the private `api::{clustering, learning, vectors}` modules are deliberately -> *not* opened up โ€” the tools reach the services through `ToolContext` instead. The only +> _not_ opened up โ€” the tools reach the services through `ToolContext` instead. The only > module moves are the ones ยง1.5.1 requires (`cleanup::domain`, `cleanup::repository`, and the > ingestion-progress types). @@ -659,6 +659,7 @@ lock: {...} | null }` > be called from a library-crate handler. The library-side twin above replaces it. This note > exists only because the superseded plan keeps resurfacing from stale handoff notes โ€” if you > arrived here looking for the `pub(crate)` change, this is the answer. + - **Read-only verification.** `build()` is pure. Its module doc states "Pure orchestrator: reads only injected ports, never touches a provider," and a search across `domain/builder.rs`, `domain/classifier.rs`, and `repository/adapters.rs` finds no `INSERT`/`UPDATE`/`DELETE` โ€” the @@ -733,13 +734,13 @@ model, whereas clamping is binding. #### 5.3 Rate-limit summary -| Tool | Limit/min | Rationale | -| ------------------------------------------------------------------------------------------------------------- | --------- | ------------------------------------ | -| `search_emails` | 30 | preserved from `tools.yaml` | -| `get_email`, `list_recent_emails`, `count_emails`, `get_email_thread`, `get_insights`, `list_rules` | 20 | preserved (global default) | -| `find_similar_emails`, `list_attachments`, `get_sync_status`, `get_learning_metrics` | 30 | single indexed read or in-memory | -| `list_accounts`, `list_clusters` | 20 | small fan-out | -| `list_subscriptions`, `preview_cleanup_plan` | 5 | multi-query fan-out over the corpus | +| Tool | Limit/min | Rationale | +| --------------------------------------------------------------------------------------------------- | --------- | ----------------------------------- | +| `search_emails` | 30 | preserved from `tools.yaml` | +| `get_email`, `list_recent_emails`, `count_emails`, `get_email_thread`, `get_insights`, `list_rules` | 20 | preserved (global default) | +| `find_similar_emails`, `list_attachments`, `get_sync_status`, `get_learning_metrics` | 30 | single indexed read or in-memory | +| `list_accounts`, `list_clusters` | 20 | small fan-out | +| `list_subscriptions`, `preview_cleanup_plan` | 5 | multi-query fan-out over the corpus | #### 5.4 Preserving the seven existing tools @@ -761,7 +762,7 @@ of all seven โ€” making the change deliberate and visible rather than incidental Two deliberate behaviour changes, both minor: 1. **Error results are flagged.** Today a failing MCP tool returns `{"error": "..."}` inside a - *successful* `CallToolResult`. The dispatch path will keep that same JSON body but also set + _successful_ `CallToolResult`. The dispatch path will keep that same JSON body but also set `is_error: true`, which is what the MCP spec intends. Clients that ignore the flag see identical text. 2. **Chat calls are now rate-limited and audited.** That is the point of the change, but it is @@ -772,14 +773,14 @@ Two deliberate behaviour changes, both minor: ### 6. What gets deleted -| Location | Change | -| --------------------------------------- | ----------------------------------------------------------------- | -| `api/ai.rs:1199` `build_tool_definitions` | Deleted โ€” replaced by `registry.chat_definitions()` | -| `api/ai.rs:1275` `build_tool_executor` | Reduced to a ~5-line closure over `registry.dispatch()` | -| `mcp/server.rs` `#[tool_router]` + 7 `#[tool]` bodies | Moved to `tools/readonly/` | -| `mcp/server.rs` `rate_limiter` field | Moved to the registry | -| `mcp/server.rs` `audit()` method | Moved into `ToolRegistry::dispatch` | -| `mcp/tools/email.rs` | Params structs move to `tools/readonly/params.rs`; row structs to `tools/readonly/emails.rs`; `mcp/tools/` is removed | +| Location | Change | +| ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `api/ai.rs:1199` `build_tool_definitions` | Deleted โ€” replaced by `registry.chat_definitions()` | +| `api/ai.rs:1275` `build_tool_executor` | Reduced to a ~5-line closure over `registry.dispatch()` | +| `mcp/server.rs` `#[tool_router]` + 7 `#[tool]` bodies | Moved to `tools/readonly/` | +| `mcp/server.rs` `rate_limiter` field | Moved to the registry | +| `mcp/server.rs` `audit()` method | Moved into `ToolRegistry::dispatch` | +| `mcp/tools/email.rs` | Params structs move to `tools/readonly/params.rs`; row structs to `tools/readonly/emails.rs`; `mcp/tools/` is removed | Net: roughly 215 duplicated lines in `ai.rs` and ~430 in `mcp/server.rs` collapse into one declaration table plus handler modules. @@ -800,16 +801,16 @@ fn list_resource_templates(&self, ..) -> .. ListResourceTemplatesResult ..; fn read_resource(&self, params: ReadResourceRequestParams, ..) -> .. ReadResourceResult ..; ``` -| URI | Kind | Backing | -| ------------------ | -------- | ----------------------------------------------------------------------- | -| `insights://summary` | concrete | reuses the `get_insights` handler | -| `email://{id}` | template | reuses the `get_email` handler | -| `thread://{key}` | template | `SELECT ... WHERE thread_key = ?` โ€” takes the thread key **directly** | +| URI | Kind | Backing | +| -------------------- | -------- | --------------------------------------------------------------------- | +| `insights://summary` | concrete | reuses the `get_insights` handler | +| `email://{id}` | template | reuses the `get_email` handler | +| `thread://{key}` | template | `SELECT ... WHERE thread_key = ?` โ€” takes the thread key **directly** | `list_resources` returns only `insights://summary`; the two parameterized entries belong in `list_resource_templates` as `RawResourceTemplate::new(uri_template, name)`. -Note the `thread://` asymmetry: the existing `get_email_thread` **tool** takes an *email id* +Note the `thread://` asymmetry: the existing `get_email_thread` **tool** takes an _email id_ and resolves its `thread_key` internally, whereas the `thread://{key}` **resource** is keyed by the thread key itself. Both are correct for their surface; the difference must be documented so the two are not confused. @@ -838,10 +839,10 @@ path is the smallest correct implementation: `#[prompt_router]` on an inherent i `#[prompt]` per prompt, and `#[prompt_handler(router = self.prompt_router)]` on the `ServerHandler` impl. -| Prompt | Arguments | Content | -| --------------- | ---------------------------- | -------------------------------------------------------------------------------- | -| `triage-inbox` | `limit` (optional, default 20) | Instructs the model to call `list_recent_emails`, then categorize by urgency and propose actions. | -| `weekly-report` | none | Instructs the model to call `get_insights` and `list_subscriptions`, then produce a written summary. | +| Prompt | Arguments | Content | +| --------------- | ------------------------------ | ---------------------------------------------------------------------------------------------------- | +| `triage-inbox` | `limit` (optional, default 20) | Instructs the model to call `list_recent_emails`, then categorize by urgency and propose actions. | +| `weekly-report` | none | Instructs the model to call `get_insights` and `list_subscriptions`, then produce a written summary. | Both return a `GetPromptResult` holding a single user-role `PromptMessage::new_text`. Because our `get_info()` is hand-written, `.enable_prompts()` must be added to it manually โ€” @@ -971,12 +972,12 @@ HTTP, and cannot edit config files โ€” they pass argv and environment. Mode is an enum resolved through the existing figment chain, **not** an ad-hoc argv check: -| Precedence | Source | Status | -| ---------- | ---------------------- | ------------------------------------------------------------------------------------------------------- | -| 1 highest | CLI flag | **Shipped this cycle** โ€” e.g. `--mcp-stdio` | -| 2 | `EMAILIBRIUM_MCP_MODE` | **Shipped this cycle** | +| Precedence | Source | Status | +| ---------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| 1 highest | CLI flag | **Shipped this cycle** โ€” e.g. `--mcp-stdio` | +| 2 | `EMAILIBRIUM_MCP_MODE` | **Shipped this cycle** | | 3 | config file | โš ๏ธ **Designed, NOT implemented this cycle** โ€” `config.yaml` does not exist; env + flag are the only shipped sources. Wiring a mode key here silently does nothing. | -| 4 lowest | default | **Shipped this cycle** โ€” `http`, zero change for existing users | +| 4 lowest | default | **Shipped this cycle** โ€” `http`, zero change for existing users | The table above is the full intended design; only rows 1, 2, and 4 exist in code. Row 3 carries its caveat inline deliberately, because the failure shape is the worst kind: a reader who wires @@ -1013,15 +1014,15 @@ the protocol stream and surface as intermittent, hard-to-diagnose parse errors. The logging profile must therefore be derived from the **resolved mode, before tracing initialisation**: -| Mode | Console layer | ANSI | HTTP listener | -| ------- | ------------- | ---- | -------------- | +| Mode | Console layer | ANSI | HTTP listener | +| ------- | ------------- | ---- | ----------------- | | `http` | stdout | on | binds (unchanged) | | `stdio` | **stderr** | off | **does not bind** | **`main()` must be reordered โ€” the current structure makes this impossible as written.** Today tracing is initialised at `main.rs:89` (the console layer is `fmt::layer().with_ansi(true)` at `:82`, defaulting to stdout), and argv is not read until -`main.rs:92`. Mode resolution therefore happens *after* the logging profile is already fixed. +`main.rs:92`. Mode resolution therefore happens _after_ the logging profile is already fixed. Implementing stdio requires hoisting mode resolution above the `tracing_subscriber::registry()` block at `main.rs:80-89`, ahead of the existing `--download-model` / `--download-models` / `--verify-models` CLI branches. This is a small change but not an optional one: leave the order @@ -1077,13 +1078,13 @@ state that must be squared away first; nothing after them compiles until they ar **Step 0 โ€” reconcile the split contract. โœ… COMPLETE.** Verified against the tree: -| Step | Work | State | -| ---- | ---- | ----- | -| 0.1 | `main.rs` re-exports the shared modules from the library instead of re-declaring them (ยง1.5) | โœ… `main.rs:21` โ€” `pub use emailibrium::{โ€ฆ}`; only `mod api;` and `mod cleanup;` remain binary-side, which is correct | -| 0.2 | Expose `tools`, `mcp`, and `mcp_service(ctx)` via `lib.rs` | โœ… `lib.rs` now exports `mcp`, `tools`, `sync_lock` and the split `cleanup` | -| 0.3 | Widen `ToolContext` to the fields in ยง2.2 | โœ… all five present, including `sync_progress: Option` | -| 0.4 | Convert the A3 handlers to `ToolContext` + `Result<_, ToolError>` | โœ… done | -| 0.5 | Hoist `cleanup::domain` + `repository`; library-side `PlanBuilder` wiring helper | โœ… both โ€” `lib.rs` carries `pub mod cleanup { pub mod domain; pub mod repository; }`, `api` and `orchestrator` stay binary-side | +| Step | Work | State | +| ---- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| 0.1 | `main.rs` re-exports the shared modules from the library instead of re-declaring them (ยง1.5) | โœ… `main.rs:21` โ€” `pub use emailibrium::{โ€ฆ}`; only `mod api;` and `mod cleanup;` remain binary-side, which is correct | +| 0.2 | Expose `tools`, `mcp`, and `mcp_service(ctx)` via `lib.rs` | โœ… `lib.rs` now exports `mcp`, `tools`, `sync_lock` and the split `cleanup` | +| 0.3 | Widen `ToolContext` to the fields in ยง2.2 | โœ… all five present, including `sync_progress: Option` | +| 0.4 | Convert the A3 handlers to `ToolContext` + `Result<_, ToolError>` | โœ… done | +| 0.5 | Hoist `cleanup::domain` + `repository`; library-side `PlanBuilder` wiring helper | โœ… both โ€” `lib.rs` carries `pub mod cleanup { pub mod domain; pub mod repository; }`, `api` and `orchestrator` stay binary-side | The two type identities have collapsed, so `emailibrium::db::Database` and the binary's `crate::db::Database` are now one type, and `backend/tests/` can reach the shipping code. The @@ -1092,20 +1093,20 @@ destination, re-exported from `api/ingestion.rs` under the old names, zero wire- **Then the original sequence** โ€” status verified against the tree: -| Step | Work | State | -| ---- | ---- | ----- | -| 1 | `tools/{mod.rs, config.rs}` โ€” registry, dispatch, config types | โœ… **done** | -| 2 | Move the seven existing handlers into `tools/readonly/` | โœ… **done** โ€” all fifteen declared in `tools/mod.rs::declarations()`; `mcp/server.rs` has zero `#[tool(` macros left | -| 3 | Rewrite `mcp/server.rs` onto `build_tool_router` + `#[tool_handler(router = self.tool_router)]` | โœ… **done** โ€” `build_tool_router` at `mcp/server.rs:60`, `ToolRoute::new_dyn` at `:67`, explicit router attribute at `:126` | -| 4 | Collapse `ai.rs` onto `registry.chat_definitions()` / `registry.dispatch()` | โœ… **done** โ€” both builders deleted, `ai.rs` 1619 โ†’ 1355 lines | -| 5 | Migration 029; `source` on the audit path; Tier 3 tests | โœ… **done** โ€” `029_mcp_tool_audit_source.sql` exists; audit moved to `tools/audit.rs` and binds `source` (`:21`, `:37`, `:45`) | -| 6 | Wire the eight A3 tools into the declaration table | โœ… **done** โ€” all eight present in `declarations()` | -| 7 | A5 resources and prompts | โœ… **done** โ€” `mcp/resources.rs` routes reads through registry enforcement; `get_info` (`mcp/server.rs:182-184`) declares `.enable_tools().enable_prompts().enable_resources()` | -| 8 | stdio mode (ยง10), including the `transport-io` Cargo feature | โœ… **done** โ€” feature added (`Cargo.toml:103`); `resolve_mcp_mode()` runs at `main.rs:164`, ahead of `registry()` at `:188` and `.init()` at `:197`; stdio selects a stderr, ANSI-off console layer (`:180-184`) | +| Step | Work | State | +| ---- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | `tools/{mod.rs, config.rs}` โ€” registry, dispatch, config types | โœ… **done** | +| 2 | Move the seven existing handlers into `tools/readonly/` | โœ… **done** โ€” all fifteen declared in `tools/mod.rs::declarations()`; `mcp/server.rs` has zero `#[tool(` macros left | +| 3 | Rewrite `mcp/server.rs` onto `build_tool_router` + `#[tool_handler(router = self.tool_router)]` | โœ… **done** โ€” `build_tool_router` at `mcp/server.rs:60`, `ToolRoute::new_dyn` at `:67`, explicit router attribute at `:126` | +| 4 | Collapse `ai.rs` onto `registry.chat_definitions()` / `registry.dispatch()` | โœ… **done** โ€” both builders deleted, `ai.rs` 1619 โ†’ 1355 lines | +| 5 | Migration 029; `source` on the audit path; Tier 3 tests | โœ… **done** โ€” `029_mcp_tool_audit_source.sql` exists; audit moved to `tools/audit.rs` and binds `source` (`:21`, `:37`, `:45`) | +| 6 | Wire the eight A3 tools into the declaration table | โœ… **done** โ€” all eight present in `declarations()` | +| 7 | A5 resources and prompts | โœ… **done** โ€” `mcp/resources.rs` routes reads through registry enforcement; `get_info` (`mcp/server.rs:182-184`) declares `.enable_tools().enable_prompts().enable_resources()` | +| 8 | stdio mode (ยง10), including the `transport-io` Cargo feature | โœ… **done** โ€” feature added (`Cargo.toml:103`); `resolve_mcp_mode()` runs at `main.rs:164`, ahead of `registry()` at `:188` and `.init()` at `:197`; stdio selects a stderr, ANSI-off console layer (`:180-184`) | `config/tools.yaml` **is now loaded** โ€” `main.rs:544` calls `tools::config::ToolsConfig::load("../config")`; the `TODO(A1)` is gone. -> ### โœ… Resolved โ€” and the fix eliminated the bug *class*, not just the instance. +> ### โœ… Resolved โ€” and the fix eliminated the bug _class_, not just the instance. > > `sync_progress` and `pipeline_locks` briefly shipped as permanently `None`: the builders > existed but `From<&AppState>` never called them. No compile error (the fields are `Option`), @@ -1155,12 +1156,12 @@ destination, re-exported from `api/ingestion.rs` under the old names, zero wire- Every significant defect this branch produced is the same shape, and naming it is more useful than the individual fixes: -| # | Instance | The two representations | -| - | -------- | ----------------------- | -| 1 | The original problem | `ai.rs` hand-written schemas vs the `#[tool]` macros | -| 2 | Flattened `Denied`/`RateLimited` | one audit status covering two opposite caller actions | -| 3 | `CallSource` on `thread://` | the enum's documented intent vs the value passed | -| 4 | `thread://` limiter rejection | mapped to `Denied` where dispatch maps `RateLimited` | +| # | Instance | The two representations | +| --- | -------------------------------- | ----------------------------------------------------- | +| 1 | The original problem | `ai.rs` hand-written schemas vs the `#[tool]` macros | +| 2 | Flattened `Denied`/`RateLimited` | one audit status covering two opposite caller actions | +| 3 | `CallSource` on `thread://` | the enum's documented intent vs the value passed | +| 4 | `thread://` limiter rejection | mapped to `Denied` where dispatch maps `RateLimited` | **Every one sits at a seam where a special case bypasses the shared path**, and **two of the four are `thread://` specifically** โ€” because it is the only resource that cannot go through @@ -1277,6 +1278,7 @@ Steps 1โ€“4 are the consolidation and are independently shippable: a reviewer ca in by `resource_reads_are_distinguishable_from_tool_calls_in_the_audit_trail` (`mcp/resources.rs:553`), and the reasoning now lives in the code's own doc comment (`:189`) rather than only here. + - **Pre-existing debt, surfaced but not fixed here: two parallel progress types.** `api::ingestion` and `vectors::ingestion` each define an `IngestionPhase` and an `IngestionProgress` with the same names, different variants, and different `phase` field @@ -1343,7 +1345,7 @@ startup logs and ANSI escapes would corrupt the stream on first connect. At the `emailibrium=info` filter this fails every time, not intermittently. The fix has an ordering constraint. The writer is chosen when the subscriber is constructed, -so the mode must be read *before* `.init()` at `main.rs:89` โ€” earlier than the existing CLI +so the mode must be read _before_ `.init()` at `main.rs:89` โ€” earlier than the existing CLI flags, which are all parsed from `std::env::args()` at `main.rs:92` and after. ยง10.2 records the same constraint from the design side. This also constrains any future config-file layer for the mode: the `figment` chain that would read it runs later in startup, so a file layer needs @@ -1386,10 +1388,10 @@ topology in ยง1.5: `main.rs` re-declares modules that `lib.rs` also exports, so tests linking the library cannot reach binary-side handlers. That is a compile constraint, not a testing preference โ€” which is why `ToolContext` exists. Until the registry lands: -| Tier | Coverage | State | -| ---- | ------------------------------------------------------ | ---------------------------------------------------- | -| 1 | Per-handler unit tests | Present for the new handlers and the shared validators in `tools/readonly/mod.rs`. | -| 2 | Registry dispatch, rate limiting, audit rows | Blocked on step 1. | +| Tier | Coverage | State | +| ---- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| 1 | Per-handler unit tests | Present for the new handlers and the shared validators in `tools/readonly/mod.rs`. | +| 2 | Registry dispatch, rate limiting, audit rows | Blocked on step 1. | | 3 | Parity โ€” chat definitions equal the `tools/list` response | Blocked on step 4. This is the assertion that prevents the `ai.rs` drift from recurring, so it should not be dropped. | The regression gate from ยง12 has not run: it requires the seven existing tools to pass through @@ -1401,10 +1403,10 @@ the registry unchanged, and the registry does not exist yet. MCP is served from the same Axum process as the REST API โ€” no second port, no separate daemon. -| Mode | Selector | Behaviour | -| ----------------- | ------------------------------------------------- | -------------------------------------------------------- | -| `http` (default) | โ€” | Streamable HTTP at `http://localhost:8080/api/v1/mcp`. | -| `stdio` | `--mcp-stdio` or `EMAILIBRIUM_MCP_MODE=stdio` | JSON-RPC over stdin/stdout; the HTTP server does not start. | +| Mode | Selector | Behaviour | +| ---------------- | --------------------------------------------- | ----------------------------------------------------------- | +| `http` (default) | โ€” | Streamable HTTP at `http://localhost:8080/api/v1/mcp`. | +| `stdio` | `--mcp-stdio` or `EMAILIBRIUM_MCP_MODE=stdio` | JSON-RPC over stdin/stdout; the HTTP server does not start. | The CLI flag wins when both are set. The two spellings are deliberately asymmetric โ€” the flag is a boolean shorthand, the environment variable takes the mode name โ€” so there is no diff --git a/docs/releasing.md b/docs/releasing.md index f1c22e0..b45332c 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -23,13 +23,13 @@ cargo install git-cliff # provides changelog generation **Cut a release (single command):** ```bash -make release VERSION=0.1.0 +just release VERSION=0.1.0 # Or for pre-releases: -make release VERSION=0.2.0-alpha.1 +just release VERSION=0.2.0-alpha.1 ``` -`make release` handles all version bumping, changelog regeneration, git commit, tag, and push. +`just release` handles all version bumping, changelog regeneration, git commit, tag, and push. ## Step-by-Step @@ -44,7 +44,7 @@ cargo install git-cliff ### 2. Version Bumping (automated) -`make release VERSION=x.y.z` automatically updates: +`just release VERSION=x.y.z` automatically updates: | File | Field | | ---------------------------------- | -------------------- | @@ -140,19 +140,19 @@ docker compose pull # pulls :latest # Or checkout a previous tag git checkout v0.1.0 -make install -make dev +just install +just dev ``` ## Useful Commands | Command | Description | | -------------------------------- | ------------------------------------------------------------- | -| `make release-check` | Run full CI pipeline | -| `make release-tag VERSION=x.y.z` | Create annotated tag | -| `make release-push` | Push latest tag to origin | -| `make release VERSION=x.y.z` | Full one-command release (bump, changelog, commit, tag, push) | -| `make changelog` | Regenerate CHANGELOG.md via git-cliff | -| `make release-check` | Run full CI pipeline only | -| `make release-tag VERSION=x.y.z` | Create annotated tag only | -| `make release-push` | Push latest tag to origin only | +| `just release-check` | Run full CI pipeline | +| `just release-tag VERSION=x.y.z` | Create annotated tag | +| `just release-push` | Push latest tag to origin | +| `just release VERSION=x.y.z` | Full one-command release (bump, changelog, commit, tag, push) | +| `just changelog` | Regenerate CHANGELOG.md via git-cliff | +| `just release-check` | Run full CI pipeline only | +| `just release-tag VERSION=x.y.z` | Create annotated tag only | +| `just release-push` | Push latest tag to origin only | diff --git a/docs/setup-guide.md b/docs/setup-guide.md index d2e3a33..e7ee8ad 100644 --- a/docs/setup-guide.md +++ b/docs/setup-guide.md @@ -1,7 +1,7 @@ # Emailibrium Setup Guide This guide walks through setting up Emailibrium for local development. -Run `make setup` for an interactive wizard that automates these steps. +Run `just setup` for an interactive wizard that automates these steps. ## Prerequisites @@ -17,7 +17,7 @@ Run `make setup` for an interactive wizard that automates these steps. Check all prerequisites at once: ```bash -make setup-prereqs +just setup-prereqs ``` ### Git Submodules @@ -34,7 +34,7 @@ Secrets live in `secrets/dev/` (gitignored). The setup script auto-generates cryptographic secrets and prompts for OAuth credentials. ```bash -make setup-secrets +just setup-secrets ``` ### Auto-generated secrets @@ -89,7 +89,7 @@ No API keys, no external services, no data leaves your machine. To avoid the first-use download delay: ```bash -make download-models +just download-models ``` Or download individually: @@ -105,7 +105,7 @@ npx tsx scripts/models.ts download --default ### Check Your Configuration ```bash -make diagnose +just diagnose ``` Shows embedding status, LLM model status, Ollama availability, and cloud API keys. @@ -125,7 +125,7 @@ See [Configuration Reference](configuration-reference.md) for all options. Emailibrium supports a tiered AI architecture. Configure providers with: ```bash -make setup-ai +just setup-ai ``` ### ONNX (default, local) @@ -159,10 +159,10 @@ Choose between Docker and native development. ### Docker Development (recommended for first run) ```bash -make setup-docker # Build images, optionally start services -make docker-up-dev # Start with hot-reload -make docker-logs # Tail logs -make docker-down # Stop +just setup-docker # Build images, optionally start services +just docker-up-dev # Start with hot-reload +just docker-logs # Tail logs +just docker-down # Stop ``` Docker Compose starts: PostgreSQL, Redis, backend (Rust), frontend (React). @@ -170,8 +170,8 @@ Docker Compose starts: PostgreSQL, Redis, backend (Rust), frontend (React). ### Native Development ```bash -make install # Install all dependencies -make dev # Start backend + frontend dev servers +just install # Install all dependencies +just dev # Start backend + frontend dev servers ``` Native dev uses SQLite by default (configured in `configs/config.development.yaml`). @@ -181,7 +181,7 @@ Native dev uses SQLite by default (configured in `configs/config.development.yam Run all validation checks: ```bash -make setup-validate +just setup-validate ``` This checks: secrets, backend compilation, frontend build, Docker health, @@ -237,9 +237,9 @@ stderr and `data/logs/emailibrium.log`. The mode is an enum, `http` (default) or `stdio`, and can be set two ways: -| Source | Form | -| ------- | -------------------------- | -| CLI | `--mcp-stdio` | +| Source | Form | +| ------- | ---------------------------- | +| CLI | `--mcp-stdio` | | Env var | `EMAILIBRIUM_MCP_MODE=stdio` | The CLI flag wins if both are set. Note the two spellings are not symmetric: the flag is a @@ -284,7 +284,7 @@ is not running; a 404 means it started without the MCP routes mounted. - Ensure Docker Desktop is running and has enough disk space - Try `docker system prune -f` to clean old images -- Rebuild without cache: `make docker-build-no-cache` +- Rebuild without cache: `just docker-build-no-cache` ### "cargo check failed" @@ -299,8 +299,8 @@ is not running; a 404 means it started without the MCP routes mounted. ### "Backend not reachable on localhost:8080" - Check if port 8080 is already in use: `lsof -i :8080` -- For Docker: check container logs with `make docker-logs-backend` -- For native: check `make -C backend dev` output +- For Docker: check container logs with `just docker-logs-backend` +- For native: check `cd backend && just dev` output ### "ONNX model download slow" diff --git a/docs/user-guide.md b/docs/user-guide.md index 0c350e6..4872f3f 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -217,7 +217,7 @@ When you launch Emailibrium for the first time, a guided onboarding flow walks y You can return to onboarding settings at any time via Settings. -> **Developer setup:** Run `make setup` on the command line for a guided wizard that configures prerequisites, secrets, OAuth apps, and AI providers before launching the app. See the [Setup Guide](setup-guide.md). +> **Developer setup:** Run `just setup` on the command line for a guided wizard that configures prerequisites, secrets, OAuth apps, and AI providers before launching the app. See the [Setup Guide](setup-guide.md). ### Chat diff --git a/frontend/Makefile b/frontend/Makefile deleted file mode 100644 index 1f62e84..0000000 --- a/frontend/Makefile +++ /dev/null @@ -1,167 +0,0 @@ -# ============================================================================ -# Emailibrium Frontend โ€” React/TypeScript (pnpm + Turborepo) -# ============================================================================ -# -# Quick Start: -# make install - Install dependencies -# make dev - Start Vite dev server -# make build - Build all packages -# make test - Run tests -# ============================================================================ - -SHELL := /bin/bash -PNPM := pnpm -TURBO := pnpm turbo -WEB := apps/web - -BOLD := $(shell tput bold 2>/dev/null || echo '') -GREEN := $(shell tput setaf 2 2>/dev/null || echo '') -YELLOW := $(shell tput setaf 3 2>/dev/null || echo '') -BLUE := $(shell tput setaf 4 2>/dev/null || echo '') -RED := $(shell tput setaf 1 2>/dev/null || echo '') -RESET := $(shell tput sgr0 2>/dev/null || echo '') - -.DEFAULT_GOAL := help - -# ============================================================================ -# Default Target -# ============================================================================ - -.PHONY: help -help: - @echo "$(BOLD)$(BLUE)โ•โ•โ• Emailibrium Frontend (TypeScript/React) โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•$(RESET)" - @echo "" - @echo "$(BOLD)Install & Build$(RESET)" - @echo " install - Install pnpm dependencies" - @echo " build - Build all packages (Turborepo)" - @echo " dev - Start Vite dev server (port 3000)" - @echo " storybook - Start Storybook component explorer" - @echo " clean - Clean build artifacts" - @echo "" - @echo "$(BOLD)Test$(RESET)" - @echo " test - Run unit tests (Vitest)" - @echo " test-e2e - Run E2E tests (Playwright)" - @echo "" - @echo "$(BOLD)Lint & Format$(RESET)" - @echo " lint - Lint TypeScript (ESLint)" - @echo " format - Format code (Prettier)" - @echo " format-check - Check formatting" - @echo " typecheck - TypeScript type check" - @echo " deadcode - Check for unused exports" - @echo "" - @echo "$(BOLD)Security$(RESET)" - @echo " audit - Security audit (pnpm audit)" - @echo "" - @echo "$(BOLD)Dependencies$(RESET)" - @echo " upgrade - Upgrade deps (within semver)" - @echo " upgrade-latest - Upgrade to latest major versions" - @echo " outdated - Show outdated packages" - @echo "" - @echo "$(BOLD)Documentation$(RESET)" - @echo " lint-docs - Lint Markdown + YAML in frontend/" - @echo " format-docs - Format Markdown + YAML in frontend/" - -# ============================================================================ -# Install & Build -# ============================================================================ - -.PHONY: install -install: ## Install dependencies - @$(PNPM) install - -.PHONY: build -build: ## Build all packages - @$(TURBO) build - -.PHONY: dev -dev: ## Start Vite dev server - @cd $(WEB) && npx vite - -.PHONY: storybook -storybook: ## Start Storybook - @cd $(WEB) && npx storybook dev -p 6006 - -.PHONY: clean -clean: ## Clean build artifacts - @rm -rf $(WEB)/dist $(WEB)/storybook-static node_modules/.cache .turbo - -# ============================================================================ -# Test -# ============================================================================ - -.PHONY: test -test: ## Run unit tests (Vitest) - @$(TURBO) test - -.PHONY: test-e2e -test-e2e: ## Run E2E tests (Playwright) - @cd $(WEB) && npx playwright test - -# ============================================================================ -# Lint & Format -# ============================================================================ - -.PHONY: lint -lint: ## Lint TypeScript (ESLint โ€” strict) - @cd $(WEB) && npx --no-install eslint src/ --ext .ts,.tsx - -.PHONY: format -format: ## Format code (Prettier) - @cd $(WEB) && npx prettier --write 'src/**/*.{ts,tsx}' - -.PHONY: format-check -format-check: ## Check formatting - @cd $(WEB) && npx prettier --check 'src/**/*.{ts,tsx}' - -.PHONY: typecheck -typecheck: ## TypeScript type check - @cd $(WEB) && npx tsc --noEmit - -.PHONY: deadcode -deadcode: ## Check for unused exports - @cd $(WEB) && npx ts-prune src/ 2>/dev/null || echo "$(YELLOW)ts-prune not installed. Run: pnpm add -Dw ts-prune$(RESET)" - -# ============================================================================ -# Security -# ============================================================================ - -.PHONY: audit -audit: ## Security audit - @$(PNPM) audit --prod - -# ============================================================================ -# Dependency Management -# ============================================================================ - -.PHONY: upgrade -upgrade: ## Upgrade dependencies (within semver) - @$(PNPM) update -r - @echo "$(GREEN)Frontend packages upgraded.$(RESET)" - -.PHONY: upgrade-latest -upgrade-latest: ## Upgrade to latest major versions - @$(PNPM) update -r --latest - @echo "$(GREEN)Done. Run 'make typecheck' to verify.$(RESET)" - -.PHONY: outdated -outdated: ## Show outdated packages - @$(PNPM) outdated -r 2>/dev/null || true - -# ============================================================================ -# Documentation -# ============================================================================ - -.PHONY: lint-docs -lint-docs: ## Lint Markdown + YAML in frontend/ (strict) - @command -v markdownlint-cli2 >/dev/null 2>&1 || { echo "$(RED)markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2$(RESET)"; exit 1; } - @command -v yamllint >/dev/null 2>&1 || { echo "$(RED)yamllint not installed. Run: pip install yamllint$(RESET)"; exit 1; } - @markdownlint-cli2 '**/*.md' '#node_modules' - @yamllint -c ../.yamllint.yaml . - -.PHONY: format-docs -format-docs: ## Format Markdown + YAML in frontend/ - @npx --no-install prettier --write '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc --ignore-path .gitignore - -.PHONY: format-check-docs -format-check-docs: ## Check Markdown + YAML formatting - @npx --no-install prettier --check '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc --ignore-path .gitignore diff --git a/frontend/justfile b/frontend/justfile new file mode 100644 index 0000000..4457bb7 --- /dev/null +++ b/frontend/justfile @@ -0,0 +1,199 @@ +# ============================================================================ +# Emailibrium Frontend โ€” React/TypeScript (pnpm + Turborepo) +# ============================================================================ +# +# Quick Start: +# just install - Install dependencies +# just dev - Start Vite dev server +# just build - Build all packages +# just test - Run tests +# +# Invoked from the repo root as: +# just --justfile frontend/justfile --working-directory frontend +# +# `just --list` groups recipes by category (alphabetically); add --unsorted to +# see them in the order they are defined below. +# ============================================================================ + +set shell := ["bash", "-cu"] + +PNPM := "pnpm" +TURBO := "pnpm turbo" +WEB := "apps/web" + +# Colours. The `|| echo ''` fallback keeps these harmless when tput or TERM is +# unavailable (CI, non-tty) โ€” same guard the Makefile used. +BOLD := `tput bold 2>/dev/null || echo ''` +GREEN := `tput setaf 2 2>/dev/null || echo ''` +YELLOW := `tput setaf 3 2>/dev/null || echo ''` +BLUE := `tput setaf 4 2>/dev/null || echo ''` +RED := `tput setaf 1 2>/dev/null || echo ''` +RESET := `tput sgr0 2>/dev/null || echo ''` + +# ============================================================================ +# Default Recipe +# ============================================================================ + +# Show the categorised command listing (default recipe) +@help: + echo "{{ BOLD }}{{ BLUE }}โ•โ•โ• Emailibrium Frontend (TypeScript/React) โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{ RESET }}" + echo "" + echo "{{ BOLD }}Install & Build{{ RESET }}" + echo " install - Install pnpm dependencies" + echo " build - Build all packages (Turborepo)" + echo " dev - Start Vite dev server (port 3000)" + echo " storybook - Start Storybook component explorer" + echo " clean - Clean build artifacts" + echo "" + echo "{{ BOLD }}Test{{ RESET }}" + echo " test - Run unit tests (Vitest)" + echo " test-e2e - Run E2E tests (Playwright)" + echo "" + echo "{{ BOLD }}Lint & Format{{ RESET }}" + echo " lint - Lint TypeScript (ESLint)" + echo " format - Format code (Prettier)" + echo " format-check - Check formatting" + echo " typecheck - TypeScript type check" + echo " deadcode - Check for unused exports" + echo "" + echo "{{ BOLD }}Security{{ RESET }}" + echo " audit - Security audit (pnpm audit)" + echo "" + echo "{{ BOLD }}Dependencies{{ RESET }}" + echo " upgrade - Upgrade deps (within semver)" + echo " upgrade-latest - Upgrade to latest major versions" + echo " outdated - Show outdated packages" + echo "" + echo "{{ BOLD }}Documentation{{ RESET }}" + echo " lint-docs - Lint Markdown + YAML in frontend/" + echo " format-docs - Format Markdown + YAML in frontend/" + +# ============================================================================ +# Install & Build +# ============================================================================ + +# Install pnpm dependencies +[group('Install & Build')] +@install: + {{ PNPM }} install + +# Build all packages (Turborepo) +[group('Install & Build')] +@build: + {{ TURBO }} build + +# Start Vite dev server (port 3000) +[group('Install & Build')] +@dev: + cd {{ WEB }} && npx vite + +# Start Storybook component explorer (port 6006) +[group('Install & Build')] +@storybook: + cd {{ WEB }} && npx storybook dev -p 6006 + +# Clean build artifacts +[group('Install & Build')] +@clean: + rm -rf {{ WEB }}/dist {{ WEB }}/storybook-static node_modules/.cache .turbo + +# ============================================================================ +# Test +# ============================================================================ + +# Run unit tests (Vitest via Turborepo) โ€” fails the recipe when tests fail +[group('Test')] +@test: + {{ TURBO }} test + +# Run E2E tests (Playwright) +[group('Test')] +@test-e2e: + cd {{ WEB }} && npx playwright test + +# ============================================================================ +# Lint & Format +# ============================================================================ + +# Lint TypeScript (ESLint โ€” strict) +[group('Lint & Format')] +@lint: + cd {{ WEB }} && npx --no-install eslint src/ --ext .ts,.tsx + +# Format code (Prettier) +[group('Lint & Format')] +@format: + cd {{ WEB }} && npx prettier --write 'src/**/*.{ts,tsx}' + +# Check formatting without writing +[group('Lint & Format')] +@format-check: + cd {{ WEB }} && npx prettier --check 'src/**/*.{ts,tsx}' + +# TypeScript type check +[group('Lint & Format')] +@typecheck: + cd {{ WEB }} && npx tsc --noEmit + +# The `|| echo` fallback is deliberate: ts-prune is an optional dev tool, so a +# missing binary prints install instructions instead of failing the build. +[doc('Check for unused exports (ts-prune)')] +[group('Lint & Format')] +@deadcode: + cd {{ WEB }} && npx ts-prune src/ 2>/dev/null || echo "{{ YELLOW }}ts-prune not installed. Run: pnpm add -Dw ts-prune{{ RESET }}" + +# ============================================================================ +# Security +# ============================================================================ + +# Security audit (pnpm audit) โ€” fails the recipe when vulnerabilities are found +[group('Security')] +@audit: + {{ PNPM }} audit --prod + +# ============================================================================ +# Dependency Management +# ============================================================================ + +# Upgrade dependencies (within semver) +[group('Dependencies')] +@upgrade: + {{ PNPM }} update -r + echo "{{ GREEN }}Frontend packages upgraded.{{ RESET }}" + +# Upgrade to latest major versions +[group('Dependencies')] +@upgrade-latest: + {{ PNPM }} update -r --latest + echo "{{ GREEN }}Done. Run 'just typecheck' to verify.{{ RESET }}" + +# The `|| true` is deliberate: `pnpm outdated` exits non-zero merely because +# updates exist, so this informational listing must not be treated as a gate. +[doc('Show outdated packages')] +[group('Dependencies')] +@outdated: + {{ PNPM }} outdated -r 2>/dev/null || true + +# ============================================================================ +# Documentation +# ============================================================================ + +# Lint Markdown + YAML in frontend/ (strict) +[group('Documentation')] +lint-docs: + #!/usr/bin/env bash + set -euo pipefail + command -v markdownlint-cli2 >/dev/null 2>&1 || { echo "{{ RED }}markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2{{ RESET }}"; exit 1; } + command -v yamllint >/dev/null 2>&1 || { echo "{{ RED }}yamllint not installed. Run: pip install yamllint{{ RESET }}"; exit 1; } + markdownlint-cli2 '**/*.md' '#node_modules' + yamllint -c ../.yamllint.yaml . + +# Format Markdown + YAML in frontend/ +[group('Documentation')] +@format-docs: + npx --no-install prettier --write '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc --ignore-path .gitignore + +# Check Markdown + YAML formatting +[group('Documentation')] +@format-check-docs: + npx --no-install prettier --check '**/*.md' '**/*.{yaml,yml}' --config ../.prettierrc --ignore-path .gitignore diff --git a/justfile b/justfile new file mode 100644 index 0000000..9cbe466 --- /dev/null +++ b/justfile @@ -0,0 +1,669 @@ +# ============================================================================ +# Emailibrium โ€” Root justfile +# ============================================================================ +# Delegates to backend/justfile and frontend/justfile. +# Provides cross-cutting recipes for CI, Docker, releases, and docs. +# +# Quick Start: +# just - Show all available recipes (help) +# just --list - Terse auto-generated recipe list +# just install - Install all dependencies +# just dev - Start full stack (native) +# just docker-up-dev - Start full stack (Docker) +# just ci - Run full CI pipeline +# just VERSION=x.y.z release - Tag and release +# ============================================================================ + +# ============================================================================ +# Settings, Variables and Configuration +# ============================================================================ + +# Match the Makefile's `SHELL := /bin/bash` โ€” several recipes use bash-isms +# ([[ ]], &>, ${!var}). just's default shell is `sh -cu`, which would break them. +set shell := ["bash", "-c"] + +BACKEND_DIR := "backend" +FRONTEND_DIR := "frontend" + +COMPOSE := "docker compose" +COMPOSE_DEV := COMPOSE + " -f docker-compose.yml -f docker-compose.dev.yml" + +# Delegation prefixes. just does not have make's `-C`; --justfile picks the file +# and --working-directory sets the cwd the sub-recipes run in. +BACKEND := "just --justfile " + BACKEND_DIR + "/justfile --working-directory " + BACKEND_DIR +FRONTEND := "just --justfile " + FRONTEND_DIR + "/justfile --working-directory " + FRONTEND_DIR + +# Empty when lychee is not on PATH (link checking is optional โ€” see links-check). +LYCHEE := `command -v lychee 2>/dev/null || echo ""` + +# Colors. `|| echo ''` is a terminal-capability probe, not an error mask: tput +# fails when TERM is unset (CI, pipes) and we degrade to uncolored output. +BOLD := `tput bold 2>/dev/null || echo ''` +GREEN := `tput setaf 2 2>/dev/null || echo ''` +YELLOW := `tput setaf 3 2>/dev/null || echo ''` +BLUE := `tput setaf 4 2>/dev/null || echo ''` +RED := `tput setaf 1 2>/dev/null || echo ''` +RESET := `tput sgr0 2>/dev/null || echo ''` + +# Argument-style variables, so `just VERSION=0.1.0 release` works like +# `make release VERSION=0.1.0`. The recipes below also accept the version / +# model positionally, and tolerate a literal `VERSION=`/`MODEL=` prefix. +VERSION := "" +MODEL := "" + +# find with -prune avoids traversing multi-GB Rust target/ and node_modules/ dirs +# (prettier's own glob walker enters all dirs before filtering via .prettierignore) +# Keep this list in sync with .markdownlint-cli2.jsonc's `ignores` โ€” otherwise +# format-check-md walks agent-tool scaffolding (.agents/, .codex/, .optimizer/) +# that lint-md correctly skips, and the two checks disagree about the same file. +PRUNE_DIRS := '\( -name node_modules -o -name target -o -name ruvector -o -name .claude -o -name .claude-flow -o -name .git -o -name .agentic-qe -o -name .swarm -o -name .beads -o -name .agents -o -name .codex -o -name .optimizer -o -name .git-rewrite -o -name dist -o -name storybook-static -o -name coverage \) -prune' + +# ============================================================================ +# Default Recipe +# ============================================================================ + +# Show all available recipes, grouped +[group('help')] +help: + #!/usr/bin/env bash + set -euo pipefail + cat <<'HELP' + {{BOLD}}{{BLUE}}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—{{RESET}} + {{BOLD}}{{BLUE}}โ•‘ Emailibrium justfile โ•‘{{RESET}} + {{BOLD}}{{BLUE}}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + + {{BOLD}}Quick Start:{{RESET}} + just setup - Guided first-time setup wizard + just install - Install all dependencies + just dev - Start backend + frontend (native) + just dev-llm - Start with built-in LLM (llama.cpp) + just models - Show available LLM models + just embedding-models - Show available embedding models + just download-model - Download a model (MODEL=) + just docker-up-dev - Start full stack (Docker) + just ci - Run full CI pipeline + just test - Run all tests + + {{BOLD}}{{BLUE}}โ•โ•โ• Setup & Onboarding โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + setup - Guided first-time setup wizard + setup-prereqs - Check all prerequisites + setup-secrets - Generate/configure secrets + setup-ai - Configure AI providers + setup-docker - Set up Docker environment + setup-validate - Validate entire setup + + {{BOLD}}{{BLUE}}โ•โ•โ• Install & Build โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + install - Install all dependencies (backend + frontend) + build - Build everything + dev - Start full stack dev servers (native) + dev-llm - Start with built-in LLM (llama.cpp) + clean - Clean all build artifacts + clean-data - Remove all local data (DB, vectors) + clean-all - Clean build artifacts + all local data + + {{BOLD}}{{BLUE}}โ•โ•โ• AI & Models โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + models - Show available LLM models + embedding-models - Show available embedding models + download-model MODEL=x - Download a specific model + download-models - Download AI models (ONNX + GGUF) + diagnose - Show AI configuration diagnostics + + {{BOLD}}{{BLUE}}โ•โ•โ• Test โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + test - Run all tests (backend + frontend) + + {{BOLD}}{{BLUE}}โ•โ•โ• Lint & Format โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + lint - Lint everything (code + docs) + format - Format everything (code + docs) + format-check - Check formatting (no changes) + typecheck - TypeScript type check + + {{BOLD}}{{BLUE}}โ•โ•โ• Security & Quality โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + audit - Security audit all dependencies + deadcode - Check for dead code + ci - Full CI pipeline + ci-full - CI + link checking + + {{BOLD}}{{BLUE}}โ•โ•โ• Dependency Management โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + upgrade - Upgrade all deps (within semver) + outdated - Show outdated deps (no changes) + + {{BOLD}}{{BLUE}}โ•โ•โ• Documentation โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + lint-md - Lint Markdown files + lint-yaml - Lint YAML files + links-check - Check internal links in Markdown + links-check-external - Check external links (slow) + links-check-all - Check all links + + {{BOLD}}{{BLUE}}โ•โ•โ• Docker โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + docker-up - Start production stack + docker-up-dev - Start dev stack (hot-reload) + docker-down - Stop all containers + docker-down-volumes - Stop + remove volumes (DESTROYS DATA) + docker-restart - Restart all containers + docker-build - Build Docker images + docker-build-no-cache - Build images without cache + docker-logs - Tail all container logs + docker-logs-backend - Tail backend logs + docker-logs-frontend - Tail frontend logs + docker-ps - Show container status + docker-exec-backend - Shell into backend container + docker-exec-frontend - Shell into frontend container + docker-health - Health check all containers + docker-clean - Prune dangling Docker artifacts + docker-secrets - Generate dev secrets + + {{BOLD}}{{BLUE}}โ•โ•โ• Release โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•{{RESET}} + release-check - Pre-release CI validation + release-tag VERSION=x.y.z - Create annotated tag + release-push - Push latest tag to trigger release + release VERSION=x.y.z - Full release (check + tag + push) + changelog - Regenerate CHANGELOG.md + + Run '{{BOLD}}cd backend && just --list{{RESET}}' or '{{BOLD}}cd frontend && just --list{{RESET}}' for layer-specific recipes. + HELP + +# ============================================================================ +# Setup & Onboarding +# ============================================================================ + +# Guided first-time setup wizard +[group('setup')] +setup: + @bash scripts/setup.sh + +# Check all prerequisites +[group('setup')] +setup-prereqs: + @bash scripts/setup-prereqs.sh + +# Generate/configure secrets +[group('setup')] +setup-secrets: + @bash scripts/setup-secrets.sh + +# Configure AI providers +[group('setup')] +setup-ai: + @bash scripts/setup-ai.sh + +# Set up Docker environment +[group('setup')] +setup-docker: + @bash scripts/setup-docker.sh + +# Validate entire setup +[group('setup')] +setup-validate: + @bash scripts/setup-validate.sh + +# ============================================================================ +# AI & Models +# ============================================================================ + +# Download AI models (ONNX embedding + GGUF LLM) +[group('ai')] +download-models: + #!/usr/bin/env bash + set -euo pipefail + echo "{{BOLD}}{{BLUE}}Downloading AI models...{{RESET}}" + echo "{{GREEN}}Step 1:{{RESET}} ONNX embedding model" + # The Makefile hid the real error here (`2>/dev/null || echo hint`) and still + # exited 0. Keep the hint, but surface stderr and fail honestly. + if ! (cd {{BACKEND_DIR}} && cargo run -- --download-models); then + echo " {{YELLOW}}Backend not built. Run 'just build' first.{{RESET}}" >&2 + exit 1 + fi + echo "{{GREEN}}Step 2:{{RESET}} GGUF LLM model (qwen2.5-0.5b-q4km)" + if ! (cd {{FRONTEND_DIR}}/apps/web && npx tsx ../../../scripts/models.ts download --default); then + echo " {{YELLOW}}Frontend not installed. Run 'just install' first.{{RESET}}" >&2 + exit 1 + fi + echo "{{GREEN}}Done.{{RESET}} Models cached for offline use." + +# Show AI configuration diagnostics +[group('ai')] +diagnose: + #!/usr/bin/env bash + set -euo pipefail + echo "{{BOLD}}{{BLUE}}Emailibrium AI Diagnostics{{RESET}}" + echo "โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€" + echo "" + echo "{{BOLD}}Embedding:{{RESET}}" + if [[ -d "{{BACKEND_DIR}}/.fastembed_cache" ]]; then + echo " Provider: ONNX (all-MiniLM-L6-v2)" + echo " Status: {{GREEN}}cached{{RESET}} ($(du -sh {{BACKEND_DIR}}/.fastembed_cache 2>/dev/null | cut -f1))" + else + echo " Provider: ONNX (all-MiniLM-L6-v2)" + echo " Status: {{YELLOW}}not cached (downloads on first use){{RESET}}" + fi + echo "" + echo "{{BOLD}}Generative (LLM):{{RESET}}" + CACHE="$HOME/.emailibrium/models/llm" + if [[ -d "$CACHE" ]] && find "$CACHE" -name "*.gguf" -print -quit 2>/dev/null | grep -q .; then + LLM_MODEL=$(find "$CACHE" -name "*.gguf" -print -quit 2>/dev/null | xargs basename) + SIZE=$(du -sh "$CACHE" 2>/dev/null | cut -f1) + echo " Provider: builtin ($LLM_MODEL)" + echo " Status: {{GREEN}}cached{{RESET}} ($SIZE)" + else + echo " Provider: builtin (qwen2.5-0.5b-q4km)" + echo " Status: {{YELLOW}}not cached{{RESET}}" + echo " Fix: just download-models" + fi + echo "" + echo "{{BOLD}}Ollama:{{RESET}}" + if command -v ollama &>/dev/null && ollama list &>/dev/null 2>&1; then + echo " Status: {{GREEN}}running{{RESET}}" + elif command -v ollama &>/dev/null; then + echo " Status: {{YELLOW}}installed but not running{{RESET}}" + else + echo " Status: not installed (optional)" + fi + echo "" + echo "{{BOLD}}Cloud APIs:{{RESET}}" + for var in EMAILIBRIUM_OPENAI_API_KEY EMAILIBRIUM_ANTHROPIC_API_KEY EMAILIBRIUM_GEMINI_API_KEY; do + name=$(echo "$var" | sed 's/EMAILIBRIUM_//;s/_API_KEY//') + if [[ -n "${!var:-}" ]]; then + echo " $name: {{GREEN}}configured{{RESET}}" + else + echo " $name: not configured" + fi + done + echo "" + echo "{{BOLD}}Database:{{RESET}}" + if [[ -f "{{BACKEND_DIR}}/emailibrium-dev.db" ]]; then + echo " Status: {{GREEN}}exists{{RESET}} ($(du -sh {{BACKEND_DIR}}/emailibrium-dev.db 2>/dev/null | cut -f1))" + else + echo " Status: not created yet (created on first run)" + fi + +# Show available LLM models with hardware recommendations +[group('ai')] +models: + @{{BACKEND}} models + +# Show available embedding models +[group('ai')] +embedding-models: + @{{BACKEND}} embedding-models + +# Download a model (e.g. just download-model qwen3-8b-q4km) +[group('ai')] +download-model model=MODEL: + #!/usr/bin/env bash + set -euo pipefail + # Accept `just download-model qwen3-8b-q4km`, `just download-model MODEL=qwen3-8b-q4km` + # and `just MODEL=qwen3-8b-q4km download-model`. + m="{{model}}"; m="${m#MODEL=}" + if [[ -z "$m" ]]; then + echo "{{YELLOW}}Usage: just download-model MODEL= (see: just models){{RESET}}" >&2 + exit 1 + fi + {{BACKEND}} download-model "$m" + +# ============================================================================ +# Install & Build +# ============================================================================ + +# Install all dependencies (backend `build` is its dependency-fetch step) +[group('build')] +install: + @{{BACKEND}} build + @{{FRONTEND}} install + +# Build everything +[group('build')] +build: + @{{BACKEND}} build + @{{FRONTEND}} build + +# Start full stack dev servers (native, loads secrets/dev/ as env vars) +[group('build')] +dev: + #!/usr/bin/env bash + set -euo pipefail + echo "{{GREEN}}Backend: http://localhost:8080 Frontend: http://localhost:3000{{RESET}}" + # Missing secret files yield empty values by design (dev works before + # `just setup-secrets`); an unreadable file still fails loudly under set -e. + secret() { local f="secrets/dev/$1"; if [[ -f "$f" ]]; then cat "$f"; else printf ''; fi; } + export EMAILIBRIUM_GOOGLE_CLIENT_ID="$(secret google_client_id)" + export EMAILIBRIUM_GOOGLE_CLIENT_SECRET="$(secret google_client_secret)" + export EMAILIBRIUM_MICROSOFT_CLIENT_ID="$(secret microsoft_client_id)" + export EMAILIBRIUM_MICROSOFT_CLIENT_SECRET="$(secret microsoft_client_secret)" + export JWT_SECRET="$(secret jwt_secret)" + export EMAILIBRIUM_ENCRYPTION_MASTER_PASSWORD="$(secret oauth_encryption_key)" + export RATE_LIMIT_PRESET=development + trap 'kill 0' INT TERM EXIT + {{BACKEND}} dev & + {{FRONTEND}} dev & + wait + +# Start full stack with built-in LLM (downloads ~350MB model on first run) +[group('build')] +dev-llm: + #!/usr/bin/env bash + set -euo pipefail + echo "{{GREEN}}Backend (LLM): http://localhost:8080 Frontend: http://localhost:3000{{RESET}}" + secret() { local f="secrets/dev/$1"; if [[ -f "$f" ]]; then cat "$f"; else printf ''; fi; } + export EMAILIBRIUM_GOOGLE_CLIENT_ID="$(secret google_client_id)" + export EMAILIBRIUM_GOOGLE_CLIENT_SECRET="$(secret google_client_secret)" + export EMAILIBRIUM_MICROSOFT_CLIENT_ID="$(secret microsoft_client_id)" + export EMAILIBRIUM_MICROSOFT_CLIENT_SECRET="$(secret microsoft_client_secret)" + export JWT_SECRET="$(secret jwt_secret)" + export EMAILIBRIUM_ENCRYPTION_MASTER_PASSWORD="$(secret oauth_encryption_key)" + export RATE_LIMIT_PRESET=development + trap 'kill 0' INT TERM EXIT + {{BACKEND}} dev-llm & + {{FRONTEND}} dev & + wait + +# Clean all build artifacts +[group('build')] +clean: + @{{BACKEND}} clean + @{{FRONTEND}} clean + +# Remove all local data (DB, vectors) โ€” fresh start +[group('build')] +clean-data: + @{{BACKEND}} clean-data + +# Clean build artifacts + all local data +[group('build')] +clean-all: + @{{BACKEND}} clean-all + @{{FRONTEND}} clean + +# ============================================================================ +# Test +# ============================================================================ + +# Run all tests +[group('test')] +test: + @{{BACKEND}} test + @{{FRONTEND}} test + +# ============================================================================ +# Lint & Format +# ============================================================================ + +# Lint everything (code + docs) +[group('lint')] +lint: lint-docs + @{{BACKEND}} lint + @{{FRONTEND}} lint + +# Format everything (code + docs) +[group('lint')] +format: format-docs + @{{BACKEND}} format + @{{FRONTEND}} format + +# Check formatting (no changes) +[group('lint')] +format-check: format-check-docs + @{{BACKEND}} format-check + @{{FRONTEND}} format-check + +# Type check (frontend) +[group('lint')] +typecheck: + @{{FRONTEND}} typecheck + +# ============================================================================ +# Security & Quality +# ============================================================================ + +# Security audit all dependencies +[group('security')] +audit: + @{{BACKEND}} audit + @{{FRONTEND}} audit + +# Check for dead code +[group('security')] +deadcode: + @{{BACKEND}} deadcode + @{{FRONTEND}} deadcode + +# Full CI pipeline +[group('security')] +ci: format-check lint typecheck test + +# Full CI + link checking +[group('security')] +ci-full: ci links-check + +# ============================================================================ +# Dependency Management +# ============================================================================ + +# Upgrade all dependencies (within semver) +[group('deps')] +upgrade: + @{{BACKEND}} upgrade + @{{FRONTEND}} upgrade + +# Show outdated dependencies (no changes) +[group('deps')] +outdated: + @{{BACKEND}} outdated + @{{FRONTEND}} outdated + +# ============================================================================ +# Documentation (Markdown, YAML, Links) +# ============================================================================ + +# Lint Markdown files (strict โ€” fails on errors or missing tool) +[group('docs')] +lint-md: + @echo "{{GREEN}}Linting Markdown...{{RESET}}" + @command -v markdownlint-cli2 >/dev/null 2>&1 || { echo "{{RED}}markdownlint-cli2 not installed. Run: npm i -g markdownlint-cli2{{RESET}}"; exit 1; } + @markdownlint-cli2 '**/*.md' '#**/node_modules' '#**/target' '#.claude/worktrees/**' '#ruvector/**' + +# Lint YAML files (strict โ€” fails on errors or missing tool) +[group('docs')] +lint-yaml: + @echo "{{GREEN}}Linting YAML...{{RESET}}" + @command -v yamllint >/dev/null 2>&1 || { echo "{{RED}}yamllint not installed. Run: pip install yamllint{{RESET}}"; exit 1; } + @find . \( -name node_modules -o -name target -o -name ruvector -o -name .claude -o -name .claude-flow \) -prune -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs -r yamllint -c .yamllint.yaml + +# Lint all docs (Markdown + YAML) +[group('docs')] +lint-docs: lint-md lint-yaml + +# Format Markdown files +[group('docs')] +format-md: + @find . {{PRUNE_DIRS}} -o -name '*.md' -print | xargs npx prettier --write --no-error-on-unmatched-pattern + +# Format YAML files +[group('docs')] +format-yaml: + @find . {{PRUNE_DIRS}} -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs npx prettier --write --no-error-on-unmatched-pattern + +# Format docs (Markdown + YAML) +[group('docs')] +format-docs: format-md format-yaml + +# Check Markdown formatting (no changes) +[group('docs')] +format-check-md: + @find . {{PRUNE_DIRS}} -o -name '*.md' -print | xargs npx prettier --check --no-error-on-unmatched-pattern + +# Check YAML formatting (no changes) +[group('docs')] +format-check-yaml: + @find . {{PRUNE_DIRS}} -o \( -name '*.yaml' -o -name '*.yml' \) ! -name 'pnpm-lock.yaml' -print | xargs npx prettier --check --no-error-on-unmatched-pattern + +# Check docs formatting (Markdown + YAML) +[group('docs')] +format-check-docs: format-check-md format-check-yaml + +# Check internal links in Markdown +[group('docs')] +links-check: + #!/usr/bin/env bash + set -euo pipefail + echo "{{GREEN}}Checking local file links...{{RESET}}" + if [ -n "{{LYCHEE}}" ]; then + {{LYCHEE}} --scheme file --include-fragments --config .lychee.toml '**/*.md' + else + echo "{{YELLOW}}lychee not installed. Run: cargo install lychee{{RESET}}" + fi + +# Check external links (may take minutes) +[group('docs')] +links-check-external: + #!/usr/bin/env bash + set -euo pipefail + echo "{{GREEN}}Checking external links...{{RESET}}" + if [ -n "{{LYCHEE}}" ]; then + {{LYCHEE}} --scheme https --scheme http --config .lychee.toml '**/*.md' + else + echo "{{YELLOW}}lychee not installed. Run: cargo install lychee{{RESET}}" + fi + +# Check all links (internal + external) +[group('docs')] +links-check-all: links-check links-check-external + +# ============================================================================ +# Docker +# ============================================================================ + +# Start production stack +[group('docker')] +docker-up: + @echo "{{GREEN}}Starting Emailibrium stack...{{RESET}}" + @{{COMPOSE}} up -d + @echo "{{GREEN}}Backend: http://localhost:8080 Frontend: http://localhost:3000{{RESET}}" + +# Start dev stack (hot-reload) +[group('docker')] +docker-up-dev: + @echo "{{GREEN}}Starting Emailibrium dev stack...{{RESET}}" + @{{COMPOSE_DEV}} up -d + @echo "{{GREEN}}Backend: http://localhost:8080 Frontend: http://localhost:3000{{RESET}}" + +# Stop and remove containers +[group('docker')] +docker-down: + @{{COMPOSE}} down + +# Stop + remove volumes (DESTROYS DATA) +[group('docker')] +docker-down-volumes: + @{{COMPOSE}} down -v + +# Restart all containers +[group('docker')] +docker-restart: docker-down docker-up + +# Build Docker images +[group('docker')] +docker-build: + @{{COMPOSE}} build + +# Build images without cache +[group('docker')] +docker-build-no-cache: + @{{COMPOSE}} build --no-cache + +# Tail logs from all containers +[group('docker')] +docker-logs: + @{{COMPOSE}} logs -f + +# Tail backend logs +[group('docker')] +docker-logs-backend: + @{{COMPOSE}} logs -f backend + +# Tail frontend logs +[group('docker')] +docker-logs-frontend: + @{{COMPOSE}} logs -f frontend + +# Show running containers +[group('docker')] +docker-ps: + @{{COMPOSE}} ps + +# Shell into backend container +[group('docker')] +docker-exec-backend: + @{{COMPOSE}} exec backend sh + +# Shell into frontend container +[group('docker')] +docker-exec-frontend: + @{{COMPOSE}} exec frontend sh + +# Health check all containers +[group('docker')] +docker-health: + @{{COMPOSE}} ps --format "table {{{{.Name}}\t{{{{.Status}}\t{{{{.Ports}}" + +# Prune dangling Docker artifacts +[group('docker')] +docker-clean: + # `|| true` retained: pruning is best-effort cleanup scoped to this project's + # label, and a no-op/absent-daemon prune must not fail the recipe. + @docker system prune -f --filter "label=com.docker.compose.project=emailibrium" 2>/dev/null || true + +# Generate development secrets +[group('docker')] +docker-secrets: + @mkdir -p secrets/dev + @openssl rand -base64 32 > secrets/dev/jwt_secret + @openssl rand -base64 32 > secrets/dev/oauth_encryption_key + @echo "postgres://emailibrium:devpass@postgres:5432/emailibrium" > secrets/dev/database_url + @echo "devpass" > secrets/dev/db_password + @chmod 600 secrets/dev/* + @echo "{{GREEN}}Secrets generated in secrets/dev/{{RESET}}" + +# ============================================================================ +# Release +# ============================================================================ + +# Pre-release CI validation +[group('release')] +release-check: ci + @echo "{{GREEN}}Release checks passed. Ready to tag.{{RESET}}" + +# Tag a release (usage: just release-tag VERSION=0.1.0) +[group('release')] +release-tag version=VERSION: + #!/usr/bin/env bash + set -euo pipefail + v="{{version}}"; v="${v#VERSION=}" + if [ -z "$v" ]; then echo "{{YELLOW}}Usage: just release-tag VERSION=0.1.0{{RESET}}"; exit 1; fi + git tag -a "v$v" -m "Release v$v" + echo "{{GREEN}}Tagged v$v. Push with: git push origin v$v{{RESET}}" + +# Push latest tag to trigger release workflow +[group('release')] +release-push: + #!/usr/bin/env bash + set -euo pipefail + # `|| TAG=""` is not error masking: "no tags yet" is the expected first-run + # state, and the -z check below turns it into an explicit exit 1. + TAG=$(git describe --tags --abbrev=0 2>/dev/null) || TAG="" + if [ -z "$TAG" ]; then echo "{{YELLOW}}No tags found.{{RESET}}"; exit 1; fi + echo "{{GREEN}}Pushing $TAG to origin...{{RESET}}" + git push origin "$TAG" + +# Cut a release (bumps versions, updates CHANGELOG, commits, tags, pushes). Usage: just release VERSION=0.1.0 +[group('release')] +release version=VERSION: + #!/usr/bin/env bash + set -euo pipefail + v="{{version}}"; v="${v#VERSION=}" + [ -n "$v" ] || { echo "usage: just release VERSION=X.Y.Z" >&2; exit 1; } + ./scripts/release.sh "$v" + +# Regenerate CHANGELOG.md from git history using git-cliff +[group('release')] +changelog: + git-cliff --output CHANGELOG.md diff --git a/scripts/setup-docker.sh b/scripts/setup-docker.sh index 94d8f01..b4d09ef 100755 --- a/scripts/setup-docker.sh +++ b/scripts/setup-docker.sh @@ -132,9 +132,9 @@ if [[ "$(echo "$start_choice" | tr '[:upper:]' '[:lower:]')" == "y" ]]; then echo " Backend: http://localhost:8080" echo " Frontend: http://localhost:3000" echo "" - echo " Logs: make docker-logs" - echo " Stop: make docker-down" + echo " Logs: just docker-logs" + echo " Stop: just docker-down" else echo "" - echo " ${YELLOW}Skipped.${RESET} Start later with: make docker-up" + echo " ${YELLOW}Skipped.${RESET} Start later with: just docker-up" fi diff --git a/scripts/setup-prereqs.sh b/scripts/setup-prereqs.sh index 7d01367..faf047c 100755 --- a/scripts/setup-prereqs.sh +++ b/scripts/setup-prereqs.sh @@ -40,7 +40,7 @@ check_tool() { node) version=$(node --version 2>/dev/null | sed 's/^v//') ;; pnpm) version=$(pnpm --version 2>/dev/null) ;; docker) version=$(docker --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) ;; - make) version=$(make --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) ;; + just) version=$(just --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) ;; esac if [[ -z "$version" ]]; then @@ -100,7 +100,7 @@ check_tool "Node.js" "node" "22.12" "brew install node@22 OR https:/ check_tool "pnpm" "pnpm" "10.32" "corepack enable && corepack prepare pnpm@latest --activate" check_tool "Docker" "docker" "24.0" "https://docs.docker.com/get-docker/" check_docker_compose -check_tool "Make" "make" "3.81" "Xcode CLI: xcode-select --install OR brew install make" +check_tool "just" "just" "1.14" "brew install just OR cargo install just OR mise use -g just" echo "" echo "${BOLD}Submodules:${RESET}" diff --git a/scripts/setup-secrets.sh b/scripts/setup-secrets.sh index 97ae36e..c92881d 100755 --- a/scripts/setup-secrets.sh +++ b/scripts/setup-secrets.sh @@ -115,7 +115,7 @@ else write_secret "google_client_secret" "$google_secret" echo " ${GREEN}[saved]${RESET} Google OAuth credentials" else - echo " ${YELLOW}[skipped]${RESET} Google OAuth (you can configure later with: make setup-secrets)" + echo " ${YELLOW}[skipped]${RESET} Google OAuth (you can configure later with: just setup-secrets)" # Write placeholder so Docker Compose doesn't fail on missing files if [[ ! -f "$SECRETS_DIR/google_client_id" ]]; then write_secret "google_client_id" "placeholder-configure-later" @@ -146,7 +146,7 @@ else write_secret "microsoft_client_secret" "$ms_secret" echo " ${GREEN}[saved]${RESET} Microsoft OAuth credentials" else - echo " ${YELLOW}[skipped]${RESET} Microsoft OAuth (you can configure later with: make setup-secrets)" + echo " ${YELLOW}[skipped]${RESET} Microsoft OAuth (you can configure later with: just setup-secrets)" if [[ ! -f "$SECRETS_DIR/microsoft_client_id" ]]; then write_secret "microsoft_client_id" "placeholder-configure-later" fi diff --git a/scripts/setup-validate.sh b/scripts/setup-validate.sh index b8546a7..c34deab 100755 --- a/scripts/setup-validate.sh +++ b/scripts/setup-validate.sh @@ -137,7 +137,7 @@ if command -v docker &>/dev/null && docker info &>/dev/null 2>&1; then echo " Container status:" (cd "$PROJECT_ROOT" && docker compose ps --format "table {{.Name}}\t{{.Status}}" 2>/dev/null) | sed 's/^/ /' else - check "Docker containers" "warn" "no containers running (start with: make docker-up)" + check "Docker containers" "warn" "no containers running (start with: just docker-up)" fi else check "Docker" "warn" "Docker not available" @@ -182,11 +182,11 @@ echo "${BOLD}Summary: $PASS passed, $WARN warnings, $FAIL failed (of $TOTAL chec echo "" if [[ $FAIL -eq 0 && $WARN -eq 0 ]]; then - echo "${GREEN}${BOLD}Everything looks good! Run 'make dev' or 'make docker-up' to start.${RESET}" + echo "${GREEN}${BOLD}Everything looks good! Run 'just dev' or 'just docker-up' to start.${RESET}" elif [[ $FAIL -eq 0 ]]; then echo "${GREEN}${BOLD}Core setup is complete.${RESET} Warnings above are optional features." - echo "Run 'make dev' or 'make docker-up' to start." + echo "Run 'just dev' or 'just docker-up' to start." else echo "${RED}${BOLD}Some checks failed.${RESET} Address the failures above before running." - echo "Re-run: make setup-validate" + echo "Re-run: just setup-validate" fi diff --git a/scripts/setup.sh b/scripts/setup.sh index 681a8c5..7e8c3a7 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -104,7 +104,7 @@ menu() { run_step "setup-docker.sh" "Step 4: Docker Environment" run_step "setup-validate.sh" "Step 5: Validation" echo "" - echo "${BOLD}${GREEN}Setup complete! Run 'make dev' to start developing.${RESET}" + echo "${BOLD}${GREEN}Setup complete! Run 'just dev' to start developing.${RESET}" ;; 1) run_step "setup-prereqs.sh" "Prerequisites" ;; 2) run_step "setup-secrets.sh" "Secrets" ;; From 4aa1c51c84692d0b5a2d7ab265be1ce5759034ad Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 15:00:59 -0700 Subject: [PATCH 17/24] =?UTF-8?q?chore(autopilot:ci-build-optimization):?= =?UTF-8?q?=20cross-phase=20optimization=20=E2=80=94=20gate=20PASSED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/actions/setup-rust/action.yml | 29 +++++++++++++++++++++ .github/workflows/ci.yml | 37 ++++----------------------- .github/workflows/release.yml | 9 +------ 3 files changed, 35 insertions(+), 40 deletions(-) create mode 100644 .github/actions/setup-rust/action.yml diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 0000000..9d6837a --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,29 @@ +name: Setup Rust +description: >- + Install the exact Rust toolchain pinned in backend/rust-toolchain.toml. + Extracted during the ci-build-optimization pipeline's cross-phase optimization + pass: the resolve-then-install pair was duplicated in six places (five jobs in + ci.yml, one in release.yml), so bumping the MSRV meant editing six blocks and + any one of them could silently drift. + +inputs: + components: + description: Comma-separated rustup components (e.g. "rustfmt" or "rustfmt, clippy"). + required: false + default: "" + +runs: + using: composite + steps: + # backend/rust-toolchain.toml is the single source of truth for the Rust + # version. Installing a bare @stable drifts from it, and rustup would then + # silently install the pinned toolchain a SECOND time on first cargo use + # inside backend/ โ€” so read the pin and install exactly it, once. + - name: Resolve pinned toolchain + id: rust + shell: bash + run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ steps.rust.outputs.channel }} + components: ${{ inputs.components }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74a8106..8f76c68 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,16 +29,8 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - # backend/rust-toolchain.toml is the single source of truth for the Rust - # version. Installing bare @stable here drifts from it, and rustup would - # silently install the pinned toolchain a second time on first cargo use - # inside backend/ โ€” so read the pin and install exactly it, once. - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master + - uses: ./.github/actions/setup-rust with: - toolchain: ${{ steps.rust.outputs.channel }} components: rustfmt - name: Check formatting working-directory: backend @@ -51,12 +43,8 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master + - uses: ./.github/actions/setup-rust with: - toolchain: ${{ steps.rust.outputs.channel }} components: clippy - uses: Swatinem/rust-cache@v2 with: @@ -90,12 +78,7 @@ jobs: # ubuntu-*-arm images HEAVY_RUNNER is expected to point at. - name: Install lld run: sudo apt-get update && sudo apt-get install -y lld - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ steps.rust.outputs.channel }} + - uses: ./.github/actions/setup-rust # Pin the version: the archive producer and consumer must be the same # nextest, and an unpinned install lets a release landing between the two # jobs mismatch them (nextest's own archiving docs call this out). @@ -144,12 +127,7 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ steps.rust.outputs.channel }} + - uses: ./.github/actions/setup-rust - uses: Swatinem/rust-cache@v2 with: workspaces: backend @@ -177,12 +155,7 @@ jobs: - uses: actions/checkout@v7 with: submodules: true - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ steps.rust.outputs.channel }} + - uses: ./.github/actions/setup-rust # Must match rust-build's pinned version exactly โ€” the archive is produced # and consumed by nextest, and a version skew between the two jobs is a # confusing failure mode nextest's archiving docs warn about. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c52a19b..cedd104 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,15 +72,8 @@ jobs: with: submodules: true - # Backend - # backend/rust-toolchain.toml is the single source of truth for the Rust - # version โ€” read the pin rather than installing bare @stable, which drifts. - - name: Resolve pinned toolchain - id: rust - run: echo "channel=$(sed -n 's/^channel *= *"\(.*\)"/\1/p' backend/rust-toolchain.toml)" >> "$GITHUB_OUTPUT" - - uses: dtolnay/rust-toolchain@master + - uses: ./.github/actions/setup-rust with: - toolchain: ${{ steps.rust.outputs.channel }} components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 with: From fa1eda827b9ea5442a164750e8af68f181d1abd4 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Fri, 31 Jul 2026 15:06:23 -0700 Subject: [PATCH 18/24] =?UTF-8?q?fix(autopilot:ci-build-optimization):=20i?= =?UTF-8?q?ntegration=20court=20REMAND=20=E2=80=94=20charges=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cross-vendor integration court (GPT jury, Claude writer โ€” the first properly-seated panel in this pipeline) returned REMAND with two charges. Both were verified against the repo and both were real; both had been missed by the author's own Tier-3 reviews. 1. FALSE-GREEN RELEASE ASSURANCE. Phase 6's docker.yml pull_request filter omitted inputs the image actually COPYs โ€” backend/entrypoint.sh, backend/migrations/**, backend/rust-toolchain.toml, and the ruvector submodule gitlink. A packaging regression in any of them could merge with the Docker jobs never running, defeating the guarantee the trigger exists to provide. Filter now covers every COPYed input; backend/src/** stays excluded deliberately, with the reason recorded inline. 2. INCOMPLETE TASK-RUNNER MIGRATION. Live `make` commands survived phase 7 in user-facing places the sweep never covered (it grepped docs and `make ` in scripts, but not source, and not the `make -C` form): - OnboardingFlow.tsx:122 โ€” the PRODUCT UI showed users `make dev` - backend/src/main.rs:207 โ€” the CLI printed `Run 'make models'` - scripts/setup-ai.sh, scripts/setup-validate.sh โ€” `make -C ...` All fixed. A repo-wide sweep across .sh/.rs/.tsx/.ts/.yml/.json now finds no live make invocation. cargo check, tsc --noEmit, shellcheck, just lint and just format-check all clean afterwards. A third charge (ADR-032 marked Accepted) was NOT upheld: this repo uses Accepted for implemented decisions and Proposed for design-only (cf ADR-031). Court record: .autopilot/court/ci-build-optimization/integration.md --- .../ci-build-optimization/integration.md | 89 +++++++++++++++++++ .github/workflows/docker.yml | 12 +++ backend/src/main.rs | 2 +- .../features/onboarding/OnboardingFlow.tsx | 2 +- scripts/setup-ai.sh | 2 +- scripts/setup-validate.sh | 2 +- 6 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 .autopilot/court/ci-build-optimization/integration.md diff --git a/.autopilot/court/ci-build-optimization/integration.md b/.autopilot/court/ci-build-optimization/integration.md new file mode 100644 index 0000000..8b1910b --- /dev/null +++ b/.autopilot/court/ci-build-optimization/integration.md @@ -0,0 +1,89 @@ +# qe-court record โ€” ci-build-optimization, integration (develop -> main) + +**Delivery:** the full 8-phase CI/build/Docker optimization, `main...develop` +(44 files, +2387/-1438) plus the cross-phase optimization pass. +**Date:** 2026-07-31 ยท **Seat:** the release go/no-go, per ADR-124. + +## Panel โ€” properly seated for the first time in this pipeline + +The user fixed the jury routing before this convening (`.claude/skills/qe-court/config.json`). +Earlier phase courts could not seat a jury at all because 4 of 8 roles, including the +jury, routed to Cognitum, which is unconfigured in this environment. + +| Role | Provider | Vendor | +| --- | --- | --- | +| Writer (author under review) | claude-code | Claude | +| Jury | codex | GPT | +| Deeper reviewer | codex (high effort) | GPT | + +`writerIsNeverJuror`: **SATISFIED** โ€” the jury is a different vendor from the author. +`minDistinctVendors: 2`: **SATISFIED**. + +**Caveat, stated plainly:** the jury's run was partly polluted by an injected plugin +preamble (the accepted risk recorded in `config.json._acceptedRisk`). It nonetheless +read the real diff โ€” the transcript shows it running +`git diff --unified=40 main...develop` over the justfiles, ADR-032 and the setup +scripts โ€” and returned a substantive, file-cited verdict. It is counted as a real +juror. The kill round and overturn round were NOT run, so this is still weaker than +the full ADR-124 protocol. + +## Verdict: REMAND โ€” charges upheld and fixed + +Both charges were verified against the repo before being accepted, and both were +things the author's own Tier-3 reviews had missed. + +### Charge 1 โ€” false-green release assurance (UPHELD, FIXED) + +Phase 6 added a paths-scoped `pull_request` trigger to `docker.yml` so image +packaging is verified pre-merge. The filter omitted inputs the image actually +consumes: `backend/entrypoint.sh`, `backend/migrations/**`, +`backend/rust-toolchain.toml`, and the `ruvector` submodule gitlink โ€” all of them +`COPY`ed by `backend/Dockerfile`. A packaging regression in any of those could merge +with the Docker jobs never running, defeating the guarantee the trigger was added to +provide. + +**Fix:** filter extended to every COPYed input, with a comment tying it back to the +Dockerfile's COPY lines. `backend/src/**` is deliberately excluded and the reason +recorded inline โ€” it is COPYed, but the rust-build/rust-test jobs already compile it +and a cold release build per source change is not worth the cost. + +### Charge 2 โ€” incomplete task-runner migration (UPHELD, FIXED) + +Phase 7 deleted the Makefiles, but live `make` commands survived in user-facing +places the author's sweep never covered โ€” it grepped docs and `make ` in +shell scripts, but not source code, and not the `make -C ` form in scripts: + +- `frontend/apps/web/src/features/onboarding/OnboardingFlow.tsx:122` โ€” the **product + onboarding UI** displayed `make dev` to end users. +- `backend/src/main.rs:207` โ€” the CLI printed `Run 'make models'`. +- `scripts/setup-ai.sh:92`, `scripts/setup-validate.sh:108` โ€” `make -C backend build` + / `make -C frontend install`. + +Every one pointed at a command that no longer exists. + +**Fix:** all corrected; a repo-wide sweep across `.sh`, `.rs`, `.tsx`, `.ts`, `.yml`, +`.json` now returns no live `make` invocation. Verified after the edits: +`cargo check` clean, `tsc --noEmit` clean, `shellcheck` clean, `just lint` and +`just format-check` green. + +### Charge 3 โ€” ADR-032 status (NOT UPHELD) + +The jury flagged ADR-032 as "Accepted" while the branch claims implementation. This +repo's ADR convention uses **Accepted** for decisions that are made and implemented +(ADR-031 uses **Proposed** for design-only). No change. + +## For the human judge + +**Strongest case FOR merging:** the core engineering holds up โ€” 1177-test parity +proven by execution across the nextest migration, toolchain drift removed and then +de-duplicated, four swallowed exit codes fixed, and two genuinely broken release +images (wrong build stage; glibc mismatch that built clean but could not run) now +fixed and verified on the amd64 architecture that actually ships. + +**Strongest case AGAINST:** CI proves images *build*, not that the system *runs*. +Three disclosed gaps remain open and are not addressed by this branch โ€” the Compose +healthcheck passes a flag `main.rs` does not parse (so the container can never report +healthy), the frontend image cannot serve standalone, and the Lighthouse budgets are +all `warn` so that job still cannot fail. None is a regression introduced here; all +three are recorded as parking-lot items. A container-start/health smoke test is the +single highest-value thing still missing. diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9b92b70..4abf5b0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -10,13 +10,25 @@ on: # pay for a cold release build; regular Rust/frontend CI already covers compilation. pull_request: paths: + # Every input the image actually consumes. Cross-check against the COPY + # lines in backend/Dockerfile when either changes โ€” an input that is COPYed + # but NOT listed here can regress packaging and still merge green, which + # would defeat the whole point of this pre-merge trigger. - "backend/Dockerfile" - "frontend/Dockerfile" - "backend/Cargo.toml" - "backend/Cargo.lock" + - "backend/rust-toolchain.toml" # selects the builder toolchain + - "backend/entrypoint.sh" # COPYed into the runtime stage + - "backend/migrations/**" # COPYed into the runtime stage + - "ruvector" # submodule gitlink; the build resolves path deps into it - ".dockerignore" - "docker-compose*.yml" - ".github/workflows/docker.yml" + # Deliberately NOT backend/src/**: it is COPYed, but a cold release build on + # every backend source change is expensive and the regular rust-build / + # rust-test jobs already compile that code. This trigger exists to prove + # PACKAGING, which the paths above govern. # NOTE: deliberately NOT triggered on tags. release.yml already builds and # pushes both images on a tag; duplicating it here ran two concurrent cold # Rust release builds writing to the same gha cache scope. diff --git a/backend/src/main.rs b/backend/src/main.rs index ea350a2..f363816 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -204,7 +204,7 @@ async fn main() -> anyhow::Result<()> { .map_err(|e| anyhow::anyhow!("{e}")); } else { eprintln!("Usage: emailibrium --download-model "); - eprintln!("Run 'make models' to see available models."); + eprintln!("Run 'just models' to see available models."); std::process::exit(1); } } diff --git a/frontend/apps/web/src/features/onboarding/OnboardingFlow.tsx b/frontend/apps/web/src/features/onboarding/OnboardingFlow.tsx index 1815748..c76d228 100644 --- a/frontend/apps/web/src/features/onboarding/OnboardingFlow.tsx +++ b/frontend/apps/web/src/features/onboarding/OnboardingFlow.tsx @@ -119,7 +119,7 @@ function ServerHealthBadge({ status }: { status: BackendStatus }) {