From 4eee6d4977558a88e0f115645ccde9eab7239ae6 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Tue, 14 Jul 2026 02:33:22 +0100 Subject: [PATCH 1/2] db: create lore_indexed_commits before migrating the lore table create_all_tables() migrates an existing lore table before it checks whether lore_indexed_commits exists. When the lore table predates the date_timestamp column, migrate_lore_table() calls reconcile_lore_indexed_commits(), which opens lore_indexed_commits and fails on databases created before that table was introduced: Error: Table 'lore_indexed_commits' was not found Create lore_indexed_commits ahead of the lore migration so the reconciliation always has a table to open, and add a regression test that runs create_all_tables() against a database seeded with the old 10-column lore schema. Signed-off-by: Daniel Golle --- src/database/schema.rs | 62 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/src/database/schema.rs b/src/database/schema.rs index b4a4026..7e28ef0 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -61,16 +61,18 @@ impl SchemaManager { self.create_commit_vectors_table().await?; } + // Must exist before migrate_lore_table(), which reconciles + // against it when adding the date_timestamp column. + if !table_names.iter().any(|n| n == "lore_indexed_commits") { + self.create_lore_indexed_commits_table().await?; + } + if !table_names.iter().any(|n| n == "lore") { self.create_lore_table().await?; } else { self.migrate_lore_table().await?; } - if !table_names.iter().any(|n| n == "lore_indexed_commits") { - self.create_lore_indexed_commits_table().await?; - } - if !table_names.iter().any(|n| n == "lore_vectors") { self.create_lore_vectors_table().await?; } @@ -1654,3 +1656,55 @@ impl SchemaManager { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// Databases created before lore_indexed_commits existed have a lore + /// table without the date_timestamp column. create_all_tables() must + /// create lore_indexed_commits before migrating the lore table, since + /// the migration reconciles against it. + #[tokio::test] + async fn test_create_all_tables_migrates_old_lore_schema() { + let tmpdir = TempDir::new().unwrap(); + let connection = lancedb::connect(tmpdir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let old_schema = Arc::new(Schema::new(vec![ + Field::new("git_commit_sha", DataType::Utf8, false), + Field::new("from", DataType::Utf8, false), + Field::new("date", DataType::Utf8, false), + Field::new("message_id", DataType::Utf8, false), + Field::new("in_reply_to", DataType::Utf8, true), + Field::new("subject", DataType::Utf8, false), + Field::new("references", DataType::Utf8, true), + Field::new("recipients", DataType::Utf8, false), + Field::new("body", DataType::Utf8, false), + Field::new("symbols", DataType::Utf8, false), + ])); + let empty_batch = RecordBatch::new_empty(old_schema.clone()); + connection + .create_table("lore", vec![empty_batch]) + .execute() + .await + .unwrap(); + + let manager = SchemaManager::new(connection.clone()); + manager.create_all_tables().await.unwrap(); + + let tables = connection.table_names().execute().await.unwrap(); + assert!(tables.iter().any(|n| n == "lore_indexed_commits")); + + let lore = connection.open_table("lore").execute().await.unwrap(); + assert!(lore + .schema() + .await + .unwrap() + .column_with_name("date_timestamp") + .is_some()); + } +} From c54cbf120d9bc9e806338314e603b5cbd568e5fd Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Tue, 14 Jul 2026 02:33:39 +0100 Subject: [PATCH 2/2] index: add pipermail mbox archive indexing Projects like U-Boot publish their mailing list archives with pipermail (Mailman 2) rather than public-inbox, as monthly mbox files linked from an archive index page. Add a --pipermail option to semcode-index that downloads and indexes such archives into the existing lore tables: semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ The index page is scraped for YYYY-Month.txt(.gz) links and each monthly file is downloaded to /pipermail///, then split into individual messages and stored through the same insertion path as lore emails, so lore, dig and vlore work on them unchanged. Since there is no backing git repository, each message is identified by a blake3 hash of its content, recorded in lore_indexed_commits for incremental refreshes. Pipermail does not escape body lines beginning with "From ", so the mbox splitter only accepts separator lines with the full From_ shape (envelope address followed by an asctime date) preceded by a blank line. This avoids splitting on inline git format-patch headers and on prose starting with "From ". --pipermail without arguments refreshes every previously downloaded archive: only months missing locally and the newest month (which keeps growing until the next month starts) are fetched again, and message hashes already recorded are skipped. A --pipermail-since date (month granularity, accepting the same formats as the lore search date filters) bounds how far back an archive is downloaded and indexed. The cutoff is recorded in the archive directory as archive.since and keeps applying to refreshes; archives fetched without a cutoff never reach further back than their oldest already-downloaded month. The email parsing previously embedded in parse_email_from_commit() moves to a shared parse_email_content() used by both the lore and the mbox paths. Signed-off-by: Daniel Golle --- README.md | 4 +- docs/lore.md | 44 ++++ src/bin/index.rs | 246 ++++++++++++++++++++++ src/indexer.rs | 58 +++++- src/lib.rs | 1 + src/pipermail.rs | 532 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 883 insertions(+), 2 deletions(-) create mode 100644 src/pipermail.rs diff --git a/README.md b/README.md index 03200f2..f030505 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ Rust indexing is now supported. This just uses treesitter, but all the semcode features are there. Now with lore indexing! semcode-index --lore lkml,netdev (or any list names) will -pull down the latest git archive from those lists. See [the lore documentation](docs/lore.md) +pull down the latest git archive from those lists. Projects using pipermail +archives instead can use semcode-index --pipermail . See +[the lore documentation](docs/lore.md) for more details. This is a database schema change, so you'll need to reindex. Recent commits introduced indexes for git commit history, as well as diff --git a/docs/lore.md b/docs/lore.md index 9a2130c..9a86f3e 100644 --- a/docs/lore.md +++ b/docs/lore.md @@ -104,6 +104,50 @@ When called without arguments, `--lore` discovers all git repositories under not yet in the database. Use this for a simple one-command workflow to keep lore archives up to date. +### Indexing Pipermail Archives + +Projects that publish their mailing list archives with pipermail (Mailman 2) +instead of public-inbox/lore can be indexed with `--pipermail`, passing the +base URL of the archive index page: + +```bash +# Index a pipermail archive (e.g., the U-Boot mailing list) +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ + +# Index multiple archives at once +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/,https://lists.denx.de/pipermail/eldk/ + +# Refresh all previously downloaded pipermail archives +semcode-index --pipermail +``` + +To avoid downloading decades of history, limit how far back the archives go +with `--pipermail-since` (month granularity): + +```bash +# Only index emails from the previous year onwards +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ --pipermail-since "1 year ago" + +# Absolute dates work too +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ --pipermail-since 2025-01-01 +``` + +The cutoff is recorded in the archive directory (`archive.since`) and +persists across refreshes: `semcode-index --pipermail` keeps honouring it. +Archives downloaded without a cutoff never reach further back than their +oldest already-downloaded month. To extend an archive further into the +past later, pass an earlier `--pipermail-since` explicitly. + +The monthly mbox files (`YYYY-Month.txt.gz`) linked from the index page are +downloaded to `/pipermail///` and split into individual +messages, which are stored in the same lore tables as public-inbox archives. +All `lore`, `dig`, and `vlore` commands work on them identically. + +Since there is no backing git repository, each message is identified by a +content hash instead of a commit SHA. Refreshing re-downloads only months +that are missing locally plus the newest month (which keeps growing until +the next month starts); already-indexed messages are skipped. + ### Optional: Generate Vector Embeddings for Semantic Search To enable semantic search with the `vlore` command: diff --git a/src/bin/index.rs b/src/bin/index.rs index 214d9d0..9facb78 100644 --- a/src/bin/index.rs +++ b/src/bin/index.rs @@ -4,6 +4,7 @@ use clap::Parser; use colored::Colorize; use semcode::indexer::{ list_shas_in_range, process_commits_pipeline, process_lore_commits_pipeline, + process_mbox_messages_pipeline, }; use semcode::{measure, process_database_path, CodeVectorizer, DatabaseManager}; // Temporary call relationships are now embedded in function JSON columns @@ -104,6 +105,20 @@ struct Args { #[arg(long, value_name = "LIST", value_delimiter = ',', num_args = 0..)] lore: Option>, + /// Download and index pipermail mbox archives into /pipermail// + /// Accepts comma-separated archive base URLs + /// (e.g., --pipermail https://lists.denx.de/pipermail/u-boot/) + /// Without arguments, refreshes all previously downloaded archives + #[arg(long, value_name = "URL", value_delimiter = ',', num_args = 0..)] + pipermail: Option>, + + /// Only download and index pipermail archives from this date onwards + /// (month granularity). Accepts 'YYYY-MM-DD' or relative dates like + /// '1 year ago'. The cutoff is recorded per archive and refreshes + /// keep honouring it unless an earlier date is given explicitly. + #[arg(long, value_name = "DATE", requires = "pipermail")] + pipermail_since: Option, + // ==================== Multi-Branch Indexing ==================== /// Index a specific branch (can be specified multiple times) /// Example: --branch main --branch develop @@ -712,6 +727,106 @@ async fn index_lore_archive( }) } +/// Shared settings and insertion counters for a pipermail indexing run +struct PipermailRun { + batch_size: usize, + since: Option<(u32, u32)>, + batches_inserted: Arc, + optimization_check_timer: Arc>, +} + +/// Download new monthly mbox files for a pipermail archive and index any +/// messages not yet in the database. `existing` carries the set of already +/// indexed message ids and is updated as messages are queued for insertion. +async fn process_pipermail_archive( + base_url: &str, + db_path: &str, + db_manager: &Arc, + run: &PipermailRun, + existing: &mut HashSet, +) -> Result { + use semcode::pipermail; + + let base_url = pipermail::normalize_base_url(base_url); + let archive_dir = pipermail::archive_storage_dir(db_path, &base_url)?; + std::fs::create_dir_all(&archive_dir)?; + pipermail::save_archive_url(&archive_dir, &base_url)?; + + // An explicit cutoff is recorded for later refreshes; without one, + // fall back to the cutoff recorded by a previous run. + let since = match run.since { + Some(s) => { + pipermail::save_archive_since(&archive_dir, s)?; + Some(s) + } + None => pipermail::load_archive_since(&archive_dir), + }; + + println!("Fetching archive index from {}...", base_url); + let index_html = { + let url = base_url.clone(); + tokio::task::spawn_blocking(move || pipermail::fetch_index_page(&url)).await?? + }; + let remote_files = pipermail::discover_archive_files(&index_html); + if remote_files.is_empty() { + return Err(anyhow::anyhow!("No monthly archives found at {}", base_url)); + } + + let to_download = pipermail::files_to_download(&remote_files, &archive_dir, since)?; + println!( + "{} monthly archives on server, {} to download", + remote_files.len(), + to_download.len() + ); + for file in to_download { + let url = format!("{}{}", base_url, file.file_name); + let dest = archive_dir.join(&file.file_name); + println!(" Downloading {}", file.file_name); + tokio::task::spawn_blocking(move || pipermail::download_file(&url, &dest)).await??; + } + + let mut new_emails = 0usize; + let mut total_emails = 0usize; + for (file, path) in pipermail::list_local_mbox_files(&archive_dir)? { + if since.is_some_and(|s| file.key() < s) { + continue; + } + let content = pipermail::read_mbox_file(&path)?; + let messages = pipermail::split_mbox(&content); + total_emails += messages.len(); + + let mut new_messages = Vec::new(); + for message in messages { + let id = semcode::hash::compute_blake3_hash(&message); + if existing.insert(id.clone()) { + new_messages.push((id, message)); + } + } + if new_messages.is_empty() { + continue; + } + + println!( + " {}: indexing {} new messages", + file.file_name, + new_messages.len() + ); + new_emails += process_mbox_messages_pipeline( + new_messages, + db_manager.clone(), + run.batch_size, + run.batches_inserted.clone(), + run.optimization_check_timer.clone(), + ) + .await?; + } + + Ok(LoreIndexResult { + new_emails, + total_emails, + }) +} + // ==================== Branch Indexing Support ==================== /// Collect branches to index from the various branch-related CLI flags @@ -1308,6 +1423,137 @@ async fn main() -> Result<()> { } } + // Handle --pipermail option if provided + if let Some(pipermail_args) = &args.pipermail { + // If --pipermail has arguments, download and index the specified archives + // If --pipermail has no arguments, refresh all previously downloaded archives + let base_urls: Vec = if !pipermail_args.is_empty() { + pipermail_args.clone() + } else { + let saved = semcode::pipermail::discover_saved_archives(&database_path)?; + if saved.is_empty() { + println!("No pipermail archives have been downloaded yet."); + println!(); + println!("To index pipermail mailing list archives, specify archive base URLs:"); + println!(" semcode-index --pipermail [,...]"); + println!(); + println!("Example:"); + println!(" semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/"); + return Ok(()); + } + println!("Found {} pipermail archive(s) to refresh:", saved.len()); + for (_, url) in &saved { + println!(" - {}", url); + } + saved.into_iter().map(|(_, url)| url).collect() + }; + + info!( + "Pipermail archive processing requested for {} archives", + base_urls.len() + ); + + let since = match args.pipermail_since.as_deref() { + Some(date_str) => { + let (year, month) = semcode::pipermail::since_month(date_str)?; + println!("Limiting archives to {}-{:02} onwards", year, month); + Some((year, month)) + } + None => None, + }; + + let db_manager = + DatabaseManager::new(&database_path, args.source.to_string_lossy().to_string()).await?; + db_manager.create_tables().await?; + let db_manager = Arc::new(db_manager); + + let start_time = std::time::Instant::now(); + let run = PipermailRun { + batch_size: 1024, + since, + batches_inserted: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + optimization_check_timer: Arc::new(std::sync::Mutex::new(std::time::Instant::now())), + }; + + println!("Checking for already-indexed messages..."); + let mut existing = db_manager.get_indexed_lore_commits().await?; + + let mut total_new_emails = 0usize; + let mut total_emails_all_archives = 0usize; + let mut failed_archives: Vec<(String, String)> = Vec::new(); + + for base_url in &base_urls { + println!("\n=== Processing pipermail archive: {} ===", base_url); + match process_pipermail_archive( + base_url, + &database_path, + &db_manager, + &run, + &mut existing, + ) + .await + { + Ok(result) => { + println!( + "Indexed {} new emails from {} (total in archive: {})", + result.new_emails, base_url, result.total_emails + ); + total_new_emails += result.new_emails; + total_emails_all_archives += result.total_emails; + } + Err(e) => { + eprintln!("Error processing {}: {:#}", base_url, e); + failed_archives.push((base_url.clone(), e.to_string())); + } + } + } + + let total_time = start_time.elapsed(); + + println!("\n=== Pipermail Email Indexing Complete ==="); + println!("Total time: {:.1}s", total_time.as_secs_f64()); + println!( + "Archives processed: {}/{}", + base_urls.len() - failed_archives.len(), + base_urls.len() + ); + println!("New emails indexed: {}", total_new_emails); + println!( + "Total emails across archives: {}", + total_emails_all_archives + ); + + if !failed_archives.is_empty() { + eprintln!("\nFailed archives:"); + for (name, err) in &failed_archives { + eprintln!(" {}: {}", name, err); + } + } + + if total_new_emails > 0 { + println!("\nCompacting lore tables..."); + match db_manager.compact_lore_tables().await { + Ok(_) => println!("Lore table compaction completed successfully"), + Err(e) => error!("Failed to compact lore tables: {}", e), + } + + println!("\nUpdating FTS indices for lore table..."); + match db_manager.ensure_lore_fts_indices().await { + Ok(_) => {} + Err(e) => eprintln!("Warning: Failed to ensure FTS indices: {}", e), + } + match db_manager.optimize_lore_fts_indices().await { + Ok(_) => println!("FTS indices updated successfully"), + Err(e) => eprintln!("Warning: Failed to optimize FTS indices: {}", e), + } + } + + println!("\nTo query this database, run:"); + println!(" semcode --database {}", database_path); + + return Ok(()); + } + // Validate mutually exclusive options if args.git.is_some() && args.commits.is_some() { return Err(anyhow::anyhow!( diff --git a/src/indexer.rs b/src/indexer.rs index a7a9f48..7ae240d 100644 --- a/src/indexer.rs +++ b/src/indexer.rs @@ -421,6 +421,13 @@ pub fn parse_email_from_commit( let blob = object.try_into_blob()?; let email_content = String::from_utf8_lossy(blob.data.as_slice()).to_string(); + parse_email_content(&email_content, commit_sha) +} + +/// Parse a raw email message (headers and body) into a LoreEmailInfo. +/// The id becomes the git_commit_sha field: the containing commit SHA for +/// public-inbox archives, or a content hash for mbox-sourced messages. +pub fn parse_email_content(email_content: &str, id: &str) -> Result { // Parse email headers let mut headers = EmailHeaders::new(); @@ -480,7 +487,7 @@ pub fn parse_email_from_commit( let date_timestamp = parse_rfc2822_to_timestamp(&headers.date); Ok(crate::LoreEmailInfo { - git_commit_sha: commit_sha.to_string(), + git_commit_sha: id.to_string(), from: headers.from, date: headers.date, date_timestamp, @@ -1058,6 +1065,55 @@ pub async fn process_lore_commits_pipeline( Ok(()) } +/// Parse raw mbox messages in parallel and insert them into the lore tables. +/// `messages` holds (id, raw email) pairs where id is a stable content hash +/// used as the lore git_commit_sha. Returns the number of emails stored. +pub async fn process_mbox_messages_pipeline( + messages: Vec<(String, String)>, + db_manager: Arc, + batch_size: usize, + batches_inserted: Arc, + optimization_check_timer: Arc>, +) -> Result { + use rayon::prelude::*; + + let emails: Vec = messages + .par_iter() + .filter_map(|(id, content)| match parse_email_content(content, id) { + Ok(email) => Some(email), + Err(e) => { + warn!("Failed to parse mbox message {}: {}", id, e); + None + } + }) + .collect(); + + let mut inserted = 0usize; + for chunk in emails.chunks(batch_size) { + let failed_indices = db_manager.insert_lore_emails(chunk).await?; + let failed_set: HashSet = failed_indices.into_iter().collect(); + + // Record ids only for emails that were actually stored, so that + // failed emails are retried on the next run. + let ids: Vec = chunk + .iter() + .enumerate() + .filter(|(i, _)| !failed_set.contains(i)) + .map(|(_, e)| e.git_commit_sha.clone()) + .collect(); + inserted += ids.len(); + if !ids.is_empty() { + db_manager.insert_lore_indexed_commits(&ids).await?; + } + + let total_batches = batches_inserted.fetch_add(1, Ordering::Relaxed) + 1; + check_and_optimize_if_needed(&db_manager, 0, total_batches, &optimization_check_timer) + .await; + } + + Ok(inserted) +} + /// Index commits in a git range pub async fn index_git_commits( repo_path: &PathBuf, diff --git a/src/lib.rs b/src/lib.rs index b1c79d0..4aa2af3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod git_range; pub mod hash; pub mod indexer; pub mod perf_monitor; +pub mod pipermail; pub mod symbol_walkback; pub mod text_utils; mod treesitter_analyzer; diff --git a/src/pipermail.rs b/src/pipermail.rs new file mode 100644 index 0000000..798c169 --- /dev/null +++ b/src/pipermail.rs @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +//! Pipermail (Mailman 2) mailing list archive support +//! +//! Downloads the monthly mbox archives published by a pipermail archive +//! index page (e.g. https://lists.denx.de/pipermail/u-boot/) and splits +//! them into individual email messages for lore indexing. + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// File written into each archive directory recording the base URL, +/// so that `--pipermail` without arguments can refresh the archive. +pub const URL_FILE: &str = "archive.url"; + +/// File written into each archive directory recording the +/// `--pipermail-since` cutoff as "YYYY-MM", so refreshes keep honouring it. +pub const SINCE_FILE: &str = "archive.since"; + +const MONTH_NAMES: [&str; 12] = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +/// A monthly mbox archive published on a pipermail index page +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchiveFile { + pub file_name: String, + pub year: u32, + pub month: u32, +} + +impl ArchiveFile { + pub fn key(&self) -> (u32, u32) { + (self.year, self.month) + } +} + +/// Parse an archive file stem like "2026-February" into (year, month) +pub fn month_key(stem: &str) -> Option<(u32, u32)> { + let (year, month_name) = stem.split_once('-')?; + let year = year.parse::().ok()?; + let month = MONTH_NAMES.iter().position(|m| *m == month_name)? as u32 + 1; + Some((year, month)) +} + +/// Strip the ".txt" or ".txt.gz" suffix from an archive file name +fn archive_file_stem(file_name: &str) -> Option<&str> { + file_name + .strip_suffix(".txt.gz") + .or_else(|| file_name.strip_suffix(".txt")) +} + +/// Extract the monthly mbox files linked from a pipermail index page. +/// Prefers the gzipped variant when both are listed. Results are sorted +/// chronologically (oldest first). +pub fn discover_archive_files(index_html: &str) -> Vec { + static HREF_RE: OnceLock = OnceLock::new(); + let re = HREF_RE.get_or_init(|| { + regex::Regex::new(r#"(?i)href="(\d{4}-[A-Za-z]+\.txt(?:\.gz)?)""#).unwrap() + }); + + let mut by_month: std::collections::HashMap<(u32, u32), ArchiveFile> = + std::collections::HashMap::new(); + + for cap in re.captures_iter(index_html) { + let file_name = cap[1].to_string(); + let Some(stem) = archive_file_stem(&file_name) else { + continue; + }; + let Some((year, month)) = month_key(stem) else { + continue; + }; + let entry = ArchiveFile { + file_name: file_name.clone(), + year, + month, + }; + by_month + .entry((year, month)) + .and_modify(|existing| { + if file_name.ends_with(".gz") && !existing.file_name.ends_with(".gz") { + *existing = entry.clone(); + } + }) + .or_insert(entry); + } + + let mut files: Vec = by_month.into_values().collect(); + files.sort_by_key(|f| f.key()); + files +} + +/// List locally downloaded monthly mbox files, sorted chronologically +pub fn list_local_mbox_files(dir: &Path) -> Result> { + let mut files = Vec::new(); + + if !dir.exists() { + return Ok(files); + } + + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let Some(file_name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + let Some(stem) = archive_file_stem(file_name) else { + continue; + }; + let Some((year, month)) = month_key(stem) else { + continue; + }; + files.push(( + ArchiveFile { + file_name: file_name.to_string(), + year, + month, + }, + path, + )); + } + + files.sort_by_key(|(f, _)| f.key()); + Ok(files) +} + +/// Parse a user-supplied date ("2025-07-01", "1 year ago") into a +/// (year, month) archive cutoff +pub fn since_month(date_str: &str) -> Result<(u32, u32)> { + use chrono::Datelike; + + let parsed = chrono_english::parse_date_string( + date_str.trim(), + chrono::Utc::now(), + chrono_english::Dialect::Us, + ) + .map_err(|e| { + anyhow::anyhow!( + "Invalid date '{}': {}. Use 'YYYY-MM-DD' or relative dates like '1 year ago'", + date_str, + e + ) + })?; + + Ok((parsed.year() as u32, parsed.month())) +} + +/// Decide which remote files need downloading: months missing locally, +/// plus the newest local month (it keeps growing until the next month +/// starts). Everything is bounded by `since` when given, otherwise by the +/// oldest month already downloaded, so a `--pipermail-since` cutoff +/// persists across refreshes. +pub fn files_to_download( + remote: &[ArchiveFile], + dir: &Path, + since: Option<(u32, u32)>, +) -> Result> { + let local = list_local_mbox_files(dir)?; + let local_keys: std::collections::HashSet<(u32, u32)> = + local.iter().map(|(f, _)| f.key()).collect(); + let newest_local = local.iter().map(|(f, _)| f.key()).max(); + let horizon = since.or_else(|| local.iter().map(|(f, _)| f.key()).min()); + + Ok(remote + .iter() + .filter(|f| { + if horizon.is_some_and(|h| f.key() < h) { + return false; + } + !local_keys.contains(&f.key()) || newest_local == Some(f.key()) + }) + .cloned() + .collect()) +} + +/// Ensure the archive base URL ends with a trailing slash +pub fn normalize_base_url(base_url: &str) -> String { + let mut url = base_url.trim().to_string(); + if !url.ends_with('/') { + url.push('/'); + } + url +} + +/// Derive the local storage directory for a pipermail archive: +/// `/pipermail//` +pub fn archive_storage_dir(db_path: &str, base_url: &str) -> Result { + let without_scheme = base_url + .strip_prefix("https://") + .or_else(|| base_url.strip_prefix("http://")) + .ok_or_else(|| anyhow::anyhow!("Archive URL must start with http:// or https://"))?; + + let mut segments = without_scheme.split('/').filter(|s| !s.is_empty()); + let host = segments + .next() + .ok_or_else(|| anyhow::anyhow!("Invalid archive URL: {}", base_url))?; + let list = segments + .next_back() + .ok_or_else(|| anyhow::anyhow!("Archive URL has no list name: {}", base_url))?; + + if host.contains("..") || list.contains("..") { + return Err(anyhow::anyhow!("Invalid archive URL: {}", base_url)); + } + + Ok(PathBuf::from(db_path) + .join("pipermail") + .join(host) + .join(list)) +} + +/// Record the base URL of an archive so it can be refreshed later +pub fn save_archive_url(dir: &Path, base_url: &str) -> Result<()> { + std::fs::write(dir.join(URL_FILE), format!("{}\n", base_url)) + .with_context(|| format!("Failed to write {}", dir.join(URL_FILE).display())) +} + +/// Record the cutoff month so refreshes keep honouring it +pub fn save_archive_since(dir: &Path, since: (u32, u32)) -> Result<()> { + let path = dir.join(SINCE_FILE); + std::fs::write(&path, format!("{}-{:02}\n", since.0, since.1)) + .with_context(|| format!("Failed to write {}", path.display())) +} + +/// Load a previously recorded cutoff month, if any +pub fn load_archive_since(dir: &Path) -> Option<(u32, u32)> { + let content = std::fs::read_to_string(dir.join(SINCE_FILE)).ok()?; + let (year, month) = content.trim().split_once('-')?; + let year = year.parse::().ok()?; + let month = month.parse::().ok()?; + (1..=12).contains(&month).then_some((year, month)) +} + +/// Discover previously downloaded archives under `/pipermail/`. +/// Returns (archive directory, base URL) pairs. +pub fn discover_saved_archives(db_path: &str) -> Result> { + let base = PathBuf::from(db_path).join("pipermail"); + let mut archives = Vec::new(); + + fn walk(dir: &Path, archives: &mut Vec<(PathBuf, String)>) -> Result<()> { + let url_file = dir.join(URL_FILE); + if url_file.is_file() { + let url = std::fs::read_to_string(&url_file)?.trim().to_string(); + if !url.is_empty() { + archives.push((dir.to_path_buf(), url)); + } + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + walk(&path, archives)?; + } + } + Ok(()) + } + + if base.is_dir() { + walk(&base, &mut archives)?; + } + + archives.sort(); + Ok(archives) +} + +/// Fetch the archive index page for a pipermail list (blocking) +pub fn fetch_index_page(base_url: &str) -> Result { + let response = reqwest::blocking::get(base_url) + .with_context(|| format!("Failed to fetch archive index {}", base_url))?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Failed to fetch archive index {}: HTTP {}", + base_url, + response.status() + )); + } + Ok(response.text()?) +} + +/// Download a monthly mbox file to the given destination (blocking). +/// Writes to a temporary file first so interrupted downloads are not +/// mistaken for complete archives. +pub fn download_file(url: &str, dest: &Path) -> Result<()> { + let response = + reqwest::blocking::get(url).with_context(|| format!("Failed to download {}", url))?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Failed to download {}: HTTP {}", + url, + response.status() + )); + } + let bytes = response.bytes()?; + + let tmp = dest.with_extension("part"); + std::fs::write(&tmp, &bytes).with_context(|| format!("Failed to write {}", tmp.display()))?; + std::fs::rename(&tmp, dest)?; + Ok(()) +} + +/// Read a downloaded mbox file, transparently decompressing gzip. +/// Invalid UTF-8 sequences are replaced rather than rejected since old +/// archives commonly contain legacy 8-bit encodings. +pub fn read_mbox_file(path: &Path) -> Result { + use std::io::Read; + + let raw = std::fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; + let bytes = if path.extension().and_then(|e| e.to_str()) == Some("gz") { + let mut decoder = flate2::read::GzDecoder::new(&raw[..]); + let mut decompressed = Vec::new(); + decoder + .read_to_end(&mut decompressed) + .with_context(|| format!("Failed to decompress {}", path.display()))?; + decompressed + } else { + raw + }; + + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} + +/// Match a pipermail mbox message separator. Pipermail does not escape +/// body lines starting with "From ", so the match requires the full +/// From_ line shape (envelope address followed by an asctime date) to +/// avoid splitting on inline patches ("From Mon Sep 17 ...") or +/// prose. The address is usually obfuscated as "user at domain" but a +/// plain address is also accepted. +fn is_mbox_separator(line: &str) -> bool { + static SEPARATOR_RE: OnceLock = OnceLock::new(); + let re = SEPARATOR_RE.get_or_init(|| { + regex::Regex::new( + r"^From (\S+ at \S+|\S+@\S+) +[A-Z][a-z]{2} [A-Z][a-z]{2} +\d{1,2} +\d{1,2}:\d{2}:\d{2} \d{4}$", + ) + .unwrap() + }); + re.is_match(line) +} + +/// Split a pipermail mbox archive into individual messages. A message +/// starts at a From_ separator line preceded by a blank line (or the +/// start of the file); the separator line itself is not included in the +/// returned message text. +pub fn split_mbox(content: &str) -> Vec { + let mut messages = Vec::new(); + let mut current: Vec<&str> = Vec::new(); + let mut in_message = false; + let mut prev_blank = true; + + for line in content.lines() { + if prev_blank && is_mbox_separator(line) { + if in_message && !current.is_empty() { + messages.push(current.join("\n")); + } + current.clear(); + in_message = true; + } else if in_message { + current.push(line); + } + prev_blank = line.is_empty(); + } + + if in_message && !current.is_empty() { + messages.push(current.join("\n")); + } + + messages +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_month_key() { + assert_eq!(month_key("2026-February"), Some((2026, 2))); + assert_eq!(month_key("1999-December"), Some((1999, 12))); + assert_eq!(month_key("2026-Foo"), None); + assert_eq!(month_key("February"), None); + } + + #[test] + fn test_discover_archive_files() { + let html = r#" + [ Gzip'd Text 1 MB ] + [ Gzip'd Text 4 MB ] + [ Text 5 MB ] + [ Gzip'd Text 2 MB ] + thread + "#; + let files = discover_archive_files(html); + let names: Vec<&str> = files.iter().map(|f| f.file_name.as_str()).collect(); + assert_eq!( + names, + vec![ + "2025-December.txt.gz", + "2026-June.txt.gz", + "2026-July.txt.gz" + ] + ); + } + + #[test] + fn test_split_mbox() { + let mbox = "\ +From alice at example.org Sun Feb 1 00:38:12 2026 +From: alice at example.org (Alice) +Subject: [PATCH] first + +Please apply. + +From 1234abcd5678ef90aaaa Mon Sep 17 00:00:00 2001 +From: Alice +Subject: inline patch + +From what I know, this line is prose and not a separator. + +From bob at example.org Mon Feb 2 12:00:00 2026 +From: bob at example.org (Bob) +Subject: Re: [PATCH] first + +Looks good. +"; + let messages = split_mbox(mbox); + assert_eq!(messages.len(), 2); + assert!(messages[0].starts_with("From: alice at example.org")); + assert!(messages[0].contains("From what I know")); + assert!(messages[0].contains("From 1234abcd5678ef90aaaa")); + assert!(messages[1].starts_with("From: bob at example.org")); + assert!(messages[1].ends_with("Looks good.")); + } + + #[test] + fn test_since_month() { + assert_eq!(since_month("2025-07-15").unwrap(), (2025, 7)); + assert_eq!(since_month("2024-01-01").unwrap(), (2024, 1)); + assert!(since_month("1 year ago").is_ok()); + assert!(since_month("not a date").is_err()); + } + + #[test] + fn test_files_to_download() { + let remote: Vec = [(2025, 11), (2025, 12), (2026, 1), (2026, 2)] + .iter() + .map(|&(year, month)| ArchiveFile { + file_name: format!("{}-{}.txt.gz", year, MONTH_NAMES[month as usize - 1]), + year, + month, + }) + .collect(); + + let dir = TempDir::new().unwrap(); + + // Fresh directory, no cutoff: everything is downloaded + let all = files_to_download(&remote, dir.path(), None).unwrap(); + assert_eq!(all.len(), 4); + + // Fresh directory with cutoff: only months at or after the cutoff + let since = files_to_download(&remote, dir.path(), Some((2026, 1))).unwrap(); + let keys: Vec<(u32, u32)> = since.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + + // Existing since-limited download: refresh re-fetches the newest + // local month and anything newer, but not older missing months + std::fs::write(dir.path().join("2026-January.txt.gz"), "x").unwrap(); + let refresh = files_to_download(&remote, dir.path(), None).unwrap(); + let keys: Vec<(u32, u32)> = refresh.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + + // Explicit earlier cutoff extends the backfill + let extend = files_to_download(&remote, dir.path(), Some((2025, 12))).unwrap(); + let keys: Vec<(u32, u32)> = extend.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2025, 12), (2026, 1), (2026, 2)]); + + // A gap inside the local range is backfilled on refresh + std::fs::write(dir.path().join("2025-November.txt.gz"), "x").unwrap(); + let gap = files_to_download(&remote, dir.path(), None).unwrap(); + let keys: Vec<(u32, u32)> = gap.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2025, 12), (2026, 1), (2026, 2)]); + } + + #[test] + fn test_save_load_archive_since() { + let dir = TempDir::new().unwrap(); + assert_eq!(load_archive_since(dir.path()), None); + save_archive_since(dir.path(), (2025, 7)).unwrap(); + assert_eq!(load_archive_since(dir.path()), Some((2025, 7))); + } + + #[test] + fn test_files_to_download_stale_partial_download() { + // An aborted unbounded run left only old months on disk. A rerun + // with a cutoff must fetch just the months at or after the cutoff, + // not everything after the newest stale month. + let remote: Vec = [(2025, 11), (2025, 12), (2026, 1), (2026, 2)] + .iter() + .map(|&(year, month)| ArchiveFile { + file_name: format!("{}-{}.txt.gz", year, MONTH_NAMES[month as usize - 1]), + year, + month, + }) + .collect(); + + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("2025-November.txt.gz"), "x").unwrap(); + + let files = files_to_download(&remote, dir.path(), Some((2026, 1))).unwrap(); + let keys: Vec<(u32, u32)> = files.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + } + + #[test] + fn test_archive_storage_dir() { + let dir = + archive_storage_dir("/tmp/db", "https://lists.denx.de/pipermail/u-boot/").unwrap(); + assert_eq!(dir, PathBuf::from("/tmp/db/pipermail/lists.denx.de/u-boot")); + assert!(archive_storage_dir("/tmp/db", "ftp://foo/bar/").is_err()); + } +}