diff --git a/README.md b/README.md index 03200f2..f030505 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ Rust indexing is now supported. This just uses treesitter, but all the semcode features are there. Now with lore indexing! semcode-index --lore lkml,netdev (or any list names) will -pull down the latest git archive from those lists. See [the lore documentation](docs/lore.md) +pull down the latest git archive from those lists. Projects using pipermail +archives instead can use semcode-index --pipermail . See +[the lore documentation](docs/lore.md) for more details. This is a database schema change, so you'll need to reindex. Recent commits introduced indexes for git commit history, as well as diff --git a/docs/lore.md b/docs/lore.md index 9a2130c..9a86f3e 100644 --- a/docs/lore.md +++ b/docs/lore.md @@ -104,6 +104,50 @@ When called without arguments, `--lore` discovers all git repositories under not yet in the database. Use this for a simple one-command workflow to keep lore archives up to date. +### Indexing Pipermail Archives + +Projects that publish their mailing list archives with pipermail (Mailman 2) +instead of public-inbox/lore can be indexed with `--pipermail`, passing the +base URL of the archive index page: + +```bash +# Index a pipermail archive (e.g., the U-Boot mailing list) +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ + +# Index multiple archives at once +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/,https://lists.denx.de/pipermail/eldk/ + +# Refresh all previously downloaded pipermail archives +semcode-index --pipermail +``` + +To avoid downloading decades of history, limit how far back the archives go +with `--pipermail-since` (month granularity): + +```bash +# Only index emails from the previous year onwards +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ --pipermail-since "1 year ago" + +# Absolute dates work too +semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/ --pipermail-since 2025-01-01 +``` + +The cutoff is recorded in the archive directory (`archive.since`) and +persists across refreshes: `semcode-index --pipermail` keeps honouring it. +Archives downloaded without a cutoff never reach further back than their +oldest already-downloaded month. To extend an archive further into the +past later, pass an earlier `--pipermail-since` explicitly. + +The monthly mbox files (`YYYY-Month.txt.gz`) linked from the index page are +downloaded to `/pipermail///` and split into individual +messages, which are stored in the same lore tables as public-inbox archives. +All `lore`, `dig`, and `vlore` commands work on them identically. + +Since there is no backing git repository, each message is identified by a +content hash instead of a commit SHA. Refreshing re-downloads only months +that are missing locally plus the newest month (which keeps growing until +the next month starts); already-indexed messages are skipped. + ### Optional: Generate Vector Embeddings for Semantic Search To enable semantic search with the `vlore` command: diff --git a/src/bin/index.rs b/src/bin/index.rs index 214d9d0..9facb78 100644 --- a/src/bin/index.rs +++ b/src/bin/index.rs @@ -4,6 +4,7 @@ use clap::Parser; use colored::Colorize; use semcode::indexer::{ list_shas_in_range, process_commits_pipeline, process_lore_commits_pipeline, + process_mbox_messages_pipeline, }; use semcode::{measure, process_database_path, CodeVectorizer, DatabaseManager}; // Temporary call relationships are now embedded in function JSON columns @@ -104,6 +105,20 @@ struct Args { #[arg(long, value_name = "LIST", value_delimiter = ',', num_args = 0..)] lore: Option>, + /// Download and index pipermail mbox archives into /pipermail// + /// Accepts comma-separated archive base URLs + /// (e.g., --pipermail https://lists.denx.de/pipermail/u-boot/) + /// Without arguments, refreshes all previously downloaded archives + #[arg(long, value_name = "URL", value_delimiter = ',', num_args = 0..)] + pipermail: Option>, + + /// Only download and index pipermail archives from this date onwards + /// (month granularity). Accepts 'YYYY-MM-DD' or relative dates like + /// '1 year ago'. The cutoff is recorded per archive and refreshes + /// keep honouring it unless an earlier date is given explicitly. + #[arg(long, value_name = "DATE", requires = "pipermail")] + pipermail_since: Option, + // ==================== Multi-Branch Indexing ==================== /// Index a specific branch (can be specified multiple times) /// Example: --branch main --branch develop @@ -712,6 +727,106 @@ async fn index_lore_archive( }) } +/// Shared settings and insertion counters for a pipermail indexing run +struct PipermailRun { + batch_size: usize, + since: Option<(u32, u32)>, + batches_inserted: Arc, + optimization_check_timer: Arc>, +} + +/// Download new monthly mbox files for a pipermail archive and index any +/// messages not yet in the database. `existing` carries the set of already +/// indexed message ids and is updated as messages are queued for insertion. +async fn process_pipermail_archive( + base_url: &str, + db_path: &str, + db_manager: &Arc, + run: &PipermailRun, + existing: &mut HashSet, +) -> Result { + use semcode::pipermail; + + let base_url = pipermail::normalize_base_url(base_url); + let archive_dir = pipermail::archive_storage_dir(db_path, &base_url)?; + std::fs::create_dir_all(&archive_dir)?; + pipermail::save_archive_url(&archive_dir, &base_url)?; + + // An explicit cutoff is recorded for later refreshes; without one, + // fall back to the cutoff recorded by a previous run. + let since = match run.since { + Some(s) => { + pipermail::save_archive_since(&archive_dir, s)?; + Some(s) + } + None => pipermail::load_archive_since(&archive_dir), + }; + + println!("Fetching archive index from {}...", base_url); + let index_html = { + let url = base_url.clone(); + tokio::task::spawn_blocking(move || pipermail::fetch_index_page(&url)).await?? + }; + let remote_files = pipermail::discover_archive_files(&index_html); + if remote_files.is_empty() { + return Err(anyhow::anyhow!("No monthly archives found at {}", base_url)); + } + + let to_download = pipermail::files_to_download(&remote_files, &archive_dir, since)?; + println!( + "{} monthly archives on server, {} to download", + remote_files.len(), + to_download.len() + ); + for file in to_download { + let url = format!("{}{}", base_url, file.file_name); + let dest = archive_dir.join(&file.file_name); + println!(" Downloading {}", file.file_name); + tokio::task::spawn_blocking(move || pipermail::download_file(&url, &dest)).await??; + } + + let mut new_emails = 0usize; + let mut total_emails = 0usize; + for (file, path) in pipermail::list_local_mbox_files(&archive_dir)? { + if since.is_some_and(|s| file.key() < s) { + continue; + } + let content = pipermail::read_mbox_file(&path)?; + let messages = pipermail::split_mbox(&content); + total_emails += messages.len(); + + let mut new_messages = Vec::new(); + for message in messages { + let id = semcode::hash::compute_blake3_hash(&message); + if existing.insert(id.clone()) { + new_messages.push((id, message)); + } + } + if new_messages.is_empty() { + continue; + } + + println!( + " {}: indexing {} new messages", + file.file_name, + new_messages.len() + ); + new_emails += process_mbox_messages_pipeline( + new_messages, + db_manager.clone(), + run.batch_size, + run.batches_inserted.clone(), + run.optimization_check_timer.clone(), + ) + .await?; + } + + Ok(LoreIndexResult { + new_emails, + total_emails, + }) +} + // ==================== Branch Indexing Support ==================== /// Collect branches to index from the various branch-related CLI flags @@ -1308,6 +1423,137 @@ async fn main() -> Result<()> { } } + // Handle --pipermail option if provided + if let Some(pipermail_args) = &args.pipermail { + // If --pipermail has arguments, download and index the specified archives + // If --pipermail has no arguments, refresh all previously downloaded archives + let base_urls: Vec = if !pipermail_args.is_empty() { + pipermail_args.clone() + } else { + let saved = semcode::pipermail::discover_saved_archives(&database_path)?; + if saved.is_empty() { + println!("No pipermail archives have been downloaded yet."); + println!(); + println!("To index pipermail mailing list archives, specify archive base URLs:"); + println!(" semcode-index --pipermail [,...]"); + println!(); + println!("Example:"); + println!(" semcode-index --pipermail https://lists.denx.de/pipermail/u-boot/"); + return Ok(()); + } + println!("Found {} pipermail archive(s) to refresh:", saved.len()); + for (_, url) in &saved { + println!(" - {}", url); + } + saved.into_iter().map(|(_, url)| url).collect() + }; + + info!( + "Pipermail archive processing requested for {} archives", + base_urls.len() + ); + + let since = match args.pipermail_since.as_deref() { + Some(date_str) => { + let (year, month) = semcode::pipermail::since_month(date_str)?; + println!("Limiting archives to {}-{:02} onwards", year, month); + Some((year, month)) + } + None => None, + }; + + let db_manager = + DatabaseManager::new(&database_path, args.source.to_string_lossy().to_string()).await?; + db_manager.create_tables().await?; + let db_manager = Arc::new(db_manager); + + let start_time = std::time::Instant::now(); + let run = PipermailRun { + batch_size: 1024, + since, + batches_inserted: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + optimization_check_timer: Arc::new(std::sync::Mutex::new(std::time::Instant::now())), + }; + + println!("Checking for already-indexed messages..."); + let mut existing = db_manager.get_indexed_lore_commits().await?; + + let mut total_new_emails = 0usize; + let mut total_emails_all_archives = 0usize; + let mut failed_archives: Vec<(String, String)> = Vec::new(); + + for base_url in &base_urls { + println!("\n=== Processing pipermail archive: {} ===", base_url); + match process_pipermail_archive( + base_url, + &database_path, + &db_manager, + &run, + &mut existing, + ) + .await + { + Ok(result) => { + println!( + "Indexed {} new emails from {} (total in archive: {})", + result.new_emails, base_url, result.total_emails + ); + total_new_emails += result.new_emails; + total_emails_all_archives += result.total_emails; + } + Err(e) => { + eprintln!("Error processing {}: {:#}", base_url, e); + failed_archives.push((base_url.clone(), e.to_string())); + } + } + } + + let total_time = start_time.elapsed(); + + println!("\n=== Pipermail Email Indexing Complete ==="); + println!("Total time: {:.1}s", total_time.as_secs_f64()); + println!( + "Archives processed: {}/{}", + base_urls.len() - failed_archives.len(), + base_urls.len() + ); + println!("New emails indexed: {}", total_new_emails); + println!( + "Total emails across archives: {}", + total_emails_all_archives + ); + + if !failed_archives.is_empty() { + eprintln!("\nFailed archives:"); + for (name, err) in &failed_archives { + eprintln!(" {}: {}", name, err); + } + } + + if total_new_emails > 0 { + println!("\nCompacting lore tables..."); + match db_manager.compact_lore_tables().await { + Ok(_) => println!("Lore table compaction completed successfully"), + Err(e) => error!("Failed to compact lore tables: {}", e), + } + + println!("\nUpdating FTS indices for lore table..."); + match db_manager.ensure_lore_fts_indices().await { + Ok(_) => {} + Err(e) => eprintln!("Warning: Failed to ensure FTS indices: {}", e), + } + match db_manager.optimize_lore_fts_indices().await { + Ok(_) => println!("FTS indices updated successfully"), + Err(e) => eprintln!("Warning: Failed to optimize FTS indices: {}", e), + } + } + + println!("\nTo query this database, run:"); + println!(" semcode --database {}", database_path); + + return Ok(()); + } + // Validate mutually exclusive options if args.git.is_some() && args.commits.is_some() { return Err(anyhow::anyhow!( diff --git a/src/database/schema.rs b/src/database/schema.rs index b4a4026..7e28ef0 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -61,16 +61,18 @@ impl SchemaManager { self.create_commit_vectors_table().await?; } + // Must exist before migrate_lore_table(), which reconciles + // against it when adding the date_timestamp column. + if !table_names.iter().any(|n| n == "lore_indexed_commits") { + self.create_lore_indexed_commits_table().await?; + } + if !table_names.iter().any(|n| n == "lore") { self.create_lore_table().await?; } else { self.migrate_lore_table().await?; } - if !table_names.iter().any(|n| n == "lore_indexed_commits") { - self.create_lore_indexed_commits_table().await?; - } - if !table_names.iter().any(|n| n == "lore_vectors") { self.create_lore_vectors_table().await?; } @@ -1654,3 +1656,55 @@ impl SchemaManager { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// Databases created before lore_indexed_commits existed have a lore + /// table without the date_timestamp column. create_all_tables() must + /// create lore_indexed_commits before migrating the lore table, since + /// the migration reconciles against it. + #[tokio::test] + async fn test_create_all_tables_migrates_old_lore_schema() { + let tmpdir = TempDir::new().unwrap(); + let connection = lancedb::connect(tmpdir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let old_schema = Arc::new(Schema::new(vec![ + Field::new("git_commit_sha", DataType::Utf8, false), + Field::new("from", DataType::Utf8, false), + Field::new("date", DataType::Utf8, false), + Field::new("message_id", DataType::Utf8, false), + Field::new("in_reply_to", DataType::Utf8, true), + Field::new("subject", DataType::Utf8, false), + Field::new("references", DataType::Utf8, true), + Field::new("recipients", DataType::Utf8, false), + Field::new("body", DataType::Utf8, false), + Field::new("symbols", DataType::Utf8, false), + ])); + let empty_batch = RecordBatch::new_empty(old_schema.clone()); + connection + .create_table("lore", vec![empty_batch]) + .execute() + .await + .unwrap(); + + let manager = SchemaManager::new(connection.clone()); + manager.create_all_tables().await.unwrap(); + + let tables = connection.table_names().execute().await.unwrap(); + assert!(tables.iter().any(|n| n == "lore_indexed_commits")); + + let lore = connection.open_table("lore").execute().await.unwrap(); + assert!(lore + .schema() + .await + .unwrap() + .column_with_name("date_timestamp") + .is_some()); + } +} diff --git a/src/indexer.rs b/src/indexer.rs index a7a9f48..7ae240d 100644 --- a/src/indexer.rs +++ b/src/indexer.rs @@ -421,6 +421,13 @@ pub fn parse_email_from_commit( let blob = object.try_into_blob()?; let email_content = String::from_utf8_lossy(blob.data.as_slice()).to_string(); + parse_email_content(&email_content, commit_sha) +} + +/// Parse a raw email message (headers and body) into a LoreEmailInfo. +/// The id becomes the git_commit_sha field: the containing commit SHA for +/// public-inbox archives, or a content hash for mbox-sourced messages. +pub fn parse_email_content(email_content: &str, id: &str) -> Result { // Parse email headers let mut headers = EmailHeaders::new(); @@ -480,7 +487,7 @@ pub fn parse_email_from_commit( let date_timestamp = parse_rfc2822_to_timestamp(&headers.date); Ok(crate::LoreEmailInfo { - git_commit_sha: commit_sha.to_string(), + git_commit_sha: id.to_string(), from: headers.from, date: headers.date, date_timestamp, @@ -1058,6 +1065,55 @@ pub async fn process_lore_commits_pipeline( Ok(()) } +/// Parse raw mbox messages in parallel and insert them into the lore tables. +/// `messages` holds (id, raw email) pairs where id is a stable content hash +/// used as the lore git_commit_sha. Returns the number of emails stored. +pub async fn process_mbox_messages_pipeline( + messages: Vec<(String, String)>, + db_manager: Arc, + batch_size: usize, + batches_inserted: Arc, + optimization_check_timer: Arc>, +) -> Result { + use rayon::prelude::*; + + let emails: Vec = messages + .par_iter() + .filter_map(|(id, content)| match parse_email_content(content, id) { + Ok(email) => Some(email), + Err(e) => { + warn!("Failed to parse mbox message {}: {}", id, e); + None + } + }) + .collect(); + + let mut inserted = 0usize; + for chunk in emails.chunks(batch_size) { + let failed_indices = db_manager.insert_lore_emails(chunk).await?; + let failed_set: HashSet = failed_indices.into_iter().collect(); + + // Record ids only for emails that were actually stored, so that + // failed emails are retried on the next run. + let ids: Vec = chunk + .iter() + .enumerate() + .filter(|(i, _)| !failed_set.contains(i)) + .map(|(_, e)| e.git_commit_sha.clone()) + .collect(); + inserted += ids.len(); + if !ids.is_empty() { + db_manager.insert_lore_indexed_commits(&ids).await?; + } + + let total_batches = batches_inserted.fetch_add(1, Ordering::Relaxed) + 1; + check_and_optimize_if_needed(&db_manager, 0, total_batches, &optimization_check_timer) + .await; + } + + Ok(inserted) +} + /// Index commits in a git range pub async fn index_git_commits( repo_path: &PathBuf, diff --git a/src/lib.rs b/src/lib.rs index b1c79d0..4aa2af3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod git_range; pub mod hash; pub mod indexer; pub mod perf_monitor; +pub mod pipermail; pub mod symbol_walkback; pub mod text_utils; mod treesitter_analyzer; diff --git a/src/pipermail.rs b/src/pipermail.rs new file mode 100644 index 0000000..798c169 --- /dev/null +++ b/src/pipermail.rs @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 +//! Pipermail (Mailman 2) mailing list archive support +//! +//! Downloads the monthly mbox archives published by a pipermail archive +//! index page (e.g. https://lists.denx.de/pipermail/u-boot/) and splits +//! them into individual email messages for lore indexing. + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// File written into each archive directory recording the base URL, +/// so that `--pipermail` without arguments can refresh the archive. +pub const URL_FILE: &str = "archive.url"; + +/// File written into each archive directory recording the +/// `--pipermail-since` cutoff as "YYYY-MM", so refreshes keep honouring it. +pub const SINCE_FILE: &str = "archive.since"; + +const MONTH_NAMES: [&str; 12] = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +]; + +/// A monthly mbox archive published on a pipermail index page +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ArchiveFile { + pub file_name: String, + pub year: u32, + pub month: u32, +} + +impl ArchiveFile { + pub fn key(&self) -> (u32, u32) { + (self.year, self.month) + } +} + +/// Parse an archive file stem like "2026-February" into (year, month) +pub fn month_key(stem: &str) -> Option<(u32, u32)> { + let (year, month_name) = stem.split_once('-')?; + let year = year.parse::().ok()?; + let month = MONTH_NAMES.iter().position(|m| *m == month_name)? as u32 + 1; + Some((year, month)) +} + +/// Strip the ".txt" or ".txt.gz" suffix from an archive file name +fn archive_file_stem(file_name: &str) -> Option<&str> { + file_name + .strip_suffix(".txt.gz") + .or_else(|| file_name.strip_suffix(".txt")) +} + +/// Extract the monthly mbox files linked from a pipermail index page. +/// Prefers the gzipped variant when both are listed. Results are sorted +/// chronologically (oldest first). +pub fn discover_archive_files(index_html: &str) -> Vec { + static HREF_RE: OnceLock = OnceLock::new(); + let re = HREF_RE.get_or_init(|| { + regex::Regex::new(r#"(?i)href="(\d{4}-[A-Za-z]+\.txt(?:\.gz)?)""#).unwrap() + }); + + let mut by_month: std::collections::HashMap<(u32, u32), ArchiveFile> = + std::collections::HashMap::new(); + + for cap in re.captures_iter(index_html) { + let file_name = cap[1].to_string(); + let Some(stem) = archive_file_stem(&file_name) else { + continue; + }; + let Some((year, month)) = month_key(stem) else { + continue; + }; + let entry = ArchiveFile { + file_name: file_name.clone(), + year, + month, + }; + by_month + .entry((year, month)) + .and_modify(|existing| { + if file_name.ends_with(".gz") && !existing.file_name.ends_with(".gz") { + *existing = entry.clone(); + } + }) + .or_insert(entry); + } + + let mut files: Vec = by_month.into_values().collect(); + files.sort_by_key(|f| f.key()); + files +} + +/// List locally downloaded monthly mbox files, sorted chronologically +pub fn list_local_mbox_files(dir: &Path) -> Result> { + let mut files = Vec::new(); + + if !dir.exists() { + return Ok(files); + } + + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if !path.is_file() { + continue; + } + let Some(file_name) = path.file_name().and_then(|n| n.to_str()) else { + continue; + }; + let Some(stem) = archive_file_stem(file_name) else { + continue; + }; + let Some((year, month)) = month_key(stem) else { + continue; + }; + files.push(( + ArchiveFile { + file_name: file_name.to_string(), + year, + month, + }, + path, + )); + } + + files.sort_by_key(|(f, _)| f.key()); + Ok(files) +} + +/// Parse a user-supplied date ("2025-07-01", "1 year ago") into a +/// (year, month) archive cutoff +pub fn since_month(date_str: &str) -> Result<(u32, u32)> { + use chrono::Datelike; + + let parsed = chrono_english::parse_date_string( + date_str.trim(), + chrono::Utc::now(), + chrono_english::Dialect::Us, + ) + .map_err(|e| { + anyhow::anyhow!( + "Invalid date '{}': {}. Use 'YYYY-MM-DD' or relative dates like '1 year ago'", + date_str, + e + ) + })?; + + Ok((parsed.year() as u32, parsed.month())) +} + +/// Decide which remote files need downloading: months missing locally, +/// plus the newest local month (it keeps growing until the next month +/// starts). Everything is bounded by `since` when given, otherwise by the +/// oldest month already downloaded, so a `--pipermail-since` cutoff +/// persists across refreshes. +pub fn files_to_download( + remote: &[ArchiveFile], + dir: &Path, + since: Option<(u32, u32)>, +) -> Result> { + let local = list_local_mbox_files(dir)?; + let local_keys: std::collections::HashSet<(u32, u32)> = + local.iter().map(|(f, _)| f.key()).collect(); + let newest_local = local.iter().map(|(f, _)| f.key()).max(); + let horizon = since.or_else(|| local.iter().map(|(f, _)| f.key()).min()); + + Ok(remote + .iter() + .filter(|f| { + if horizon.is_some_and(|h| f.key() < h) { + return false; + } + !local_keys.contains(&f.key()) || newest_local == Some(f.key()) + }) + .cloned() + .collect()) +} + +/// Ensure the archive base URL ends with a trailing slash +pub fn normalize_base_url(base_url: &str) -> String { + let mut url = base_url.trim().to_string(); + if !url.ends_with('/') { + url.push('/'); + } + url +} + +/// Derive the local storage directory for a pipermail archive: +/// `/pipermail//` +pub fn archive_storage_dir(db_path: &str, base_url: &str) -> Result { + let without_scheme = base_url + .strip_prefix("https://") + .or_else(|| base_url.strip_prefix("http://")) + .ok_or_else(|| anyhow::anyhow!("Archive URL must start with http:// or https://"))?; + + let mut segments = without_scheme.split('/').filter(|s| !s.is_empty()); + let host = segments + .next() + .ok_or_else(|| anyhow::anyhow!("Invalid archive URL: {}", base_url))?; + let list = segments + .next_back() + .ok_or_else(|| anyhow::anyhow!("Archive URL has no list name: {}", base_url))?; + + if host.contains("..") || list.contains("..") { + return Err(anyhow::anyhow!("Invalid archive URL: {}", base_url)); + } + + Ok(PathBuf::from(db_path) + .join("pipermail") + .join(host) + .join(list)) +} + +/// Record the base URL of an archive so it can be refreshed later +pub fn save_archive_url(dir: &Path, base_url: &str) -> Result<()> { + std::fs::write(dir.join(URL_FILE), format!("{}\n", base_url)) + .with_context(|| format!("Failed to write {}", dir.join(URL_FILE).display())) +} + +/// Record the cutoff month so refreshes keep honouring it +pub fn save_archive_since(dir: &Path, since: (u32, u32)) -> Result<()> { + let path = dir.join(SINCE_FILE); + std::fs::write(&path, format!("{}-{:02}\n", since.0, since.1)) + .with_context(|| format!("Failed to write {}", path.display())) +} + +/// Load a previously recorded cutoff month, if any +pub fn load_archive_since(dir: &Path) -> Option<(u32, u32)> { + let content = std::fs::read_to_string(dir.join(SINCE_FILE)).ok()?; + let (year, month) = content.trim().split_once('-')?; + let year = year.parse::().ok()?; + let month = month.parse::().ok()?; + (1..=12).contains(&month).then_some((year, month)) +} + +/// Discover previously downloaded archives under `/pipermail/`. +/// Returns (archive directory, base URL) pairs. +pub fn discover_saved_archives(db_path: &str) -> Result> { + let base = PathBuf::from(db_path).join("pipermail"); + let mut archives = Vec::new(); + + fn walk(dir: &Path, archives: &mut Vec<(PathBuf, String)>) -> Result<()> { + let url_file = dir.join(URL_FILE); + if url_file.is_file() { + let url = std::fs::read_to_string(&url_file)?.trim().to_string(); + if !url.is_empty() { + archives.push((dir.to_path_buf(), url)); + } + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + walk(&path, archives)?; + } + } + Ok(()) + } + + if base.is_dir() { + walk(&base, &mut archives)?; + } + + archives.sort(); + Ok(archives) +} + +/// Fetch the archive index page for a pipermail list (blocking) +pub fn fetch_index_page(base_url: &str) -> Result { + let response = reqwest::blocking::get(base_url) + .with_context(|| format!("Failed to fetch archive index {}", base_url))?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Failed to fetch archive index {}: HTTP {}", + base_url, + response.status() + )); + } + Ok(response.text()?) +} + +/// Download a monthly mbox file to the given destination (blocking). +/// Writes to a temporary file first so interrupted downloads are not +/// mistaken for complete archives. +pub fn download_file(url: &str, dest: &Path) -> Result<()> { + let response = + reqwest::blocking::get(url).with_context(|| format!("Failed to download {}", url))?; + if !response.status().is_success() { + return Err(anyhow::anyhow!( + "Failed to download {}: HTTP {}", + url, + response.status() + )); + } + let bytes = response.bytes()?; + + let tmp = dest.with_extension("part"); + std::fs::write(&tmp, &bytes).with_context(|| format!("Failed to write {}", tmp.display()))?; + std::fs::rename(&tmp, dest)?; + Ok(()) +} + +/// Read a downloaded mbox file, transparently decompressing gzip. +/// Invalid UTF-8 sequences are replaced rather than rejected since old +/// archives commonly contain legacy 8-bit encodings. +pub fn read_mbox_file(path: &Path) -> Result { + use std::io::Read; + + let raw = std::fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; + let bytes = if path.extension().and_then(|e| e.to_str()) == Some("gz") { + let mut decoder = flate2::read::GzDecoder::new(&raw[..]); + let mut decompressed = Vec::new(); + decoder + .read_to_end(&mut decompressed) + .with_context(|| format!("Failed to decompress {}", path.display()))?; + decompressed + } else { + raw + }; + + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} + +/// Match a pipermail mbox message separator. Pipermail does not escape +/// body lines starting with "From ", so the match requires the full +/// From_ line shape (envelope address followed by an asctime date) to +/// avoid splitting on inline patches ("From Mon Sep 17 ...") or +/// prose. The address is usually obfuscated as "user at domain" but a +/// plain address is also accepted. +fn is_mbox_separator(line: &str) -> bool { + static SEPARATOR_RE: OnceLock = OnceLock::new(); + let re = SEPARATOR_RE.get_or_init(|| { + regex::Regex::new( + r"^From (\S+ at \S+|\S+@\S+) +[A-Z][a-z]{2} [A-Z][a-z]{2} +\d{1,2} +\d{1,2}:\d{2}:\d{2} \d{4}$", + ) + .unwrap() + }); + re.is_match(line) +} + +/// Split a pipermail mbox archive into individual messages. A message +/// starts at a From_ separator line preceded by a blank line (or the +/// start of the file); the separator line itself is not included in the +/// returned message text. +pub fn split_mbox(content: &str) -> Vec { + let mut messages = Vec::new(); + let mut current: Vec<&str> = Vec::new(); + let mut in_message = false; + let mut prev_blank = true; + + for line in content.lines() { + if prev_blank && is_mbox_separator(line) { + if in_message && !current.is_empty() { + messages.push(current.join("\n")); + } + current.clear(); + in_message = true; + } else if in_message { + current.push(line); + } + prev_blank = line.is_empty(); + } + + if in_message && !current.is_empty() { + messages.push(current.join("\n")); + } + + messages +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_month_key() { + assert_eq!(month_key("2026-February"), Some((2026, 2))); + assert_eq!(month_key("1999-December"), Some((1999, 12))); + assert_eq!(month_key("2026-Foo"), None); + assert_eq!(month_key("February"), None); + } + + #[test] + fn test_discover_archive_files() { + let html = r#" + [ Gzip'd Text 1 MB ] + [ Gzip'd Text 4 MB ] + [ Text 5 MB ] + [ Gzip'd Text 2 MB ] + thread + "#; + let files = discover_archive_files(html); + let names: Vec<&str> = files.iter().map(|f| f.file_name.as_str()).collect(); + assert_eq!( + names, + vec![ + "2025-December.txt.gz", + "2026-June.txt.gz", + "2026-July.txt.gz" + ] + ); + } + + #[test] + fn test_split_mbox() { + let mbox = "\ +From alice at example.org Sun Feb 1 00:38:12 2026 +From: alice at example.org (Alice) +Subject: [PATCH] first + +Please apply. + +From 1234abcd5678ef90aaaa Mon Sep 17 00:00:00 2001 +From: Alice +Subject: inline patch + +From what I know, this line is prose and not a separator. + +From bob at example.org Mon Feb 2 12:00:00 2026 +From: bob at example.org (Bob) +Subject: Re: [PATCH] first + +Looks good. +"; + let messages = split_mbox(mbox); + assert_eq!(messages.len(), 2); + assert!(messages[0].starts_with("From: alice at example.org")); + assert!(messages[0].contains("From what I know")); + assert!(messages[0].contains("From 1234abcd5678ef90aaaa")); + assert!(messages[1].starts_with("From: bob at example.org")); + assert!(messages[1].ends_with("Looks good.")); + } + + #[test] + fn test_since_month() { + assert_eq!(since_month("2025-07-15").unwrap(), (2025, 7)); + assert_eq!(since_month("2024-01-01").unwrap(), (2024, 1)); + assert!(since_month("1 year ago").is_ok()); + assert!(since_month("not a date").is_err()); + } + + #[test] + fn test_files_to_download() { + let remote: Vec = [(2025, 11), (2025, 12), (2026, 1), (2026, 2)] + .iter() + .map(|&(year, month)| ArchiveFile { + file_name: format!("{}-{}.txt.gz", year, MONTH_NAMES[month as usize - 1]), + year, + month, + }) + .collect(); + + let dir = TempDir::new().unwrap(); + + // Fresh directory, no cutoff: everything is downloaded + let all = files_to_download(&remote, dir.path(), None).unwrap(); + assert_eq!(all.len(), 4); + + // Fresh directory with cutoff: only months at or after the cutoff + let since = files_to_download(&remote, dir.path(), Some((2026, 1))).unwrap(); + let keys: Vec<(u32, u32)> = since.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + + // Existing since-limited download: refresh re-fetches the newest + // local month and anything newer, but not older missing months + std::fs::write(dir.path().join("2026-January.txt.gz"), "x").unwrap(); + let refresh = files_to_download(&remote, dir.path(), None).unwrap(); + let keys: Vec<(u32, u32)> = refresh.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + + // Explicit earlier cutoff extends the backfill + let extend = files_to_download(&remote, dir.path(), Some((2025, 12))).unwrap(); + let keys: Vec<(u32, u32)> = extend.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2025, 12), (2026, 1), (2026, 2)]); + + // A gap inside the local range is backfilled on refresh + std::fs::write(dir.path().join("2025-November.txt.gz"), "x").unwrap(); + let gap = files_to_download(&remote, dir.path(), None).unwrap(); + let keys: Vec<(u32, u32)> = gap.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2025, 12), (2026, 1), (2026, 2)]); + } + + #[test] + fn test_save_load_archive_since() { + let dir = TempDir::new().unwrap(); + assert_eq!(load_archive_since(dir.path()), None); + save_archive_since(dir.path(), (2025, 7)).unwrap(); + assert_eq!(load_archive_since(dir.path()), Some((2025, 7))); + } + + #[test] + fn test_files_to_download_stale_partial_download() { + // An aborted unbounded run left only old months on disk. A rerun + // with a cutoff must fetch just the months at or after the cutoff, + // not everything after the newest stale month. + let remote: Vec = [(2025, 11), (2025, 12), (2026, 1), (2026, 2)] + .iter() + .map(|&(year, month)| ArchiveFile { + file_name: format!("{}-{}.txt.gz", year, MONTH_NAMES[month as usize - 1]), + year, + month, + }) + .collect(); + + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("2025-November.txt.gz"), "x").unwrap(); + + let files = files_to_download(&remote, dir.path(), Some((2026, 1))).unwrap(); + let keys: Vec<(u32, u32)> = files.iter().map(|f| f.key()).collect(); + assert_eq!(keys, vec![(2026, 1), (2026, 2)]); + } + + #[test] + fn test_archive_storage_dir() { + let dir = + archive_storage_dir("/tmp/db", "https://lists.denx.de/pipermail/u-boot/").unwrap(); + assert_eq!(dir, PathBuf::from("/tmp/db/pipermail/lists.denx.de/u-boot")); + assert!(archive_storage_dir("/tmp/db", "ftp://foo/bar/").is_err()); + } +}