From 96acd2be9561ad841975e46171020d8b5c0612cd Mon Sep 17 00:00:00 2001 From: Leb-Sun <266895696+Leb-Sun@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:03:31 +0000 Subject: [PATCH] fix(steam): sweep game files dropped from a depot manifest --- .../wn-steam-client/rust/src/depot_cleanup.rs | 1182 +++++++++++++++++ .../wn-steam-client/rust/src/depot_config.rs | 25 +- .../rust/src/depot_downloader.rs | 242 ++++ .../main/cpp/wn-steam-client/rust/src/jni.rs | 4 +- .../main/cpp/wn-steam-client/rust/src/lib.rs | 1 + .../stores/steam/service/SteamService.kt | 15 + .../service/SteamServiceDownloadFinalize.kt | 2 + .../steam/service/SteamServiceDownloadMain.kt | 5 +- .../stores/steam/service/SteamServiceLogin.kt | 44 + .../stores/steam/wnsteam/WnSteamSession.kt | 6 +- 10 files changed, 1522 insertions(+), 4 deletions(-) create mode 100644 app/src/main/cpp/wn-steam-client/rust/src/depot_cleanup.rs diff --git a/app/src/main/cpp/wn-steam-client/rust/src/depot_cleanup.rs b/app/src/main/cpp/wn-steam-client/rust/src/depot_cleanup.rs new file mode 100644 index 000000000..33b6ab70d --- /dev/null +++ b/app/src/main/cpp/wn-steam-client/rust/src/depot_cleanup.rs @@ -0,0 +1,1182 @@ +use crate::content_manifest::ContentManifest; +use crate::depot_config::{atomic_write_synced, DepotConfigStore, INVALID_MANIFEST_ID}; +use crate::depot_downloader::ResolvedDepotSpec; +use crate::depot_writer::DEPOT_FILE_FLAG_DIRECTORY; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +const STALE_CLEANUP_SUFFIX: &str = ".stalecleanup"; +const FILELIST_SUFFIX: &str = ".filelist"; +const FILELIST_HEADER: &str = "WNFL1"; + +fn cleanup_log(message: &str) { + crate::jni::android_log("WnSteamDepotCleanup", message); +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FileEntry { + pub name: String, + pub is_dir: bool, +} + +pub fn stale_cleanup_marker_name(depot_id: u32, manifest_id: u64) -> String { + format!("{depot_id}_{manifest_id}{STALE_CLEANUP_SUFFIX}") +} + +pub fn stale_cleanup_marker_path( + config_dir: impl AsRef, + depot_id: u32, + manifest_id: u64, +) -> PathBuf { + config_dir + .as_ref() + .join(stale_cleanup_marker_name(depot_id, manifest_id)) +} + +pub fn filelist_sidecar_path( + config_dir: impl AsRef, + depot_id: u32, + manifest_id: u64, +) -> PathBuf { + config_dir + .as_ref() + .join(format!("{depot_id}_{manifest_id}{FILELIST_SUFFIX}")) +} + +/// Persists a manifest's decrypted file list next to its cache, so the pass can +/// read a depot it holds no key for. Written only for depots that reach this +/// downloader, so an install predating sidecars has none until each depot has +/// been downloaded once (a full Verify Files does them all); until then the +/// keep-union is incomplete and the pass correctly defers. +pub fn write_filelist_sidecar( + config_dir: impl AsRef, + depot_id: u32, + manifest_id: u64, + manifest: &ContentManifest, +) -> bool { + let mut body = String::with_capacity(manifest.files.len() * 32); + let mut written = 0usize; + for file in &manifest.files { + if file.filename.contains('\n') || file.filename.contains('\r') { + continue; + } + written += 1; + let kind = if (file.flags & DEPOT_FILE_FLAG_DIRECTORY) != 0 { + 'D' + } else { + 'F' + }; + body.push(kind); + body.push(' '); + body.push_str(&file.filename); + body.push('\n'); + } + // Entry count in the header: a truncated sidecar for an installed depot would + // otherwise under-populate the keep-union and the difference gets deleted. + let blob = format!("{FILELIST_HEADER} {written}\n{body}"); + atomic_write_synced( + &filelist_sidecar_path(config_dir, depot_id, manifest_id), + blob.as_bytes(), + ) +} + +/// Writes the sidecar for an already-installed depot that predates sidecars, +/// using the cached manifest and this operation's key. No-op when the sidecar +/// exists or the manifest cache is unreadable. +pub fn backfill_filelist_sidecar(config_dir: &Path, depot: &ResolvedDepotSpec) { + if filelist_sidecar_path(config_dir, depot.depot_id, depot.manifest_id).is_file() { + return; + } + if let Some(manifest) = load_manifest( + config_dir, + depot.depot_id, + depot.manifest_id, + &depot.depot_key, + ) { + let _ = write_filelist_sidecar(config_dir, depot.depot_id, depot.manifest_id, &manifest); + } +} + +fn read_filelist_sidecar( + config_dir: &Path, + depot_id: u32, + manifest_id: u64, +) -> Option> { + let blob = fs::read_to_string(filelist_sidecar_path(config_dir, depot_id, manifest_id)).ok()?; + let mut lines = blob.lines(); + let expected: usize = lines.next()?.strip_prefix(FILELIST_HEADER)?.trim().parse().ok()?; + let mut entries = Vec::new(); + for line in lines { + let (kind, name) = match (line.get(..2), line.get(2..)) { + // An empty name is a torn last line, not an entry — dropping it lets + // the count check below catch the truncation. + (Some("F "), Some(name)) if !name.is_empty() => (false, name), + (Some("D "), Some(name)) if !name.is_empty() => (true, name), + _ => continue, + }; + entries.push(FileEntry { + name: name.to_string(), + is_dir: kind, + }); + } + if entries.len() != expected { + cleanup_log(&format!( + "cleanup: file list {depot_id}_{manifest_id} is truncated \ + ({} of {expected} entries), treating as unreadable", + entries.len() + )); + return None; + } + Some(entries) +} + +/// Records that a depot is about to move off `old_manifest_id`, so the files +/// the old manifest installed can be diffed away once the new build is fully +/// on disk. The marker survives pause/cancel — cleanup only ever runs after a +/// fully successful download, and a leftover marker is retried then. +pub fn record_pending_cleanup( + config_dir: impl AsRef, + depot_id: u32, + old_manifest_id: u64, + new_manifest_id: u64, +) -> bool { + if old_manifest_id == 0 + || old_manifest_id == INVALID_MANIFEST_ID + || old_manifest_id == new_manifest_id + { + return false; + } + write_cleanup_marker(config_dir, depot_id, old_manifest_id) +} + +/// Marks a build whose write started but never committed — its files can sit on +/// disk with no depot.config entry to ever diff them away. Dropped if the build +/// later commits (resume), else reclaimed by the next successful download. +pub fn record_aborted_build(config_dir: impl AsRef, depot_id: u32, manifest_id: u64) -> bool { + if manifest_id == 0 || manifest_id == INVALID_MANIFEST_ID { + return false; + } + write_cleanup_marker(config_dir, depot_id, manifest_id) +} + +fn write_cleanup_marker(config_dir: impl AsRef, depot_id: u32, manifest_id: u64) -> bool { + let path = stale_cleanup_marker_path(config_dir, depot_id, manifest_id); + let Some(parent) = path.parent() else { + return false; + }; + if fs::create_dir_all(parent).is_err() { + return false; + } + fs::write(path, manifest_id.to_string()).is_ok() +} + +pub fn pending_cleanup_markers(config_dir: impl AsRef) -> Vec<(u32, u64)> { + let Ok(entries) = fs::read_dir(config_dir.as_ref()) else { + return Vec::new(); + }; + let mut markers = BTreeSet::new(); + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(stem) = name.strip_suffix(STALE_CLEANUP_SUFFIX) else { + continue; + }; + let mut parts = stem.split('_'); + let (Some(depot), Some(gid), None) = (parts.next(), parts.next(), parts.next()) else { + continue; + }; + if let (Ok(depot_id), Ok(manifest_id)) = (depot.parse::(), gid.parse::()) { + markers.insert((depot_id, manifest_id)); + } + } + markers.into_iter().collect() +} + +fn remove_cleanup_marker(config_dir: impl AsRef, depot_id: u32, manifest_id: u64) { + let _ = fs::remove_file(stale_cleanup_marker_path(config_dir, depot_id, manifest_id)); +} + +/// Deletes files an old manifest installed that no current manifest references — +/// the gap left because a download only ever writes new content. +/// +/// Candidates come only from cached manifest file lists, minus the union of +/// every installed manifest's files, so anything absent from all Steam manifests +/// can never be a candidate — saves, steam_settings/ and user-added mod files +/// are structurally safe. The exception is a Steamless `.exe.original.exe` +/// backup, deleted with its primary so a stale patch can't be restored. +/// +/// Marker-driven builds are the guarantee; unmarked cached builds are swept +/// best-effort, since pruneStaleDepotManifestCache on the Kotlin side reclaims +/// those same cached lists once depot.config no longer references them. +/// Returns the number of files deleted. +pub fn run_stale_file_cleanup( + install_dir: &str, + config_dir: &Path, + depots: &[ResolvedDepotSpec], +) -> u32 { + let keys: BTreeMap = depots + .iter() + .map(|depot| (depot.depot_id, depot.depot_key.as_slice())) + .collect(); + let cfg = DepotConfigStore::load(config_dir); + + // Every old build still on disk: each cached filelist/manifest plus any + // pending marker. Sweeping cached builds (not just marked ones) reclaims + // orphans even when a marker was dropped or never recorded. + let mut old_builds = collect_cached_builds(config_dir); + old_builds.extend(pending_cleanup_markers(config_dir)); + if old_builds.is_empty() { + return 0; + } + + // The keep-union must be COMPLETE: proving a candidate is dead requires + // reading every installed depot, so if one is in progress or unreadable, + // defer the whole pass rather than risk deleting a file that moved depots. + let mut keep = BTreeSet::new(); + let mut current: BTreeMap = BTreeMap::new(); + for (depot_id, gid) in cfg.installed_entries() { + if gid == INVALID_MANIFEST_ID { + cleanup_log(&format!( + "cleanup: depot {depot_id} in progress, deferring stale-file pass" + )); + return 0; + } + match load_file_entries(config_dir, depot_id, gid, keys.get(&depot_id).copied()) { + Some(entries) => { + for entry in &entries { + keep.insert(normalized_key(&entry.name)); + } + current.insert(depot_id, gid); + } + None => { + cleanup_log(&format!( + "cleanup: installed depot {depot_id}_{gid} file list unreadable, \ + deferring stale-file pass" + )); + return 0; + } + } + } + + // An unreadable or absent depot.config loads as an EMPTY store, which is + // indistinguishable from "nothing installed" — and an empty keep-union would + // condemn every cached build, including the one just downloaded. + if current.is_empty() { + cleanup_log("cleanup: no installed depots readable, deferring stale-file pass"); + return 0; + } + + let install_root = Path::new(install_dir); + let mut deleted = 0u32; + for (depot_id, gid) in old_builds { + if current.get(&depot_id) == Some(&gid) { + // This build is the one currently installed — not stale. + remove_cleanup_marker(config_dir, depot_id, gid); + continue; + } + // Absent from depot.config means UNKNOWN, not dead: a fresh run forgets + // its depots before downloading them, so files can be on disk with no + // entry. Only a depot we can see the current build of can be diffed. + if !current.contains_key(&depot_id) { + cleanup_log(&format!( + "cleanup: depot {depot_id} not in depot.config, deferring its builds" + )); + continue; + } + let Some(entries) = + load_file_entries(config_dir, depot_id, gid, keys.get(&depot_id).copied()) + else { + if filelist_sidecar_path(config_dir, depot_id, gid).is_file() + || config_dir + .join(format!("{depot_id}_{gid}.manifest")) + .is_file() + { + // The data is on disk but this op can't read it (no key yet); + // a later op that carries the key finishes the job. + cleanup_log(&format!( + "cleanup: old file list {depot_id}_{gid} unreadable in this op, deferring" + )); + } else { + cleanup_log(&format!( + "cleanup: old file list {depot_id}_{gid} is gone, dropping marker" + )); + remove_cleanup_marker(config_dir, depot_id, gid); + } + continue; + }; + deleted += delete_build_orphans(install_root, depot_id, gid, &entries, &keep); + remove_cleanup_marker(config_dir, depot_id, gid); + } + if deleted > 0 { + cleanup_log(&format!( + "cleanup: removed {deleted} stale file(s) under '{install_dir}'" + )); + } + deleted +} + +/// Deletes the files of one old build that no current build still ships. +fn delete_build_orphans( + install_root: &Path, + depot_id: u32, + gid: u64, + entries: &[FileEntry], + keep: &BTreeSet, +) -> u32 { + let mut deleted = 0u32; + let mut dirs = BTreeSet::new(); + for entry in entries { + let key = normalized_key(&entry.name); + if keep.contains(&key) { + continue; + } + let Some(parts) = sanitized_components(&entry.name) else { + cleanup_log(&format!( + "cleanup: refusing unsafe manifest path '{}'", + entry.name + )); + continue; + }; + if has_symlinked_ancestor(install_root, &parts) { + cleanup_log(&format!( + "cleanup: '{}' is behind a symlinked directory, skipping", + entry.name + )); + continue; + } + let mut path = install_root.to_path_buf(); + path.extend(&parts); + if entry.is_dir { + dirs.insert(path); + continue; + } + if delete_stale_file(&path) { + cleanup_log(&format!( + "cleanup: deleted '{}' (depot {depot_id}, gone after manifest {gid})", + path.display() + )); + deleted += 1; + // Steamless backs up patched exes as ".original.exe"; + // restoreOriginalExecutable would resurrect a deleted exe from an + // orphaned backup, so the backup goes with its primary — but only + // an exe's backup, and never one a current manifest legitimately + // ships. + if key.ends_with(".exe") && !keep.contains(&format!("{key}.original.exe")) { + let backup = sibling_original_backup(&path); + if delete_stale_file(&backup) { + cleanup_log(&format!("cleanup: deleted backup '{}'", backup.display())); + deleted += 1; + } + } + if let Some(parent) = path.parent() { + if parent != install_root { + dirs.insert(parent.to_path_buf()); + } + } + } + } + for dir in dirs.iter().rev() { + prune_empty_dirs_up(install_root, dir, keep); + } + deleted +} + +/// Every (depot_id, gid) with a cached filelist sidecar or manifest — the set +/// of builds whose file lists this pass can diff against the current install. +fn collect_cached_builds(config_dir: &Path) -> BTreeSet<(u32, u64)> { + let mut builds = BTreeSet::new(); + let Ok(entries) = fs::read_dir(config_dir) else { + return builds; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let stem = match name.strip_suffix(FILELIST_SUFFIX) { + Some(stem) => stem, + None => match name.strip_suffix(".manifest") { + Some(stem) => stem, + None => continue, + }, + }; + let mut parts = stem.split('_'); + if let (Some(depot), Some(gid), None) = (parts.next(), parts.next(), parts.next()) { + if let (Ok(depot_id), Ok(manifest_id)) = (depot.parse::(), gid.parse::()) { + builds.insert((depot_id, manifest_id)); + } + } + } + builds +} + +fn load_manifest( + config_dir: &Path, + depot_id: u32, + manifest_id: u64, + depot_key: &[u8], +) -> Option { + let path = config_dir.join(format!("{depot_id}_{manifest_id}.manifest")); + let raw = fs::read(path).ok()?; + if raw.is_empty() { + return None; + } + let mut manifest = ContentManifest::parse(&raw)?; + manifest.decrypt_filenames(depot_key).then_some(manifest) +} + +/// Sidecar first (key-independent), then the cached manifest when this +/// operation holds the depot key. +fn load_file_entries( + config_dir: &Path, + depot_id: u32, + manifest_id: u64, + depot_key: Option<&[u8]>, +) -> Option> { + if let Some(entries) = read_filelist_sidecar(config_dir, depot_id, manifest_id) { + return Some(entries); + } + let manifest = load_manifest(config_dir, depot_id, manifest_id, depot_key?)?; + Some( + manifest + .files + .iter() + .map(|file| FileEntry { + name: file.filename.clone(), + is_dir: (file.flags & DEPOT_FILE_FLAG_DIRECTORY) != 0, + }) + .collect(), + ) +} + +/// Canonical comparison key: separators are already '/' after +/// decrypt_filenames; "."/empty components are dropped (the writer accepts +/// "./a" and "a//b" spellings) and case is folded so both sides of the +/// old-minus-current diff normalize identically. +fn normalized_key(rel: &str) -> String { + let mut key = String::with_capacity(rel.len()); + for part in rel.split('/') { + if part.is_empty() || part == "." { + continue; + } + if !key.is_empty() { + key.push('/'); + } + key.push_str(&part.to_ascii_lowercase()); + } + key +} + +/// Path components safe to delete under the install dir: plain relative +/// paths only; "."/empty components are dropped to mirror normalized_key. +/// Stricter than depot_writer::path_is_safe, which permits ':' — a file it +/// would write is then never reclaimed here. Unreachable with real Steam +/// manifests, and erring toward not deleting is the right way to be wrong. +fn sanitized_components(rel: &str) -> Option> { + if rel.starts_with('/') || rel.contains(':') { + return None; + } + let mut parts = Vec::new(); + for part in rel.split('/') { + if part.is_empty() || part == "." { + continue; + } + if part == ".." || part.bytes().any(|b| b.is_ascii_control()) { + return None; + } + parts.push(part); + } + if parts.is_empty() || parts[0].eq_ignore_ascii_case(".DepotDownloader") { + return None; + } + Some(parts) +} + +fn delete_stale_file(path: &Path) -> bool { + let Ok(meta) = fs::symlink_metadata(path) else { + return false; + }; + if meta.is_dir() { + // Manifest called it a file but the disk has a directory — leave it. + return false; + } + fs::remove_file(path).is_ok() +} + +fn sibling_original_backup(path: &Path) -> PathBuf { + let mut name = path.as_os_str().to_os_string(); + name.push(".original.exe"); + PathBuf::from(name) +} + +/// Manifest-created symlinks may target arbitrary paths; deleting through one +/// would escape the install dir, so candidates behind a symlinked directory +/// are left alone. +fn has_symlinked_ancestor(install_root: &Path, parts: &[&str]) -> bool { + let mut current = install_root.to_path_buf(); + for part in &parts[..parts.len().saturating_sub(1)] { + current.push(part); + let is_symlink = fs::symlink_metadata(¤t) + .map(|meta| meta.file_type().is_symlink()) + .unwrap_or(false); + if is_symlink { + return true; + } + } + false +} + +/// [keep] guards directories a current manifest ships empty (Mods/, Saves/): +/// games that assume they exist break if the old build's files leave them bare. +fn prune_empty_dirs_up(install_root: &Path, start: &Path, keep: &BTreeSet) { + let mut current = start; + while current != install_root && current.starts_with(install_root) { + let rel = current.strip_prefix(install_root).ok().and_then(|p| p.to_str()); + if rel.is_some_and(|r| keep.contains(&normalized_key(r))) { + break; + } + if fs::remove_dir(current).is_err() { + break; + } + let Some(parent) = current.parent() else { + break; + }; + current = parent; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::content_manifest::{END_OF_MANIFEST_MAGIC, METADATA_MAGIC, PAYLOAD_MAGIC}; + use crate::depot_writer::DEPOT_FILE_FLAG_EXECUTABLE; + use crate::proto_wire::Writer; + use std::time::{SystemTime, UNIX_EPOCH}; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "wnsteam_cleanup_{name}_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + fn raw_manifest(depot_id: u32, manifest_id: u64, files: &[(&str, u32)]) -> Vec { + let mut payload = Vec::new(); + for (filename, flags) in files { + let mut file_body = Vec::new(); + { + let mut writer = Writer::new(&mut file_body); + writer.string_field(1, filename); + writer.uint64_field(2, 1); + writer.uint32_field(3, *flags); + } + Writer::new(&mut payload).submessage_field(1, &file_body); + } + + let mut metadata = Vec::new(); + { + let mut writer = Writer::new(&mut metadata); + writer.uint32_field(1, depot_id); + writer.uint64_field(2, manifest_id); + writer.bool_field_force(4, false); + } + + let mut raw = Vec::new(); + push_section(&mut raw, PAYLOAD_MAGIC, &payload); + push_section(&mut raw, METADATA_MAGIC, &metadata); + raw.extend_from_slice(&END_OF_MANIFEST_MAGIC.to_le_bytes()); + raw + } + + fn push_section(out: &mut Vec, magic: u32, body: &[u8]) { + out.extend_from_slice(&magic.to_le_bytes()); + out.extend_from_slice(&(body.len() as u32).to_le_bytes()); + out.extend_from_slice(body); + } + + fn write_manifest(config_dir: &Path, depot_id: u32, manifest_id: u64, files: &[(&str, u32)]) { + fs::write( + config_dir.join(format!("{depot_id}_{manifest_id}.manifest")), + raw_manifest(depot_id, manifest_id, files), + ) + .unwrap(); + } + + fn write_sidecar(config_dir: &Path, depot_id: u32, manifest_id: u64, files: &[(&str, u32)]) { + let manifest = ContentManifest::parse(&raw_manifest(depot_id, manifest_id, files)).unwrap(); + assert!(write_filelist_sidecar( + config_dir, + depot_id, + manifest_id, + &manifest + )); + } + + fn touch(install: &Path, rel: &str) { + let path = install.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, b"x").unwrap(); + } + + fn spec(depot_id: u32, manifest_id: u64) -> ResolvedDepotSpec { + ResolvedDepotSpec { + depot_id, + manifest_id, + depot_key: vec![1u8; 32], + manifest_request_code: 0, + } + } + + fn config_dir(install: &Path) -> PathBuf { + let dir = install.join(".DepotDownloader"); + fs::create_dir_all(&dir).unwrap(); + dir + } + + fn install_current(config_dir: &Path, depot_id: u32, manifest_id: u64) { + let mut cfg = DepotConfigStore::load(config_dir); + cfg.finish_depot(depot_id, manifest_id); + } + + #[test] + fn marker_recording_skips_noop_transitions() { + let dir = temp_dir("marker_noop"); + assert!(!record_pending_cleanup(&dir, 100, 0, 555)); + assert!(!record_pending_cleanup(&dir, 100, INVALID_MANIFEST_ID, 555)); + assert!(!record_pending_cleanup(&dir, 100, 555, 555)); + assert!(pending_cleanup_markers(&dir).is_empty()); + + assert!(record_pending_cleanup(&dir, 100, 444, 555)); + assert_eq!(pending_cleanup_markers(&dir), vec![(100, 444)]); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn filelist_sidecar_roundtrips_files_and_dirs() { + let dir = temp_dir("sidecar_roundtrip"); + write_sidecar( + &dir, + 100, + 555, + &[("bin", DEPOT_FILE_FLAG_DIRECTORY), ("bin/game.exe", 0)], + ); + let entries = read_filelist_sidecar(&dir, 100, 555).unwrap(); + assert_eq!( + entries, + vec![ + FileEntry { + name: "bin".into(), + is_dir: true + }, + FileEntry { + name: "bin/game.exe".into(), + is_dir: false + }, + ] + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn deletes_files_dropped_by_new_manifest_and_prunes_dirs() { + let install = temp_dir("manifest_switch"); + let config = config_dir(&install); + write_manifest( + &config, + 100, + 444, + &[ + ("bin", DEPOT_FILE_FLAG_DIRECTORY), + ("bin/old", DEPOT_FILE_FLAG_DIRECTORY), + ("bin/old/legacy.dll", 0), + ("bin/game.exe", DEPOT_FILE_FLAG_EXECUTABLE), + ("data.pak", 0), + ], + ); + write_manifest( + &config, + 100, + 555, + &[ + ("bin", DEPOT_FILE_FLAG_DIRECTORY), + ("bin/game.exe", DEPOT_FILE_FLAG_EXECUTABLE), + ("data.pak", 0), + ], + ); + install_current(&config, 100, 555); + touch(&install, "bin/old/legacy.dll"); + touch(&install, "bin/game.exe"); + touch(&install, "data.pak"); + touch(&install, "steam_settings/configs.app.ini"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!(!install.join("bin/old/legacy.dll").exists()); + assert!(!install.join("bin/old").exists()); + assert!(install.join("bin/game.exe").exists()); + assert!(install.join("data.pak").exists()); + assert!(install.join("steam_settings/configs.app.ini").exists()); + assert!(install.exists()); + assert!(pending_cleanup_markers(&config).is_empty()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn narrowed_update_cleans_via_sidecars_without_other_depots_keys() { + // A narrowed update op carries only the changed depot; the other + // installed depot is represented by its sidecar alone. + let install = temp_dir("narrowed_update"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("shared.dat", 0), ("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + write_sidecar(&config, 200, 777, &[("Shared.dat", 0)]); + install_current(&config, 100, 555); + install_current(&config, 200, 777); + touch(&install, "shared.dat"); + touch(&install, "only_old.dat"); + touch(&install, "core.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!( + install.join("shared.dat").exists(), + "depot 200 still ships it" + ); + assert!(!install.join("only_old.dat").exists()); + assert!(install.join("core.dat").exists()); + assert!(pending_cleanup_markers(&config).is_empty()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn unreadable_old_list_with_data_on_disk_defers_marker() { + let install = temp_dir("defer_unreadable_old"); + let config = config_dir(&install); + write_manifest(&config, 100, 444, &[("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + install_current(&config, 100, 555); + touch(&install, "only_old.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + // Old manifest exists but this op has no key for depot 100 (and no + // old sidecar) → defer, keep the marker for an op that has the key. + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(200, 777)]); + assert_eq!(deleted, 0); + assert!(install.join("only_old.dat").exists()); + assert_eq!(pending_cleanup_markers(&config), vec![(100, 444)]); + + // Once the data is gone entirely the marker can never act → dropped. + fs::remove_file(config.join("100_444.manifest")).unwrap(); + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(200, 777)]); + assert_eq!(deleted, 0); + assert!(pending_cleanup_markers(&config).is_empty()); + assert!(install.join("only_old.dat").exists()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn unreadable_installed_depot_defers_whole_pass() { + // An installed depot whose file list can't be read (no sidecar, no key + // in this op) leaves the keep-union incomplete, so the pass defers + // rather than risk deleting a live cross-depot file. + let install = temp_dir("unreadable_defers"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + install_current(&config, 100, 555); + install_current(&config, 200, 777); // no sidecar, no key in op + touch(&install, "only_old.dat"); + touch(&install, "core.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0); + assert!(install.join("only_old.dat").exists()); + assert!(install.join("core.dat").exists()); + assert_eq!(pending_cleanup_markers(&config), vec![(100, 444)]); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn unreadable_depot_config_defers_instead_of_deleting_everything() { + // DepotConfigStore::load yields an EMPTY store on a corrupt/unreadable + // depot.config, which must not read as "nothing is installed" — an empty + // keep-union would condemn every cached build, the fresh one included. + let install = temp_dir("unreadable_config_defer"); + let config = config_dir(&install); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + write_sidecar(&config, 100, 444, &[("only_old.dat", 0)]); + touch(&install, "core.dat"); + touch(&install, "only_old.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + // Corrupt depot.config: parses to an empty store. + fs::write(config.join("depot.config"), b"{ not json").unwrap(); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0, "must defer, not sweep on an unreadable config"); + assert!(install.join("core.dat").exists(), "fresh build must survive"); + assert!(install.join("only_old.dat").exists()); + assert_eq!(pending_cleanup_markers(&config), vec![(100, 444)]); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn depot_absent_from_config_defers_its_builds() { + // A fresh run forgets its depots up front, so files can sit on disk with + // no depot.config entry. Absent means UNKNOWN, not dead. + let install = temp_dir("absent_depot_defer"); + let config = config_dir(&install); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + // Depot 200 has files + a sidecar but was forgotten from depot.config. + write_sidecar(&config, 200, 777, &[("dlc.dat", 0)]); + install_current(&config, 100, 555); + touch(&install, "core.dat"); + touch(&install, "dlc.dat"); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0, "forgotten depot's files are unknown, not stale"); + assert!(install.join("dlc.dat").exists(), "must not wipe a forgotten depot"); + assert!(install.join("core.dat").exists()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn keeps_empty_directory_the_current_manifest_ships() { + // A manifest that ships an empty Mods/ must keep it even when the old + // build's files were the only things in it. + let install = temp_dir("keep_shipped_empty_dir"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("Mods/old_mod.dat", 0)]); + write_sidecar( + &config, + 100, + 555, + &[("core.dat", 0), ("Mods", DEPOT_FILE_FLAG_DIRECTORY)], + ); + install_current(&config, 100, 555); + touch(&install, "core.dat"); + touch(&install, "Mods/old_mod.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!(!install.join("Mods/old_mod.dat").exists()); + assert!(install.join("Mods").is_dir(), "shipped empty dir must survive"); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn truncated_file_list_defers_instead_of_under_populating_keep() { + let install = temp_dir("truncated_sidecar_defer"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0), ("extra.dat", 0)]); + install_current(&config, 100, 555); + touch(&install, "only_old.dat"); + touch(&install, "core.dat"); + touch(&install, "extra.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + // Lop the last entry off the INSTALLED build's sidecar. + let path = filelist_sidecar_path(&config, 100, 555); + let full = fs::read_to_string(&path).unwrap(); + fs::write(&path, full.rsplit_once("extra.dat").unwrap().0).unwrap(); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0, "an incomplete keep-union must defer the pass"); + assert!(install.join("only_old.dat").exists()); + assert!(install.join("extra.dat").exists()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn in_progress_depot_defers_whole_pass() { + // An in-flight (INVALID) depot's target build is unknown to cleanup, so + // a file moving into it can't be proven safe — the whole pass defers + // until that depot finishes. + let install = temp_dir("in_progress_defer"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + write_sidecar(&config, 200, 777, &[("dlc.dat", 0)]); + install_current(&config, 100, 555); + let mut cfg = DepotConfigStore::load(&config); + cfg.begin_depot(200); // 200 -> INVALID (mid-download) + touch(&install, "only_old.dat"); + touch(&install, "core.dat"); + touch(&install, "dlc.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0, "deferred while depot 200 is in flight"); + assert!(install.join("only_old.dat").exists()); + assert!(install.join("dlc.dat").exists()); + assert_eq!(pending_cleanup_markers(&config), vec![(100, 444)]); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn cross_depot_file_moving_into_in_flight_depot_survives() { + // Regression: shared.dat was in depot 100's old build 444 and is not in + // its current build 555, but the in-flight depot 200 ships it. Cleanup + // can't see 200's target build, so it must not delete shared.dat. + let install = temp_dir("cross_depot_in_flight"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("shared.dat", 0), ("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + install_current(&config, 100, 555); + let mut cfg = DepotConfigStore::load(&config); + cfg.begin_depot(200); // 200 mid-download (will ship shared.dat), unreadable + touch(&install, "shared.dat"); + touch(&install, "core.dat"); + touch(&install, "only_old.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0); + assert!( + install.join("shared.dat").exists(), + "live file shipped by the in-flight depot must survive" + ); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn sweeps_cached_old_build_without_a_marker() { + // The defining B behavior: an aborted build's files are reclaimed from + // its cached sidecar even when no .stalecleanup marker exists. + let install = temp_dir("sweep_no_marker"); + let config = config_dir(&install); + write_sidecar(&config, 100, 555, &[("core.dat", 0)]); + write_sidecar(&config, 100, 777, &[("core.dat", 0), ("dropped_in_patch.dll", 0)]); + install_current(&config, 100, 555); + touch(&install, "core.dat"); + touch(&install, "dropped_in_patch.dll"); // left by an aborted update to 777 + touch(&install, "user_mod.cfg"); // user-added, in no manifest + // No marker recorded for 777. + assert!(pending_cleanup_markers(&config).is_empty()); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!( + !install.join("dropped_in_patch.dll").exists(), + "aborted build's file swept" + ); + assert!(install.join("core.dat").exists()); + assert!( + install.join("user_mod.cfg").exists(), + "user-added mod preserved" + ); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn normalization_matches_dotted_and_doubled_separator_spellings() { + assert_eq!(normalized_key("./Bin//Game.EXE"), "bin/game.exe"); + assert_eq!(normalized_key("bin/game.exe"), "bin/game.exe"); + + // Old spelled plainly, new spelled with "./" — still the same file. + let install = temp_dir("normalized_keep"); + let config = config_dir(&install); + write_sidecar(&config, 100, 444, &[("bin/x.dll", 0), ("only_old.dat", 0)]); + write_sidecar(&config, 100, 555, &[("./bin//x.dll", 0)]); + install_current(&config, 100, 555); + touch(&install, "bin/x.dll"); + touch(&install, "only_old.dat"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + assert_eq!(deleted, 1); + assert!(install.join("bin/x.dll").exists()); + assert!(!install.join("only_old.dat").exists()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn rejects_unsafe_manifest_paths() { + assert_eq!( + sanitized_components("bin/game.exe"), + Some(vec!["bin", "game.exe"]) + ); + assert_eq!( + sanitized_components("./bin//game.exe"), + Some(vec!["bin", "game.exe"]) + ); + assert_eq!(sanitized_components(""), None); + assert_eq!(sanitized_components("."), None); + assert_eq!(sanitized_components("/etc/passwd"), None); + assert_eq!(sanitized_components("../outside.dat"), None); + assert_eq!(sanitized_components("bin/../../outside.dat"), None); + assert_eq!(sanitized_components("c:/windows/system32"), None); + assert_eq!(sanitized_components(".DepotDownloader/depot.config"), None); + assert_eq!(sanitized_components(".depotdownloader/depot.config"), None); + assert_eq!(sanitized_components("bad\nname"), None); + } + + #[test] + fn deletes_steamless_backup_only_for_unkept_exe_primaries() { + let install = temp_dir("steamless_backup"); + let config = config_dir(&install); + write_manifest( + &config, + 100, + 444, + &[("old.exe", 0), ("game.exe", 0), ("data.pak", 0)], + ); + write_manifest(&config, 100, 555, &[("game.exe", 0)]); + install_current(&config, 100, 555); + touch(&install, "old.exe"); + touch(&install, "old.exe.original.exe"); + touch(&install, "game.exe"); + touch(&install, "game.exe.original.exe"); + touch(&install, "data.pak"); + touch(&install, "data.pak.original.exe"); // not an exe primary + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 3); // old.exe + its backup + data.pak + assert!(!install.join("old.exe").exists()); + assert!(!install.join("old.exe.original.exe").exists()); + assert!(install.join("game.exe").exists()); + assert!(install.join("game.exe.original.exe").exists()); + assert!(!install.join("data.pak").exists()); + assert!( + install.join("data.pak.original.exe").exists(), + "backup deletion is exe-only" + ); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn keeps_backup_shipped_by_current_manifest() { + let install = temp_dir("kept_backup"); + let config = config_dir(&install); + write_manifest(&config, 100, 444, &[("tool.exe", 0)]); + write_manifest( + &config, + 100, + 555, + &[("tool.exe.original.exe", 0), ("core.dat", 0)], + ); + install_current(&config, 100, 555); + touch(&install, "tool.exe"); + touch(&install, "tool.exe.original.exe"); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!(!install.join("tool.exe").exists()); + assert!( + install.join("tool.exe.original.exe").exists(), + "current manifest ships this exact name" + ); + let _ = fs::remove_dir_all(&install); + } + + #[cfg(unix)] + #[test] + fn skips_candidates_behind_symlinked_directories() { + let install = temp_dir("symlink_ancestor"); + let config = config_dir(&install); + let outside = temp_dir("symlink_target"); + fs::write(outside.join("precious.dat"), b"keep").unwrap(); + std::os::unix::fs::symlink(&outside, install.join("link")).unwrap(); + + write_manifest(&config, 100, 444, &[("link/precious.dat", 0)]); + write_manifest(&config, 100, 555, &[("core.dat", 0)]); + install_current(&config, 100, 555); + assert!(record_pending_cleanup(&config, 100, 444, 555)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0); + assert!(outside.join("precious.dat").exists()); + assert!(install.join("link").exists()); + let _ = fs::remove_dir_all(&install); + let _ = fs::remove_dir_all(&outside); + } + + #[test] + fn aborted_build_marker_reclaims_partial_files_after_revert() { + let install = temp_dir("aborted_revert"); + let config = config_dir(&install); + // Build 555 committed; the update to 777 was cancelled mid-write + // after the sidecar was written and one B-only file landed on disk. + write_sidecar(&config, 100, 555, &[("game.exe", 0), ("data.pak", 0)]); + write_sidecar(&config, 100, 777, &[("game.exe", 0), ("dropped_in_patch.dll", 0)]); + install_current(&config, 100, 555); + touch(&install, "game.exe"); + touch(&install, "data.pak"); + touch(&install, "dropped_in_patch.dll"); + assert!(record_aborted_build(&config, 100, 777)); + assert!(!record_aborted_build(&config, 100, 0)); + assert!(!record_aborted_build(&config, 100, INVALID_MANIFEST_ID)); + + // A later verify of 555 completed → cleanup runs. + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 1); + assert!(!install.join("dropped_in_patch.dll").exists()); + assert!(install.join("game.exe").exists()); + assert!(install.join("data.pak").exists()); + assert!(pending_cleanup_markers(&config).is_empty()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn aborted_build_marker_dropped_when_build_later_commits() { + let install = temp_dir("aborted_resumed"); + let config = config_dir(&install); + write_sidecar(&config, 100, 777, &[("game.exe", 0), ("dropped_in_patch.dll", 0)]); + install_current(&config, 100, 777); // resume finished the update + touch(&install, "game.exe"); + touch(&install, "dropped_in_patch.dll"); + assert!(record_aborted_build(&config, 100, 777)); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 777)]); + + assert_eq!(deleted, 0); + assert!(install.join("dropped_in_patch.dll").exists()); + assert!(pending_cleanup_markers(&config).is_empty()); + let _ = fs::remove_dir_all(&install); + } + + #[test] + fn marker_for_current_manifest_is_dropped_without_deletions() { + let install = temp_dir("marker_current"); + let config = config_dir(&install); + write_manifest(&config, 100, 555, &[("core.dat", 0)]); + install_current(&config, 100, 555); + touch(&install, "core.dat"); + fs::write(stale_cleanup_marker_path(&config, 100, 555), "555").unwrap(); + + let deleted = run_stale_file_cleanup(install.to_str().unwrap(), &config, &[spec(100, 555)]); + + assert_eq!(deleted, 0); + assert!(install.join("core.dat").exists()); + assert!(pending_cleanup_markers(&config).is_empty()); + let _ = fs::remove_dir_all(&install); + } +} diff --git a/app/src/main/cpp/wn-steam-client/rust/src/depot_config.rs b/app/src/main/cpp/wn-steam-client/rust/src/depot_config.rs index 1968fb69e..b048aa5fc 100644 --- a/app/src/main/cpp/wn-steam-client/rust/src/depot_config.rs +++ b/app/src/main/cpp/wn-steam-client/rust/src/depot_config.rs @@ -53,6 +53,13 @@ impl DepotConfigStore { self.installed.get(&depot_id).copied().unwrap_or(0) } + pub fn installed_entries(&self) -> Vec<(u32, u64)> { + self.installed + .iter() + .map(|(depot, manifest)| (*depot, *manifest)) + .collect() + } + pub fn is_installed(&self, depot_id: u32, manifest_id: u64) -> bool { self.installed .get(&depot_id) @@ -206,7 +213,7 @@ fn get_u32(buf: &[u8]) -> Option { Some(u32::from_le_bytes(buf.get(..4)?.try_into().ok()?)) } -fn atomic_write_synced(final_path: &Path, bytes: &[u8]) -> bool { +pub(crate) fn atomic_write_synced(final_path: &Path, bytes: &[u8]) -> bool { let Some(parent) = final_path.parent() else { return false; }; @@ -262,6 +269,22 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn installed_entries_lists_every_depot() { + let dir = temp_dir("installed_entries"); + let mut store = DepotConfigStore::load(&dir); + assert!(store.finish_depot(200, 777)); + assert!(store.finish_depot(100, 555)); + + assert_eq!(store.installed_entries(), vec![(100, 555), (200, 777)]); + assert!(store.forget_depot(200)); + assert_eq!( + DepotConfigStore::load(&dir).installed_entries(), + vec![(100, 555)] + ); + let _ = fs::remove_dir_all(&dir); + } + #[test] fn progress_sidecar_roundtrips_sorted_indices() { let dir = temp_dir("progress_sidecar_roundtrips"); diff --git a/app/src/main/cpp/wn-steam-client/rust/src/depot_downloader.rs b/app/src/main/cpp/wn-steam-client/rust/src/depot_downloader.rs index 3bd132a0c..464d1f971 100644 --- a/app/src/main/cpp/wn-steam-client/rust/src/depot_downloader.rs +++ b/app/src/main/cpp/wn-steam-client/rust/src/depot_downloader.rs @@ -1,5 +1,9 @@ use crate::cdn_client::{CdnClient, CdnManifestResult}; use crate::content_manifest::ContentManifest; +use crate::depot_cleanup::{ + backfill_filelist_sidecar, record_aborted_build, record_pending_cleanup, run_stale_file_cleanup, + write_filelist_sidecar, +}; use crate::depot_config::{DepotConfigStore, DepotProgressStore, INVALID_MANIFEST_ID}; use crate::depot_writer::{write_depot_sequential, DepotWriteOptions}; use crate::pb::ccontentserverdirectory::CContentServerDirectoryServerInfo; @@ -268,6 +272,7 @@ pub fn download_resolved_depots( None, None, None, + true, ) } @@ -290,6 +295,7 @@ pub fn download_resolved_depots_with_cancel( cancel, None, None, + true, ) } @@ -303,6 +309,10 @@ pub fn download_resolved_depots_with_cancel_progress( cancel: Option<&AtomicBool>, on_progress: Option>, code_refresher: Option>, + // One app download is several native calls (base app, then each DLC app). + // The stale-file pass needs the keep-union to span all of them, so only the + // final batch may sweep. + run_cleanup: bool, ) -> DepotDownloadResult { if let Err(error) = validate_resolved_download_inputs(install_dir, depots, servers) { return error; @@ -324,6 +334,13 @@ pub fn download_resolved_depots_with_cancel_progress( if fresh { // Reset only this batch's depots; a global discard would wipe earlier batches' records. for depot in depots { + // Before forget_depot drops the gid the old files can be diffed against. + record_pending_cleanup( + &config_dir, + depot.depot_id, + cfg.installed_manifest(depot.depot_id), + depot.manifest_id, + ); cfg.forget_depot(depot.depot_id); DepotProgressStore::remove(&config_dir, depot.depot_id, depot.manifest_id); remove_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id); @@ -348,6 +365,9 @@ pub fn download_resolved_depots_with_cancel_progress( let clean_pause = has_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id); if decide_depot_resume(fresh, &cfg, spec, clean_pause) == DepotResumeDecision::SkipInstalled { + // Installs that predate filelist sidecars get one written now so + // the stale-file keep-union can read this depot without its key. + backfill_filelist_sidecar(&config_dir, depot); result.depots_skipped += 1; continue; } @@ -357,6 +377,12 @@ pub fn download_resolved_depots_with_cancel_progress( depot.depot_id )); } + record_pending_cleanup( + &config_dir, + depot.depot_id, + cfg.installed_manifest(depot.depot_id), + depot.manifest_id, + ); if !cfg.begin_depot(depot.depot_id) { return DepotDownloadResult::fail(format!( "download: depot.config begin failed for depot {}", @@ -424,6 +450,9 @@ pub fn download_resolved_depots_with_cancel_progress( depot.depot_id )); } + // Before any game file is touched, so an aborted write still leaves a + // key-independent record of what this build may have put on disk. + let _ = write_filelist_sidecar(&config_dir, depot.depot_id, depot.manifest_id, &manifest); let depot_id = depot.depot_id; let depots_done = depot_index as u32; @@ -459,6 +488,10 @@ pub fn download_resolved_depots_with_cancel_progress( if write_result.resume_trust_safe { let _ = write_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id); } + // The write may have left this build's files on disk without ever + // committing a gid to diff them against — mark the target so the + // next successful download reclaims its orphans. + record_aborted_build(&config_dir, depot.depot_id, depot.manifest_id); return DepotDownloadResult::fail(format!( "download: depot {} write failed: {}", depot.depot_id, write_result.error @@ -466,6 +499,7 @@ pub fn download_resolved_depots_with_cancel_progress( } if !cfg.finish_depot(depot.depot_id, depot.manifest_id) { + record_aborted_build(&config_dir, depot.depot_id, depot.manifest_id); return DepotDownloadResult::fail(format!( "download: depot.config finish failed for depot {}", depot.depot_id @@ -477,6 +511,11 @@ pub fn download_resolved_depots_with_cancel_progress( result.depots_completed += 1; } + // Every failure path above returns early, so success is true here today; + // the check is kept so a future non-returning failure can't sweep. + if result.success && run_cleanup { + run_stale_file_cleanup(install_dir, &config_dir, depots); + } result } @@ -692,6 +731,209 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn deletes_files_absent_from_the_new_manifest() { + let dir = temp_dir("stale_files_swept"); + let config_dir = dir.join(".DepotDownloader"); + fs::create_dir_all(&config_dir).unwrap(); + let server = [CContentServerDirectoryServerInfo { + host: "cdn.example".into(), + https_support: "mandatory".into(), + ..Default::default() + }]; + let spec = |manifest_id| ResolvedDepotSpec { + depot_id: 100, + manifest_id, + depot_key: vec![1u8; 32], + manifest_request_code: 0, + }; + + fs::write( + config_dir.join("100_555.manifest"), + raw_layout_manifest(100, 555, "dropped_in_patch.bin", 5), + ) + .unwrap(); + let result = + download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", false, 4); + assert!(result.success, "{}", result.error); + assert!(dir.join("dropped_in_patch.bin").exists()); + + // Files outside any manifest must survive the update untouched. + fs::write(dir.join("user_notes.txt"), b"keep me").unwrap(); + + fs::write( + config_dir.join("100_777.manifest"), + raw_layout_manifest(100, 777, "patched.bin", 5), + ) + .unwrap(); + let result = + download_resolved_depots(dir.to_str().unwrap(), &[spec(777)], &server, "", false, 4); + assert!(result.success, "{}", result.error); + + assert!(dir.join("patched.bin").exists()); + assert!( + !dir.join("dropped_in_patch.bin").exists(), + "file dropped by the new manifest must be deleted" + ); + assert!(dir.join("user_notes.txt").exists()); + assert!(crate::depot_cleanup::pending_cleanup_markers(&config_dir).is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn failed_write_records_aborted_build_and_next_download_reclaims_it() { + let dir = temp_dir("aborted_write_recovery"); + let config_dir = dir.join(".DepotDownloader"); + fs::create_dir_all(&config_dir).unwrap(); + let server = [CContentServerDirectoryServerInfo { + host: "cdn.example".into(), + https_support: "mandatory".into(), + ..Default::default() + }]; + let spec = |manifest_id| ResolvedDepotSpec { + depot_id: 100, + manifest_id, + depot_key: vec![1u8; 32], + manifest_request_code: 0, + }; + + // Build 555 installs cleanly (chunkless layout manifest). + fs::write( + config_dir.join("100_555.manifest"), + raw_layout_manifest(100, 555, "game.bin", 5), + ) + .unwrap(); + let result = + download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", false, 4); + assert!(result.success, "{}", result.error); + + // Build 777 needs a real chunk from the unreachable CDN → the write + // fails mid-update, after the sidecar was persisted. + fs::write( + config_dir.join("100_777.manifest"), + raw_chunked_manifest(100, 777, "only_in_777.bin"), + ) + .unwrap(); + let result = + download_resolved_depots(dir.to_str().unwrap(), &[spec(777)], &server, "", false, 4); + assert!(!result.success); + assert!(config_dir.join("100_777.filelist").is_file()); + // Two pending markers: 555 for the 555→777 transition (in case 777 + // committed), 777 recording the aborted target itself. + assert_eq!( + crate::depot_cleanup::pending_cleanup_markers(&config_dir), + vec![(100, 555), (100, 777)] + ); + + // Simulate the partial write the aborted update left behind, then a + // fresh re-download of 555 completing successfully. + fs::write(dir.join("only_in_777.bin"), b"part").unwrap(); + let result = + download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", true, 4); + assert!(result.success, "{}", result.error); + + assert!(dir.join("game.bin").exists()); + assert!( + !dir.join("only_in_777.bin").exists(), + "aborted build's orphan must be reclaimed" + ); + assert!(crate::depot_cleanup::pending_cleanup_markers(&config_dir).is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn per_batch_fresh_download_writes_a_sidecar_for_each_batch() { + // Verify Files runs one fresh native call per app batch (base, then + // each DLC); each must leave its own sidecar and keep the other's + // depot.config entry, or the stale-file keep-union goes blind. + let dir = temp_dir("fresh_batches_keep_config"); + let config_dir = dir.join(".DepotDownloader"); + fs::create_dir_all(&config_dir).unwrap(); + let server = [CContentServerDirectoryServerInfo { + host: "cdn.example".into(), + https_support: "mandatory".into(), + ..Default::default() + }]; + let spec = |depot_id, manifest_id| ResolvedDepotSpec { + depot_id, + manifest_id, + depot_key: vec![1u8; 32], + manifest_request_code: 0, + }; + fs::write( + config_dir.join("100_555.manifest"), + raw_layout_manifest(100, 555, "base.bin", 5), + ) + .unwrap(); + fs::write( + config_dir.join("200_777.manifest"), + raw_layout_manifest(200, 777, "dlc.bin", 5), + ) + .unwrap(); + + let base = download_resolved_depots( + dir.to_str().unwrap(), + &[spec(100, 555)], + &server, + "", + true, + 4, + ); + assert!(base.success, "{}", base.error); + let dlc = download_resolved_depots( + dir.to_str().unwrap(), + &[spec(200, 777)], + &server, + "", + true, + 4, + ); + assert!(dlc.success, "{}", dlc.error); + + let cfg = DepotConfigStore::load(&config_dir); + assert!(cfg.is_installed(100, 555), "batch 2 must not wipe batch 1"); + assert!(cfg.is_installed(200, 777)); + assert!(config_dir.join("100_555.filelist").is_file()); + assert!(config_dir.join("200_777.filelist").is_file()); + let _ = fs::remove_dir_all(&dir); + } + + fn raw_chunked_manifest(depot_id: u32, manifest_id: u64, filename: &str) -> Vec { + let mut chunk_body = Vec::new(); + { + let mut writer = Writer::new(&mut chunk_body); + writer.bytes_field(1, &[7u8; 20]); + writer.tag(2, crate::proto_wire::WireType::Fixed32); + writer.raw_bytes(&0x1234_5678u32.to_le_bytes()); + writer.uint64_field(3, 0); + writer.uint32_field(4, 4); + writer.uint32_field(5, 4); + } + let mut file_body = Vec::new(); + { + let mut writer = Writer::new(&mut file_body); + writer.string_field(1, filename); + writer.uint64_field(2, 4); + writer.submessage_field(6, &chunk_body); + } + let mut payload = Vec::new(); + Writer::new(&mut payload).submessage_field(1, &file_body); + + let mut metadata = Vec::new(); + { + let mut writer = Writer::new(&mut metadata); + writer.uint32_field(1, depot_id); + writer.uint64_field(2, manifest_id); + writer.bool_field_force(4, false); + } + + let mut raw = Vec::new(); + push_section(&mut raw, PAYLOAD_MAGIC, &payload); + push_section(&mut raw, METADATA_MAGIC, &metadata); + raw.extend_from_slice(&END_OF_MANIFEST_MAGIC.to_le_bytes()); + raw + } + fn temp_dir(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!( "wnsteam_downloader_{name}_{}", diff --git a/app/src/main/cpp/wn-steam-client/rust/src/jni.rs b/app/src/main/cpp/wn-steam-client/rust/src/jni.rs index e2e470140..fcc8b4cd4 100644 --- a/app/src/main/cpp/wn-steam-client/rust/src/jni.rs +++ b/app/src/main/cpp/wn-steam-client/rust/src/jni.rs @@ -37,7 +37,7 @@ unsafe extern "C" { fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32; } -fn android_log(tag: &str, message: &str) { +pub(crate) fn android_log(tag: &str, message: &str) { #[cfg(target_os = "android")] { let Ok(tag) = CString::new(tag) else { @@ -3263,6 +3263,7 @@ pub extern "system" fn Java_com_winlator_cmod_feature_stores_steam_wnsteam_WnSte fresh: jboolean, ca_bundle_path: JString, max_workers: jint, + run_cleanup: jboolean, listener: JObject, ) { let Some(handle) = (unsafe { from_session_handle_mut(handle) }) else { @@ -3463,6 +3464,7 @@ pub extern "system" fn Java_com_winlator_cmod_feature_stores_steam_wnsteam_WnSte Some(download_cancel.as_ref()), Some(progress_cb), Some(code_refresher_cb), + run_cleanup != 0, ); dispatch_download_complete(listener, result); }); diff --git a/app/src/main/cpp/wn-steam-client/rust/src/lib.rs b/app/src/main/cpp/wn-steam-client/rust/src/lib.rs index 04df26dc0..74f386af5 100644 --- a/app/src/main/cpp/wn-steam-client/rust/src/lib.rs +++ b/app/src/main/cpp/wn-steam-client/rust/src/lib.rs @@ -20,6 +20,7 @@ pub mod cmsg_protobuf_header; pub mod content_manifest; pub mod crypto; pub mod depot_chunk; +pub mod depot_cleanup; pub mod depot_config; pub mod depot_downloader; pub mod depot_writer; diff --git a/app/src/main/feature/stores/steam/service/SteamService.kt b/app/src/main/feature/stores/steam/service/SteamService.kt index 51d9325ac..71f051ad3 100644 --- a/app/src/main/feature/stores/steam/service/SteamService.kt +++ b/app/src/main/feature/stores/steam/service/SteamService.kt @@ -3827,6 +3827,12 @@ class SteamService : Service() { } val installedManifestIds = readInstalledDepotManifestIds(appDirPath) + val hasDepotConfig = + File(File(appDirPath, ".DepotDownloader"), "depot.config").exists() + // Steam denied a key for these, so they never enter depot.config even + // though the download completed — the completeness gate excludes them + // too. Without this they'd report an update forever. + val deniedDepots = readDeniedDepots(appDirPath) val cachedManifestFiles: Set = File(appDirPath, ".DepotDownloader").list()?.toHashSet() ?: emptySet() // Resolve manifests once per depot; the filter below picks which need updating and the size calc reuses these. @@ -3841,7 +3847,16 @@ class SteamService : Service() { val installedManifestId = installedManifestIds[depotId] if (installedManifestId != null) { installedManifestId != manifest.gid + } else if (depotId in deniedDepots) { + false + } else if (hasDepotConfig) { + // depot.config exists but omits this depot, so it isn't cleanly + // installed. Trusting a cached manifest here would report "no + // updates" for a depot that never finished downloading. + true } else { + // Legacy install with no depot.config — the cached manifest is + // the only installed-version signal there is. "${depotId}_${manifest.gid}.manifest" !in cachedManifestFiles } } diff --git a/app/src/main/feature/stores/steam/service/SteamServiceDownloadFinalize.kt b/app/src/main/feature/stores/steam/service/SteamServiceDownloadFinalize.kt index 57437a064..bd4829d86 100644 --- a/app/src/main/feature/stores/steam/service/SteamServiceDownloadFinalize.kt +++ b/app/src/main/feature/stores/steam/service/SteamServiceDownloadFinalize.kt @@ -478,6 +478,8 @@ internal suspend fun SteamService.Companion.completeAppDownload( runCatching { MarkerUtils.removeMarker(appDirPath, Marker.STEAM_DRM_PATCHED) } runCatching { MarkerUtils.removeMarker(appDirPath, Marker.STEAM_DRM_UNPACK_CHECKED) } + pruneStaleDepotManifestCache(appDirPath) + // Same reason as above: a Room exception here used to FAIL a fully-downloaded game with the COMPLETE marker already on disk. val mainAppId = downloadInfo.gameId val service = instance diff --git a/app/src/main/feature/stores/steam/service/SteamServiceDownloadMain.kt b/app/src/main/feature/stores/steam/service/SteamServiceDownloadMain.kt index f2eb5a1ba..958ba7845 100644 --- a/app/src/main/feature/stores/steam/service/SteamServiceDownloadMain.kt +++ b/app/src/main/feature/stores/steam/service/SteamServiceDownloadMain.kt @@ -1070,7 +1070,7 @@ internal fun SteamService.Companion.downloadApp( java.util.concurrent.atomic.AtomicLong(wnDepotBytes.values.sum()) // Throttle DownloadRecord progress persistence. val wnLastPersistMs = java.util.concurrent.atomic.AtomicLong(0L) - for (batch in wnBatches) { + for ((batchIndex, batch) in wnBatches.withIndex()) { val (batchAppId, batchDepotIds, batchManifestIds) = batch kotlinx.coroutines.suspendCancellableCoroutine { cont -> wnSessionForDownload.downloadApp( @@ -1083,6 +1083,9 @@ internal fun SteamService.Companion.downloadApp( caPath, // "Download Speed" setting → parallel chunk-download worker count. PrefManager.downloadSpeed, + // Sweep only after the last batch: the keep-union + // has to see every batch's depots. + batchIndex == wnBatches.lastIndex, object : WnDownloadListener { override fun onProgress( depotId: Int, diff --git a/app/src/main/feature/stores/steam/service/SteamServiceLogin.kt b/app/src/main/feature/stores/steam/service/SteamServiceLogin.kt index 3e843984d..611b0ea14 100644 --- a/app/src/main/feature/stores/steam/service/SteamServiceLogin.kt +++ b/app/src/main/feature/stores/steam/service/SteamServiceLogin.kt @@ -582,6 +582,50 @@ internal fun SteamService.Companion.readInstalledDepotManifestIds(appDirPath: St emptyMap() } +/** + * Drops cached "{depotId}_{gid}.manifest" files (and their ".filelist" sidecars) + * for builds depot.config no longer lists. Manifests with a pending + * ".stalecleanup" marker are kept — the native stale-file pass still needs them + * to diff the old build away. Legacy installs with no depot.config are left + * alone, since checkForAppUpdate's cache fallback is their only version signal. + * Depots mid-download are skipped: their entry reads as INVALID, which would + * otherwise look like a mismatch and evict the manifest they are downloading. + */ +internal fun SteamService.Companion.pruneStaleDepotManifestCache(appDirPath: String) { + runCatching { + val installedManifests = readInstalledDepotManifestIds(appDirPath) + if (installedManifests.isEmpty()) return + val inProgressDepots = installedManifests.filterValues { it == Long.MAX_VALUE }.keys + val depotDownloaderDir = File(appDirPath, ".DepotDownloader") + depotDownloaderDir + .listFiles { f -> f.isFile && (f.name.endsWith(".manifest") || f.name.endsWith(".filelist")) } + ?.forEach { f -> + val stem = f.name.substringBeforeLast('.') + val parts = stem.split('_') + if (parts.size != 2) return@forEach + val depotId = parts[0].toIntOrNull() ?: return@forEach + val gid = parts[1].toLongOrNull() ?: return@forEach + if (depotId in inProgressDepots) return@forEach + if (installedManifests[depotId] == gid) return@forEach + if (File(depotDownloaderDir, "$stem.stalecleanup").isFile) return@forEach + if (f.delete()) { + Timber.i("Pruned stale depot manifest cache ${f.name} at $appDirPath") + } + } + // Markers with neither a manifest nor a filelist left can never be acted on. + depotDownloaderDir + .listFiles { f -> f.isFile && f.name.endsWith(".stalecleanup") } + ?.forEach { f -> + val stem = f.name.removeSuffix(".stalecleanup") + val actionable = + File(depotDownloaderDir, "$stem.manifest").isFile || + File(depotDownloaderDir, "$stem.filelist").isFile + if (!actionable && f.delete()) { + Timber.i("Dropped orphaned stale-cleanup marker ${f.name} at $appDirPath") + } + } + }.onFailure { e -> Timber.w(e, "Stale manifest prune failed for $appDirPath") } +} internal fun SteamService.Companion.cleanupCancelledUpdate(appDirPath: String) { MarkerUtils.removeMarker(appDirPath, Marker.DOWNLOAD_IN_PROGRESS_MARKER) diff --git a/app/src/main/feature/stores/steam/wnsteam/WnSteamSession.kt b/app/src/main/feature/stores/steam/wnsteam/WnSteamSession.kt index 150a39112..c52f8dfe6 100644 --- a/app/src/main/feature/stores/steam/wnsteam/WnSteamSession.kt +++ b/app/src/main/feature/stores/steam/wnsteam/WnSteamSession.kt @@ -108,6 +108,9 @@ class WnSteamSession : AutoCloseable { fresh: Boolean, caBundlePath: String, maxWorkers: Int, + // One app download is several native calls (base app, then each DLC app); + // only the last may run the stale-file sweep, whose keep-union spans them all. + runCleanup: Boolean, listener: WnDownloadListener, ) { require(depotIds.size == manifestIds.size) { @@ -119,7 +122,7 @@ class WnSteamSession : AutoCloseable { return } nativeDownloadApp(h, appId, depotIds, manifestIds, branch, installDir, - fresh, caBundlePath, maxWorkers, listener) + fresh, caBundlePath, maxWorkers, runCleanup, listener) } // Abort the current depot download. @@ -712,6 +715,7 @@ class WnSteamSession : AutoCloseable { fresh: Boolean, caBundlePath: String, maxWorkers: Int, + runCleanup: Boolean, listener: WnDownloadListener, ) @JvmStatic private external fun nativeCancelDownload(handle: Long)