Sweep game files dropped from a depot manifest

app/src/main/cpp/wn-steam-client/rust/src/depot_config.rs

@@ -53,6 +53,13 @@ impl DepotConfigStore {
         self.installed.get(&depot_id).copied().unwrap_or(0)
     }
 
+    pub fn installed_entries(&self) -> Vec<(u32, u64)> {
+        self.installed
+            .iter()
+            .map(|(depot, manifest)| (*depot, *manifest))
+            .collect()
+    }
+
     pub fn is_installed(&self, depot_id: u32, manifest_id: u64) -> bool {
         self.installed
             .get(&depot_id)
@@ -206,7 +213,7 @@ fn get_u32(buf: &[u8]) -> Option<u32> {
     Some(u32::from_le_bytes(buf.get(..4)?.try_into().ok()?))
 }
 
-fn atomic_write_synced(final_path: &Path, bytes: &[u8]) -> bool {
+pub(crate) fn atomic_write_synced(final_path: &Path, bytes: &[u8]) -> bool {
     let Some(parent) = final_path.parent() else {
         return false;
     };
@@ -262,6 +269,22 @@ mod tests {
         let _ = fs::remove_dir_all(&dir);
     }
 
+    #[test]
+    fn installed_entries_lists_every_depot() {
+        let dir = temp_dir("installed_entries");
+        let mut store = DepotConfigStore::load(&dir);
+        assert!(store.finish_depot(200, 777));
+        assert!(store.finish_depot(100, 555));
+
+        assert_eq!(store.installed_entries(), vec![(100, 555), (200, 777)]);
+        assert!(store.forget_depot(200));
+        assert_eq!(
+            DepotConfigStore::load(&dir).installed_entries(),
+            vec![(100, 555)]
+        );
+        let _ = fs::remove_dir_all(&dir);
+    }
+
     #[test]
     fn progress_sidecar_roundtrips_sorted_indices() {
         let dir = temp_dir("progress_sidecar_roundtrips");

app/src/main/cpp/wn-steam-client/rust/src/depot_downloader.rs

@@ -1,5 +1,9 @@
 use crate::cdn_client::{CdnClient, CdnManifestResult};
 use crate::content_manifest::ContentManifest;
+use crate::depot_cleanup::{
+    backfill_filelist_sidecar, record_aborted_build, record_pending_cleanup, run_stale_file_cleanup,
+    write_filelist_sidecar,
+};
 use crate::depot_config::{DepotConfigStore, DepotProgressStore, INVALID_MANIFEST_ID};
 use crate::depot_writer::{write_depot_sequential, DepotWriteOptions};
 use crate::pb::ccontentserverdirectory::CContentServerDirectoryServerInfo;
@@ -268,6 +272,7 @@ pub fn download_resolved_depots(
         None,
         None,
         None,
+        true,
     )
 }
 
@@ -290,6 +295,7 @@ pub fn download_resolved_depots_with_cancel(
         cancel,
         None,
         None,
+        true,
     )
 }
 
@@ -303,6 +309,10 @@ pub fn download_resolved_depots_with_cancel_progress(
     cancel: Option<&AtomicBool>,
     on_progress: Option<DepotProgressCallback<'_>>,
     code_refresher: Option<ManifestCodeRefresher<'_>>,
+    // One app download is several native calls (base app, then each DLC app).
+    // The stale-file pass needs the keep-union to span all of them, so only the
+    // final batch may sweep.
+    run_cleanup: bool,
 ) -> DepotDownloadResult {
     if let Err(error) = validate_resolved_download_inputs(install_dir, depots, servers) {
         return error;
@@ -324,6 +334,13 @@ pub fn download_resolved_depots_with_cancel_progress(
     if fresh {
         // Reset only this batch's depots; a global discard would wipe earlier batches' records.
         for depot in depots {
+            // Before forget_depot drops the gid the old files can be diffed against.
+            record_pending_cleanup(
+                &config_dir,
+                depot.depot_id,
+                cfg.installed_manifest(depot.depot_id),
+                depot.manifest_id,
+            );
             cfg.forget_depot(depot.depot_id);
             DepotProgressStore::remove(&config_dir, depot.depot_id, depot.manifest_id);
             remove_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id);
@@ -348,6 +365,9 @@ pub fn download_resolved_depots_with_cancel_progress(
         let clean_pause = has_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id);
         if decide_depot_resume(fresh, &cfg, spec, clean_pause) == DepotResumeDecision::SkipInstalled
         {
+            // Installs that predate filelist sidecars get one written now so
+            // the stale-file keep-union can read this depot without its key.
+            backfill_filelist_sidecar(&config_dir, depot);
             result.depots_skipped += 1;
             continue;
         }
@@ -357,6 +377,12 @@ pub fn download_resolved_depots_with_cancel_progress(
                 depot.depot_id
             ));
         }
+        record_pending_cleanup(
+            &config_dir,
+            depot.depot_id,
+            cfg.installed_manifest(depot.depot_id),
+            depot.manifest_id,
+        );
         if !cfg.begin_depot(depot.depot_id) {
             return DepotDownloadResult::fail(format!(
                 "download: depot.config begin failed for depot {}",
@@ -424,6 +450,9 @@ pub fn download_resolved_depots_with_cancel_progress(
                 depot.depot_id
             ));
         }
+        // Before any game file is touched, so an aborted write still leaves a
+        // key-independent record of what this build may have put on disk.
+        let _ = write_filelist_sidecar(&config_dir, depot.depot_id, depot.manifest_id, &manifest);
 
         let depot_id = depot.depot_id;
         let depots_done = depot_index as u32;
@@ -459,13 +488,18 @@ pub fn download_resolved_depots_with_cancel_progress(
             if write_result.resume_trust_safe {
                 let _ = write_clean_pause_marker(&config_dir, depot.depot_id, depot.manifest_id);
             }
+            // The write may have left this build's files on disk without ever
+            // committing a gid to diff them against — mark the target so the
+            // next successful download reclaims its orphans.
+            record_aborted_build(&config_dir, depot.depot_id, depot.manifest_id);
             return DepotDownloadResult::fail(format!(
                 "download: depot {} write failed: {}",
                 depot.depot_id, write_result.error
             ));
         }
 
         if !cfg.finish_depot(depot.depot_id, depot.manifest_id) {
+            record_aborted_build(&config_dir, depot.depot_id, depot.manifest_id);
             return DepotDownloadResult::fail(format!(
                 "download: depot.config finish failed for depot {}",
                 depot.depot_id
@@ -477,6 +511,11 @@ pub fn download_resolved_depots_with_cancel_progress(
         result.depots_completed += 1;
     }
 
+    // Every failure path above returns early, so success is true here today;
+    // the check is kept so a future non-returning failure can't sweep.
+    if result.success && run_cleanup {
+        run_stale_file_cleanup(install_dir, &config_dir, depots);
+    }
     result
 }
 
@@ -692,6 +731,209 @@ mod tests {
         let _ = fs::remove_dir_all(&dir);
     }
 
+    #[test]
+    fn deletes_files_absent_from_the_new_manifest() {
+        let dir = temp_dir("stale_files_swept");
+        let config_dir = dir.join(".DepotDownloader");
+        fs::create_dir_all(&config_dir).unwrap();
+        let server = [CContentServerDirectoryServerInfo {
+            host: "cdn.example".into(),
+            https_support: "mandatory".into(),
+            ..Default::default()
+        }];
+        let spec = |manifest_id| ResolvedDepotSpec {
+            depot_id: 100,
+            manifest_id,
+            depot_key: vec![1u8; 32],
+            manifest_request_code: 0,
+        };
+
+        fs::write(
+            config_dir.join("100_555.manifest"),
+            raw_layout_manifest(100, 555, "dropped_in_patch.bin", 5),
+        )
+        .unwrap();
+        let result =
+            download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", false, 4);
+        assert!(result.success, "{}", result.error);
+        assert!(dir.join("dropped_in_patch.bin").exists());
+
+        // Files outside any manifest must survive the update untouched.
+        fs::write(dir.join("user_notes.txt"), b"keep me").unwrap();
+
+        fs::write(
+            config_dir.join("100_777.manifest"),
+            raw_layout_manifest(100, 777, "patched.bin", 5),
+        )
+        .unwrap();
+        let result =
+            download_resolved_depots(dir.to_str().unwrap(), &[spec(777)], &server, "", false, 4);
+        assert!(result.success, "{}", result.error);
+
+        assert!(dir.join("patched.bin").exists());
+        assert!(
+            !dir.join("dropped_in_patch.bin").exists(),
+            "file dropped by the new manifest must be deleted"
+        );
+        assert!(dir.join("user_notes.txt").exists());
+        assert!(crate::depot_cleanup::pending_cleanup_markers(&config_dir).is_empty());
+        let _ = fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn failed_write_records_aborted_build_and_next_download_reclaims_it() {
+        let dir = temp_dir("aborted_write_recovery");
+        let config_dir = dir.join(".DepotDownloader");
+        fs::create_dir_all(&config_dir).unwrap();
+        let server = [CContentServerDirectoryServerInfo {
+            host: "cdn.example".into(),
+            https_support: "mandatory".into(),
+            ..Default::default()
+        }];
+        let spec = |manifest_id| ResolvedDepotSpec {
+            depot_id: 100,
+            manifest_id,
+            depot_key: vec![1u8; 32],
+            manifest_request_code: 0,
+        };
+
+        // Build 555 installs cleanly (chunkless layout manifest).
+        fs::write(
+            config_dir.join("100_555.manifest"),
+            raw_layout_manifest(100, 555, "game.bin", 5),
+        )
+        .unwrap();
+        let result =
+            download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", false, 4);
+        assert!(result.success, "{}", result.error);
+
+        // Build 777 needs a real chunk from the unreachable CDN → the write
+        // fails mid-update, after the sidecar was persisted.
+        fs::write(
+            config_dir.join("100_777.manifest"),
+            raw_chunked_manifest(100, 777, "only_in_777.bin"),
+        )
+        .unwrap();
+        let result =
+            download_resolved_depots(dir.to_str().unwrap(), &[spec(777)], &server, "", false, 4);
+        assert!(!result.success);
+        assert!(config_dir.join("100_777.filelist").is_file());
+        // Two pending markers: 555 for the 555→777 transition (in case 777
+        // committed), 777 recording the aborted target itself.
+        assert_eq!(
+            crate::depot_cleanup::pending_cleanup_markers(&config_dir),
+            vec![(100, 555), (100, 777)]
+        );
+
+        // Simulate the partial write the aborted update left behind, then a
+        // fresh re-download of 555 completing successfully.
+        fs::write(dir.join("only_in_777.bin"), b"part").unwrap();
+        let result =
+            download_resolved_depots(dir.to_str().unwrap(), &[spec(555)], &server, "", true, 4);
+        assert!(result.success, "{}", result.error);
+
+        assert!(dir.join("game.bin").exists());
+        assert!(
+            !dir.join("only_in_777.bin").exists(),
+            "aborted build's orphan must be reclaimed"
+        );
+        assert!(crate::depot_cleanup::pending_cleanup_markers(&config_dir).is_empty());
+        let _ = fs::remove_dir_all(&dir);
+    }
+
+    #[test]
+    fn per_batch_fresh_download_writes_a_sidecar_for_each_batch() {
+        // Verify Files runs one fresh native call per app batch (base, then
+        // each DLC); each must leave its own sidecar and keep the other's
+        // depot.config entry, or the stale-file keep-union goes blind.
+        let dir = temp_dir("fresh_batches_keep_config");
+        let config_dir = dir.join(".DepotDownloader");
+        fs::create_dir_all(&config_dir).unwrap();
+        let server = [CContentServerDirectoryServerInfo {
+            host: "cdn.example".into(),
+            https_support: "mandatory".into(),
+            ..Default::default()
+        }];
+        let spec = |depot_id, manifest_id| ResolvedDepotSpec {
+            depot_id,
+            manifest_id,
+            depot_key: vec![1u8; 32],
+            manifest_request_code: 0,
+        };
+        fs::write(
+            config_dir.join("100_555.manifest"),
+            raw_layout_manifest(100, 555, "base.bin", 5),
+        )
+        .unwrap();
+        fs::write(
+            config_dir.join("200_777.manifest"),
+            raw_layout_manifest(200, 777, "dlc.bin", 5),
+        )
+        .unwrap();
+
+        let base = download_resolved_depots(
+            dir.to_str().unwrap(),
+            &[spec(100, 555)],
+            &server,
+            "",
+            true,
+            4,
+        );
+        assert!(base.success, "{}", base.error);
+        let dlc = download_resolved_depots(
+            dir.to_str().unwrap(),
+            &[spec(200, 777)],
+            &server,
+            "",
+            true,
+            4,
+        );
+        assert!(dlc.success, "{}", dlc.error);
+
+        let cfg = DepotConfigStore::load(&config_dir);
+        assert!(cfg.is_installed(100, 555), "batch 2 must not wipe batch 1");
+        assert!(cfg.is_installed(200, 777));
+        assert!(config_dir.join("100_555.filelist").is_file());
+        assert!(config_dir.join("200_777.filelist").is_file());
+        let _ = fs::remove_dir_all(&dir);
+    }
+
+    fn raw_chunked_manifest(depot_id: u32, manifest_id: u64, filename: &str) -> Vec<u8> {
+        let mut chunk_body = Vec::new();
+        {
+            let mut writer = Writer::new(&mut chunk_body);
+            writer.bytes_field(1, &[7u8; 20]);
+            writer.tag(2, crate::proto_wire::WireType::Fixed32);
+            writer.raw_bytes(&0x1234_5678u32.to_le_bytes());
+            writer.uint64_field(3, 0);
+            writer.uint32_field(4, 4);
+            writer.uint32_field(5, 4);
+        }
+        let mut file_body = Vec::new();
+        {
+            let mut writer = Writer::new(&mut file_body);
+            writer.string_field(1, filename);
+            writer.uint64_field(2, 4);
+            writer.submessage_field(6, &chunk_body);
+        }
+        let mut payload = Vec::new();
+        Writer::new(&mut payload).submessage_field(1, &file_body);
+
+        let mut metadata = Vec::new();
+        {
+            let mut writer = Writer::new(&mut metadata);
+            writer.uint32_field(1, depot_id);
+            writer.uint64_field(2, manifest_id);
+            writer.bool_field_force(4, false);
+        }
+
+        let mut raw = Vec::new();
+        push_section(&mut raw, PAYLOAD_MAGIC, &payload);
+        push_section(&mut raw, METADATA_MAGIC, &metadata);
+        raw.extend_from_slice(&END_OF_MANIFEST_MAGIC.to_le_bytes());
+        raw
+    }
+
     fn temp_dir(name: &str) -> PathBuf {
         let dir = std::env::temp_dir().join(format!(
             "wnsteam_downloader_{name}_{}",

app/src/main/cpp/wn-steam-client/rust/src/jni.rs

@@ -37,7 +37,7 @@ unsafe extern "C" {
     fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32;
 }
 
-fn android_log(tag: &str, message: &str) {
+pub(crate) fn android_log(tag: &str, message: &str) {
     #[cfg(target_os = "android")]
     {
         let Ok(tag) = CString::new(tag) else {
@@ -3263,6 +3263,7 @@ pub extern "system" fn Java_com_winlator_cmod_feature_stores_steam_wnsteam_WnSte
     fresh: jboolean,
     ca_bundle_path: JString,
     max_workers: jint,
+    run_cleanup: jboolean,
     listener: JObject,
 ) {
     let Some(handle) = (unsafe { from_session_handle_mut(handle) }) else {
@@ -3463,6 +3464,7 @@ pub extern "system" fn Java_com_winlator_cmod_feature_stores_steam_wnsteam_WnSte
                 Some(download_cancel.as_ref()),
                 Some(progress_cb),
                 Some(code_refresher_cb),
+                run_cleanup != 0,
             );
         dispatch_download_complete(listener, result);
     });