diff --git a/packages/swift-sdk/Package.swift b/packages/swift-sdk/Package.swift index 253d84fcccd..9bc528b370b 100644 --- a/packages/swift-sdk/Package.swift +++ b/packages/swift-sdk/Package.swift @@ -32,7 +32,8 @@ let package = Package( .testTarget( name: "SwiftDashSDKTests", dependencies: ["SwiftDashSDK"], - path: "SwiftTests/SwiftDashSDKTests" + path: "SwiftTests/SwiftDashSDKTests", + resources: [.copy("Fixtures")] ), // Integration tests against a local dashmate devnet. diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/Core/Services/SDKLogger.swift b/packages/swift-sdk/Sources/SwiftDashSDK/Core/Services/SDKLogger.swift index 3bbefb52c37..d57254b06f6 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/Core/Services/SDKLogger.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/Core/Services/SDKLogger.swift @@ -213,21 +213,55 @@ enum SDKLogFormatter { } } -private final class SDKLoggerState: @unchecked Sendable { +/// Internal rather than private so the pre-install buffer can be tested on a +/// fresh instance: the process-wide `SDKLogger.state` has no way back to the +/// "no sink installed" condition once any test has installed one. +final class SDKLoggerState: @unchecked Sendable { + /// How many pre-install events are retained for replay. A host that never + /// installs a sink must not accumulate lines for the life of the process, + /// so once full the buffer keeps what it has, drops the NEWEST arrivals, + /// and reports the loss. Head-not-tail on purpose: the lines this buffer + /// exists for — `core_store_open_result` from the host's `init()` — are + /// the first ones in, and a launch that overflows does so with restore + /// and changeset lines whose loss costs far less than the store open's. + static let pendingLineLimit = 256 + private let lock = NSLock() private var sink: SDKLogFileSink? private var includeDebug = false - - func installSink(at sessionDirectory: URL, includeDebug: Bool) -> Bool { + /// Events emitted before the file sink exists. `DashModelContainer.create` + /// runs in the host's `init()`, long before `LoggingPreferences.configure()` + /// installs the sink, so without this buffer the store-open result — and + /// every other launch-path event — would only ever reach the console and + /// never the exported `swift/run.log`. + private var pendingLines: [(severity: SDKLogSeverity, line: String)] = [] + private var droppedPendingLineCount = 0 + + /// Installs the sink and replays what was emitted before it existed, in + /// emission order and under the sink's own debug filter. + func installSink(at sessionDirectory: URL, includeDebug: Bool) -> ( + installed: Bool, + droppedPendingLineCount: Int + ) { do { let newSink = try SDKLogFileSink(sessionDirectory: sessionDirectory) - lock.withLock { + // Replay under the same lock `record` takes, so a line emitted + // concurrently with the install cannot land in front of the + // backlog it actually followed. + let dropped: Int = lock.withLock { sink = newSink self.includeDebug = includeDebug + for entry in pendingLines where entry.severity != .debug || includeDebug { + newSink.write(entry.line) + } + let droppedCount = droppedPendingLineCount + pendingLines = [] + droppedPendingLineCount = 0 + return droppedCount } - return true + return (installed: true, droppedPendingLineCount: dropped) } catch { - return false + return (installed: false, droppedPendingLineCount: 0) } } @@ -237,16 +271,40 @@ private final class SDKLoggerState: @unchecked Sendable { } } - func destination(for severity: SDKLogSeverity) -> SDKLogFileSink? { - lock.withLock { + /// Routes one formatted line to the sink, or buffers it for replay when no + /// sink has been installed yet. + func record(severity: SDKLogSeverity, line: String) { + let destination: SDKLogFileSink? = lock.withLock { + guard let sink else { + guard pendingLines.count < Self.pendingLineLimit else { + droppedPendingLineCount += 1 + return nil + } + pendingLines.append((severity: severity, line: line)) + return nil + } guard severity != .debug || includeDebug else { return nil } return sink } + destination?.write(line) } func flush() { lock.withLock { sink }?.flush() } + + /// Test seam: drop the sink and everything buffered for it. `record` + /// buffers process-wide while no sink exists and `installSink` replays + /// the whole backlog into whichever session installs first, so a suite + /// asserting over a complete `run.log` must start from nothing. + func reset() { + lock.withLock { + sink = nil + includeDebug = false + pendingLines = [] + droppedPendingLineCount = 0 + } + } } // MARK: - Logging Preferences @@ -473,7 +531,7 @@ public enum SDKLogger { redacting: sensitiveValues ) - state.destination(for: severity)?.write(line) + state.record(severity: severity, line: line) let shouldMirrorToConsole: Bool switch severity { @@ -496,13 +554,36 @@ public enum SDKLogger { } static func installFileSink(at sessionDirectory: URL, includeDebug: Bool) -> Bool { - state.installSink(at: sessionDirectory, includeDebug: includeDebug) + let outcome = state.installSink( + at: sessionDirectory, + includeDebug: includeDebug + ) + if outcome.droppedPendingLineCount > 0 { + // Emitted after the replay so the gap is visible at the point in + // the file where the missing lines would have been. + event( + "log_pre_install_buffer_overflow", + category: .lifecycle, + severity: .warning, + fields: [ + "dropped_line_count": .integer( + Int64(outcome.droppedPendingLineCount) + ), + ] + ) + } + return outcome.installed } static func updateDebugSetting(_ includeDebug: Bool) { state.updateDebugSetting(includeDebug) } + /// Tests only. See `SDKLoggerState.reset()`. + static func resetForTesting() { + state.reset() + } + public static func log(_ message: String, minimumLevel level: LoggingPreset = .medium) { guard LoggingPreferences.allows(level) else { return } // Mirror to NSLog (unified logging) in addition to stdout so diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/Persistence/DashModelContainer.swift b/packages/swift-sdk/Sources/SwiftDashSDK/Persistence/DashModelContainer.swift index eaa0ff44317..18c89e80a67 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/Persistence/DashModelContainer.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/Persistence/DashModelContainer.swift @@ -1,8 +1,328 @@ +import CoreData import Foundation import SwiftData +/// Why `DashModelContainer.open` refused to open a store. Typed so a host can +/// tell a store it must not touch from a store it cannot read, and say the +/// right thing to the user instead of surfacing SwiftData's opaque +/// `loadIssueModelContainer`. +public enum DashModelContainerError: LocalizedError, Equatable { + /// The configuration was built from a `Schema` whose entity set differs + /// from the SDK's. `unexpected` names entities the SDK schema lacks; + /// `missing` names SDK entities the configuration lacks. + case schemaMismatch(unexpected: [String], missing: [String]) + public var errorDescription: String? { + switch self { + case .schemaMismatch(let unexpected, let missing): + return "The store configuration's schema does not match the SDK schema" + + " (unexpected: \(unexpected.joined(separator: ", ")); missing: \(missing.joined(separator: ", ")))." + } + } +} + /// Factory for creating SwiftData model containers for Dash Platform persistence public enum DashModelContainer { + private struct StoreFileSizes { + let main: UInt64 + let wal: UInt64 + let shm: UInt64 + + /// Shares the exporter's saturating rule so a corrupt size can never + /// trap and so both totals move together if that rule ever changes. + var total: UInt64 { + diagnosticSaturatingSum([main, wal, shm]) + } + } + + /// Which of the two open attempts produced the result being reported. + private enum StoreMigrationPath: String { + /// `DashMigrationPlan` accepted the store. + case staged + /// The staged plan rejected the store and SwiftData's inferred + /// lightweight migration was used instead — see `create`. + case inferredFallback = "inferred_fallback" + } + + /// What the store at `storeURL` is, relative to the schemas + /// `DashMigrationPlan` registers. + /// + /// This is the question staged migration answers with Cocoa 134504 when + /// it cannot place a store. It has to be asked here directly, because the + /// error SwiftData surfaces for it is `SwiftDataError.loadIssueModelContainer` + /// with no explanation and no underlying `NSError` — the same value a + /// corrupt file produces — so nothing in the thrown error distinguishes + /// the one failure the fallback may answer from every failure it must not. + enum StoreSchemaVerdict: Equatable { + /// The metadata could not be read: not a version question. + case unreadable + /// A registered schema is compatible with it. The staged plan can + /// open it, so a failure to do so is something else entirely. + case matchesRegisteredVersion + /// Written by a registered version whose live models have since + /// drifted, with exactly the drifted shapes `knownDriftedEntityHashes` + /// lists (every v4.2.0-dev.1 store, until the remaining V1/V2 shapes + /// are frozen). Inferred migration may open it. + case driftedRegisteredVersion + /// The metadata reads but does not place the store against any + /// registered version. Inferred migration must not answer this — an + /// unplaced store opened by inference is trimmed to the current + /// schema, dropping whatever it cannot map — so `open` refuses and + /// rethrows SwiftData's own error. + /// + /// There is deliberately no "written by a newer build" verdict beside + /// this one. Nothing observable here carries a direction. A hash + /// disagreement says two shapes differ, not which came first. An + /// entity the current schema lacks looks identical whether a newer + /// build added it or an older build wrote one since renamed — and this + /// SDK has done exactly that rename (`PersistentUtxo` → `PersistentTxo`, + /// see the migration notes below), so the store that would have been + /// called "from the future" is one of the oldest that exists. An + /// unregistered version identifier is ambiguous for the same reason. + /// + /// So the classification says only that it cannot place the store, + /// never why, and the host has nothing to turn into a reset prompt. + /// `reason` still records which check refused, for the log. + case unplaceable(reason: String) + + var logLabel: String { + switch self { + case .unreadable: return "unreadable" + case .matchesRegisteredVersion: return "matches_registered_version" + case .driftedRegisteredVersion: return "drifted_registered_version" + case .unplaceable(let reason): return "unplaceable:\(reason)" + } + } + } + + /// The exact per-entity version hashes a v4.2.0-dev.1 store carries for + /// the two shapes changed in place since V1 — the reason such a store no + /// longer matches V1's checksum although V1 wrote it. + /// + /// A store is "drifted" only if every entity whose hash disagrees with its + /// declared version's model carries EXACTLY the hash listed here. The hash + /// is a function of the shape, so any other shape of these two entities — + /// a newer build's, with an added attribute — has a different hash and is + /// refused, and so is a disagreement on any other entity. That is what + /// makes the fallback answer precisely the store the fixture proves and + /// nothing else: there is no "same name, unknown shape" residual left. + /// + /// `Dev1StoreUpgradeTests` pins these to the fixture. Extend only with a + /// hash read from a real store of a supported prerelease; shrink as the + /// shapes get frozen in `DashSchemaFrozenModels.swift`, after which the + /// fallback has no case left to answer and can go. + static let knownDriftedEntityHashes: [String: Data] = [ + "PersistentDocumentType": Data(base64Encoded: "w2iUSIQfuddeVRUyE/lgUE7oObvG1pWzO1Ah2IB2xBs=")!, + "PersistentIndex": Data(base64Encoded: "iJRVIyu7GslKt5zO+2oa194YXGtujJqnzcAe1FPWWa8=")!, + ] + + /// One registered version as the verdict sees it: its identifier and + /// the per-entity hashes of the model built from its live types. + struct RegisteredVersionHashes: Equatable { + let identifier: String + let entityHashes: [String: Data] + } + + /// Classifies the store from its metadata alone; never opens it. + static func classifyStore(at storeURL: URL) -> StoreSchemaVerdict { + guard let metadata = try? NSPersistentStoreCoordinator.metadataForPersistentStore( + ofType: NSSQLiteStoreType, + at: storeURL, + options: nil + ) else { return .unreadable } + + let models = DashMigrationPlan.schemas.compactMap { schema -> (String, NSManagedObjectModel)? in + NSManagedObjectModel.makeManagedObjectModel(for: schema.models) + .map { (schema.versionIdentifier.description, $0) } + } + let matches = models.contains { _, model in + model.isConfiguration(withName: nil, compatibleWithStoreMetadata: metadata) + } + return storeSchemaVerdict( + matchesRegisteredVersion: matches, + storeEntityHashes: (metadata[NSStoreModelVersionHashesKey] as? [String: Data]) ?? [:], + storeVersionIdentifiers: (metadata[NSStoreModelVersionIdentifiersKey] as? [String]) ?? [], + registered: models.map { + RegisteredVersionHashes(identifier: $0.0, entityHashes: $0.1.entityVersionHashesByName) + }, + currentEntities: Set(schema.entities.map(\.name)) + ) + } + + /// The decision behind `classifyStore`, on plain values so every branch + /// can be tested without building a store for it. Four checks, in order: + /// a registered version is compatible; the declared version identifier + /// is one this plan registered; every entity is one the current schema + /// has; and every entity disagreeing with the declared version's model + /// carries the one hash `knownDriftedEntityHashes` lists for it. Only + /// the last yields `driftedRegisteredVersion`; everything else that is + /// not a match is refused. + static func storeSchemaVerdict( + matchesRegisteredVersion: Bool, + storeEntityHashes: [String: Data], + storeVersionIdentifiers: [String], + registered: [RegisteredVersionHashes], + currentEntities: Set + ) -> StoreSchemaVerdict { + if matchesRegisteredVersion { return .matchesRegisteredVersion } + + // Nothing to place the store against — every schema failed to build a + // model. Not a fact about the store at all. + guard !registered.isEmpty else { + return .unplaceable(reason: "no_registered_models") + } + + // SwiftData writes each `VersionedSchema.versionIdentifier` into the + // store. One this plan does not register places it nowhere, but says + // nothing about which side is older: a pre-V1 store and a version + // since dropped from `DashMigrationPlan.schemas` look exactly like a + // future one from here. + let registeredIdentifiers = Set(registered.map(\.identifier)) + if let unknown = storeVersionIdentifiers.first(where: { !registeredIdentifiers.contains($0) }) { + return .unplaceable(reason: "unregistered_version_identifier=\(unknown)") + } + // No identifier at all places the store nowhere. Older stores and + // stores with truncated metadata land here too, so this is not a + // newer build and must not be reported to the user as one. + guard !storeVersionIdentifiers.isEmpty else { + return .unplaceable(reason: "no_version_identifier") + } + // Drift is a statement about hashes that disagree. With no hashes to + // read there is nothing to disagree, and every check below would pass + // vacuously — `disagreeing` empty, so `unknownShapes` empty, so + // `driftedRegisteredVersion` from a comparison that never happened, + // authorizing inferred migration over a store nothing is known about. + guard !storeEntityHashes.isEmpty else { + return .unplaceable(reason: "no_entity_hashes") + } + + // An entity the current schema does not have. Inferred migration would + // drop its table, so the store is refused — but not as a newer build: + // a rename leaves an older store carrying a name this schema no longer + // has, and `PersistentUtxo` → `PersistentTxo` is one this SDK actually + // performed. + let unknownEntities = Set(storeEntityHashes.keys).subtracting(currentEntities).sorted() + if !unknownEntities.isEmpty { + return .unplaceable( + reason: "unknown_entities=\(unknownEntities.joined(separator: "|"))" + ) + } + + // Same entity names, so which ones disagree with the version the + // store declares — and is each disagreeing shape the one known drift? + // A newer build that added an attribute keeps the name and the + // identifier but not the hash, whether it touched one of the two + // drifted entities or any other. Only the declared version's model + // is a fair comparison: later versions legitimately differ. + var unexpectedDrift: Set = [] + for version in registered where storeVersionIdentifiers.contains(version.identifier) { + let disagreeing = storeEntityHashes.filter { name, hash in + version.entityHashes[name] != hash + } + let unknownShapes = Set(disagreeing.compactMap { name, hash in + knownDriftedEntityHashes[name] == hash ? nil : name + }) + // `!disagreeing.isEmpty` is the load-bearing half: drift is what + // the fallback answers, and a store that agrees on every hash it + // carries and still is not compatible differs by something these + // hashes do not describe — an entity the store lacks entirely, + // say. Whatever that is, it is not the drift the pinned hashes + // authorize, so it does not get inferred migration. + if !disagreeing.isEmpty, unknownShapes.isEmpty { + return .driftedRegisteredVersion + } + unexpectedDrift.formUnion(unknownShapes) + } + guard !unexpectedDrift.isEmpty else { + return .unplaceable(reason: "no_entity_disagreement") + } + // Deliberately NOT `newerThanRegistered`. A hash disagreement is + // symmetric: it says the store's shape of that entity is not the live + // model's, not which of the two came first. V1/V2/V3 are still + // unfrozen (only `PersistentAssetLock` is frozen), so adding one + // attribute to any live model makes every EXISTING store disagree on + // that entity — an older store, reported as a newer one, with a reset + // offered as the remedy. Direction needs evidence this comparison does + // not have; until the models are frozen, the honest verdict is that + // the store cannot be placed. + return .unplaceable( + reason: "unexpected_entity_drift=\(unexpectedDrift.sorted().joined(separator: "|"))" + ) + } + + /// Builds the common payload for both sides of the container open. The + /// outcome deliberately describes only what SwiftData tells us: opening an + /// existing store may have included a migration, but this API does not + /// expose whether one actually ran. + private static func storeOpenFields( + succeeded: Bool, + existedBefore: Bool, + migrationPath: StoreMigrationPath, + startedAt: CFAbsoluteTime, + sizeBefore: StoreFileSizes, + sizeAfter: StoreFileSizes + ) -> [String: SDKLogValue] { + let elapsed = max(0, (CFAbsoluteTimeGetCurrent() - startedAt) * 1_000) + let duration: UInt64 + if !elapsed.isFinite { + duration = 0 + } else if elapsed >= Double(UInt64.max) { + duration = UInt64.max + } else { + duration = UInt64(elapsed) + } + let openOutcome: String + switch (succeeded, existedBefore) { + case (true, true): + openOutcome = "existing_store_opened" + case (true, false): + openOutcome = "new_store_created" + case (false, true): + openOutcome = "existing_store_open_or_migration_failed" + case (false, false): + openOutcome = "new_store_creation_failed" + } + + return [ + "container_result": .publicText(succeeded ? "opened" : "open_failed"), + "duration_ms": .unsignedInteger(duration), + "migration_path": .publicText(migrationPath.rawValue), + "result": .publicText(succeeded ? "success" : "failure"), + "store_existed_before_open": .boolean(existedBefore), + "store_main_size_bytes_after": .unsignedInteger(sizeAfter.main), + "store_main_size_bytes_before": .unsignedInteger(sizeBefore.main), + "store_open_outcome": .publicText(openOutcome), + "store_shm_size_bytes_after": .unsignedInteger(sizeAfter.shm), + "store_shm_size_bytes_before": .unsignedInteger(sizeBefore.shm), + "store_size_bytes_after": .unsignedInteger(sizeAfter.total), + "store_size_bytes_before": .unsignedInteger(sizeBefore.total), + "store_wal_size_bytes_after": .unsignedInteger(sizeAfter.wal), + "store_wal_size_bytes_before": .unsignedInteger(sizeBefore.wal), + ] + } + + /// SQLite's durable state can be mostly in the WAL immediately after an + /// app kill, so the main file alone is not a useful corruption signal. + /// Read only sizes and never include any component of the device path. + private static func storeFileSizes(at storeURL: URL) -> StoreFileSizes { + func fileSize(at url: URL) -> UInt64 { + // A fresh URL each time: `resourceValues` bridges to `NSURL`, + // which caches a value per instance, and the same `storeURL` is + // read before and after the open. A cached size would report a + // migration that grew the store as no growth at all. + let fresh = URL(fileURLWithPath: url.path) + guard let size = try? fresh.resourceValues(forKeys: [.fileSizeKey]).fileSize, + size >= 0 + else { return 0 } + return UInt64(size) + } + + return StoreFileSizes( + main: fileSize(at: storeURL), + wal: fileSize(at: URL(fileURLWithPath: storeURL.path + "-wal")), + shm: fileSize(at: URL(fileURLWithPath: storeURL.path + "-shm")) + ) + } + /// Every registered schema version's model list, parameterised on the /// one model whose shape differs between versions. /// @@ -145,21 +465,151 @@ public enum DashModelContainer { cloudKit: Bool = false, groupContainer: ModelConfiguration.GroupContainer = .automatic ) throws -> ModelContainer { - let modelConfiguration = ModelConfiguration( - schema: schema, - isStoredInMemoryOnly: false, - allowsSave: true, - groupContainer: groupContainer, - cloudKitDatabase: cloudKit ? .automatic : .none + return try open( + ModelConfiguration( + schema: schema, + isStoredInMemoryOnly: false, + allowsSave: true, + groupContainer: groupContainer, + cloudKitDatabase: cloudKit ? .automatic : .none + ) ) + } + /// The instrumented store-opening path, parameterised on the configuration. + /// + /// Public so a host that builds its own `ModelConfiguration` — DashWallet + /// does, with a per-network URL — gets the same `core_store_open_result` + /// telemetry and the same narrowly-scoped migration fallback as `create`, + /// instead of a bare `ModelContainer(for:configurations:)` that reports + /// nothing. It is also what lets `Dev1StoreUpgradeTests` drive exactly the + /// path that ships against a fixture store. + /// + /// The configuration contributes the store URL and options only. The + /// container is always built for the SDK's own `schema`, because that is + /// what `DashMigrationPlan` migrates toward; a configuration built from a + /// different `Schema` is refused with ``DashModelContainerError`` rather + /// than silently opened under the wrong one. + public static func open(_ modelConfiguration: ModelConfiguration) throws -> ModelContainer { + if let provided = modelConfiguration.schema { + let providedNames = Set(provided.entities.map(\.name)) + let sdkNames = Set(schema.entities.map(\.name)) + guard providedNames == sdkNames else { + throw DashModelContainerError.schemaMismatch( + unexpected: providedNames.subtracting(sdkNames).sorted(), + missing: sdkNames.subtracting(providedNames).sorted() + ) + } + } // Always wire the migration plan so stores created by an older SDK - // advance through the registered versioned schemas. - return try ModelContainer( - for: schema, - migrationPlan: DashMigrationPlan.self, - configurations: [modelConfiguration] - ) + // advance through the registered versioned schemas. Record only + // metadata about the store — never its device path. + let storeURL = modelConfiguration.url + let existedBefore = FileManager.default.fileExists(atPath: storeURL.path) + let sizeBefore = storeFileSizes(at: storeURL) + let started = CFAbsoluteTimeGetCurrent() + + func report( + succeeded: Bool, + migrationPath: StoreMigrationPath, + error: Error? = nil, + storeVerdict: StoreSchemaVerdict? = nil + ) { + var fields = storeOpenFields( + succeeded: succeeded, + existedBefore: existedBefore, + migrationPath: migrationPath, + startedAt: started, + sizeBefore: sizeBefore, + sizeAfter: storeFileSizes(at: storeURL) + ) + if let storeVerdict { + fields["store_verdict"] = .publicText(storeVerdict.logLabel) + } + SDKLogger.event( + "core_store_open_result", + category: .persistence, + severity: succeeded ? .info : .error, + fields: fields, + error: error, + redacting: [storeURL.path] + ) + } + + do { + let container = try ModelContainer( + for: schema, + migrationPlan: DashMigrationPlan.self, + configurations: [modelConfiguration] + ) + report(succeeded: true, migrationPath: .staged) + return container + } catch { + // Staged migration matches a store by the CHECKSUM of each + // registered `VersionedSchema`, and only `PersistentAssetLock` is + // frozen so far (see `DashSchemaFrozenModels.swift`). Every other + // V1/V2 model is still referenced live, so a shape that has drifted + // since — `PersistentDocumentType` and `PersistentIndex` for the + // v4.2.0-dev.1 stores `Dev1StoreUpgradeTests` pins — leaves the + // real store matching no registered version, and the staged open + // fails with Cocoa 134504 rather than migrating. + // + // Hosts turn that throw into `fatalError` at launch, so for THAT + // failure retry the way they already open the store themselves: + // the current schema with SwiftData's inferred lightweight + // migration and no plan. The match is deliberately exact, and it + // is made on the store rather than the error (see + // `classifyStore`). Every stage in + // `DashMigrationPlan` is `.lightweight` today, but the day a + // custom stage lands, a failure inside it must surface — a store + // that matches a registered version and still failed to open is + // exactly that case, and falling back would reopen it without the + // stage and stamp the current checksum on it, so the stage could + // never run later. And a store written by a NEWER build — a + // downgrade — would be opened by inference and silently trimmed + // to this schema, which is worse than the crash it replaces. + // Everything except "existing store, drifted registered version" + // is therefore rethrown untouched, with the verdict in the log. + let verdict: StoreSchemaVerdict = existedBefore + ? Self.classifyStore(at: storeURL) + : .unreadable + guard case .driftedRegisteredVersion = verdict else { + report(succeeded: false, migrationPath: .staged, error: error, storeVerdict: verdict) + // No typed error for any of these. Every refusal here means + // "this store cannot be placed", never "this store is newer" — + // the evidence does not distinguish them — and only the second + // would justify telling a user to update or reset. SwiftData's + // own failure goes through untouched, with the verdict in the + // log for whoever reads the export. + throw error + } + SDKLogger.event( + "core_store_staged_migration_failed", + category: .persistence, + severity: .warning, + fields: [ + "store_existed_before_open": .boolean(existedBefore), + "store_verdict": .publicText(verdict.logLabel), + ], + error: error, + redacting: [storeURL.path] + ) + do { + let container = try ModelContainer( + for: schema, + configurations: [modelConfiguration] + ) + report(succeeded: true, migrationPath: .inferredFallback) + return container + } catch let fallbackError { + report( + succeeded: false, + migrationPath: .inferredFallback, + error: fallbackError + ) + throw fallbackError + } + } } /// Create an in-memory model container for testing diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/CoreWalletDiagnosticAnalyzers.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/CoreWalletDiagnosticAnalyzers.swift new file mode 100644 index 00000000000..849bd94cd71 --- /dev/null +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/CoreWalletDiagnosticAnalyzers.swift @@ -0,0 +1,573 @@ +import Foundation + +/// Value-only analyzers shared by the diagnostic logger and its unit tests. +/// Keeping comparison and truncation here makes the tests exercise the exact +/// decisions that produce `swift/run.log`, without requiring a live Rust +/// wallet handle. +enum CoreWalletDiagnosticAnalyzer { + /// One deterministic mismatch record. `outpoint` is hashed by the logger + /// and is never rendered directly. + struct TxoDiffDetail: Sendable { + let outpoint: Data + let reason: String + let row: CoreWalletDatabaseDiagnosticSnapshot.Txo + } + + /// Aggregate DB↔Rust account/UTXO comparison plus bounded log details. + struct TxoDiff: Sendable { + let commonCount: Int + let databaseAccountOnlyCount: Int + let memoryAccountOnlyCount: Int + let databaseOnlyCount: Int + let memoryOnlyCount: Int + let fieldMismatchCount: Int + let details: [TxoDiffDetail] + let emittedDetails: [TxoDiffDetail] + let truncatedCount: Int + } + + /// Compares UTXOs by outpoint and reports each differing field separately. + /// Duplicate outpoints are resolved deterministically before comparison. + static func compareTxos( + database: [CoreWalletDatabaseDiagnosticSnapshot.Txo], + memory: [CoreWalletDatabaseDiagnosticSnapshot.Txo], + databaseAccounts: Set, + memoryAccounts: Set + ) -> TxoDiff { + var databaseByOutpoint: [Data: CoreWalletDatabaseDiagnosticSnapshot.Txo] = [:] + for row in database.sorted(by: txoOrder) where databaseByOutpoint[row.outpoint] == nil { + databaseByOutpoint[row.outpoint] = row + } + var memoryByOutpoint: [Data: CoreWalletDatabaseDiagnosticSnapshot.Txo] = [:] + for row in memory.sorted(by: txoOrder) where memoryByOutpoint[row.outpoint] == nil { + memoryByOutpoint[row.outpoint] = row + } + + let databaseOnly = databaseByOutpoint.keys + .filter { memoryByOutpoint[$0] == nil } + .sorted { $0.lexicographicallyPrecedes($1) } + let memoryOnly = memoryByOutpoint.keys + .filter { databaseByOutpoint[$0] == nil } + .sorted { $0.lexicographicallyPrecedes($1) } + + var mismatchDetails: [TxoDiffDetail] = [] + for outpoint in databaseByOutpoint.keys.sorted(by: { $0.lexicographicallyPrecedes($1) }) { + guard let databaseRow = databaseByOutpoint[outpoint], + let memoryRow = memoryByOutpoint[outpoint] + else { continue } + if databaseRow.amount != memoryRow.amount { + mismatchDetails.append(.init( + outpoint: outpoint, + reason: "amount_mismatch", + row: databaseRow + )) + } + if databaseRow.height != memoryRow.height { + mismatchDetails.append(.init( + outpoint: outpoint, + reason: "height_mismatch", + row: databaseRow + )) + } + if databaseRow.scriptPubKey != memoryRow.scriptPubKey { + mismatchDetails.append(.init( + outpoint: outpoint, + reason: "script_mismatch", + row: databaseRow + )) + } + if databaseRow.isLocked != memoryRow.isLocked { + mismatchDetails.append(.init( + outpoint: outpoint, + reason: "lock_mismatch", + row: databaseRow + )) + } + if databaseRow.account != memoryRow.account { + mismatchDetails.append(.init( + outpoint: outpoint, + reason: "account_mismatch", + row: databaseRow + )) + } + } + + var details = databaseOnly.compactMap { outpoint in + databaseByOutpoint[outpoint].map { + TxoDiffDetail(outpoint: outpoint, reason: "database_only", row: $0) + } + } + details.append(contentsOf: memoryOnly.compactMap { outpoint in + memoryByOutpoint[outpoint].map { + TxoDiffDetail(outpoint: outpoint, reason: "memory_only", row: $0) + } + }) + details.append(contentsOf: mismatchDetails) + details.sort(by: txoDetailOrder) + let limited = limitedTxoDetails(details) + + return TxoDiff( + commonCount: Set(databaseByOutpoint.keys).intersection(memoryByOutpoint.keys).count, + databaseAccountOnlyCount: databaseAccounts.subtracting(memoryAccounts).count, + memoryAccountOnlyCount: memoryAccounts.subtracting(databaseAccounts).count, + databaseOnlyCount: databaseOnly.count, + memoryOnlyCount: memoryOnly.count, + fieldMismatchCount: mismatchDetails.count, + details: details, + emittedDetails: limited.emitted, + truncatedCount: limited.truncated + ) + } + + /// One AssetLock mismatch, retaining the display outpoint only so the + /// logger can derive a stable reference from it. + struct AssetLockDiffDetail: Sendable { + let outpointDisplay: String + let reason: String + } + + /// Complete AssetLock mismatch set and its per-reason bounded projection. + struct AssetLockDiff: Sendable { + let details: [AssetLockDiffDetail] + let emittedDetails: [AssetLockDiffDetail] + let truncatedCount: Int + } + + /// Compares persisted and Rust-tracked locks without exposing transaction + /// or proof bytes. + static func compareAssetLocks( + database: [CoreWalletDatabaseDiagnosticSnapshot.AssetLock], + memory: [CoreWalletDatabaseDiagnosticSnapshot.AssetLock] + ) -> AssetLockDiff { + var databaseByOutpoint: [String: CoreWalletDatabaseDiagnosticSnapshot.AssetLock] = [:] + for row in database.sorted(by: assetLockOrder) + where databaseByOutpoint[row.outpointDisplay] == nil { + databaseByOutpoint[row.outpointDisplay] = row + } + var memoryByOutpoint: [String: CoreWalletDatabaseDiagnosticSnapshot.AssetLock] = [:] + for row in memory.sorted(by: assetLockOrder) + where memoryByOutpoint[row.outpointDisplay] == nil { + memoryByOutpoint[row.outpointDisplay] = row + } + + var details: [AssetLockDiffDetail] = [] + for outpoint in databaseByOutpoint.keys where memoryByOutpoint[outpoint] == nil { + details.append(.init(outpointDisplay: outpoint, reason: "database_only")) + } + for outpoint in memoryByOutpoint.keys where databaseByOutpoint[outpoint] == nil { + details.append(.init(outpointDisplay: outpoint, reason: "memory_only")) + } + for outpoint in databaseByOutpoint.keys.sorted() { + guard let databaseRow = databaseByOutpoint[outpoint], + let memoryRow = memoryByOutpoint[outpoint] + else { continue } + if databaseRow.fundingType != memoryRow.fundingType { + details.append(.init(outpointDisplay: outpoint, reason: "funding_type_mismatch")) + } + if databaseRow.status != memoryRow.status { + details.append(.init(outpointDisplay: outpoint, reason: "status_mismatch")) + } + if databaseRow.accountIndex != memoryRow.accountIndex { + details.append(.init(outpointDisplay: outpoint, reason: "account_index_mismatch")) + } + if databaseRow.registrationIndex != memoryRow.registrationIndex { + details.append(.init( + outpointDisplay: outpoint, + reason: "registration_index_mismatch" + )) + } + if databaseRow.amountDuffs != memoryRow.amountDuffs { + details.append(.init(outpointDisplay: outpoint, reason: "amount_mismatch")) + } + if databaseRow.hasProof != memoryRow.hasProof { + details.append(.init( + outpointDisplay: outpoint, + reason: "proof_presence_mismatch" + )) + } + } + details.sort(by: assetLockDetailOrder) + let limited = limitedAssetLockDetails(details) + return AssetLockDiff( + details: details, + emittedDetails: limited.emitted, + truncatedCount: limited.truncated + ) + } + + /// Lightweight description of a row considered by startup restore. + struct RestoreCandidate: Sendable { + enum RejectionReason: String, Sendable { + case missingAccount = "missing_account" + case invalidTxid = "invalid_txid" + case invalidAccountType = "invalid_account_type" + } + + /// Only the fields required by the restore summary. Keeping this + /// intentionally small prevents the launch-time callback from doing + /// any diagnostic outpoint/script materialization or fingerprinting. + let amount: UInt64 + let accountType: UInt32? + let standardTag: UInt8? + let rejectionReason: RejectionReason? + } + + /// Counts and values for candidates, rows actually emitted, and each + /// validation rejection reason. + struct RestoreBufferSummary: Sendable { + let candidateCount: Int + let candidateValueDuffs: UInt64 + let candidateBip44Count: Int + let candidateBip44ValueDuffs: UInt64 + let candidateCoinJoinCount: Int + let candidateCoinJoinValueDuffs: UInt64 + let builtCount: Int + let emittedCount: Int + let emittedValueDuffs: UInt64 + let emittedBip44Count: Int + let emittedBip44ValueDuffs: UInt64 + let emittedCoinJoinCount: Int + let emittedCoinJoinValueDuffs: UInt64 + let missingAccountCount: Int + let invalidTxidCount: Int + let invalidAccountTypeCount: Int + } + + /// Reconciles the candidate list with the compact FFI buffer length. An + /// errored build reports zero emitted rows even if validation failed late. + /// + /// Deliberately a single pass that accumulates into locals. This runs on + /// the launch restore path while the persistence queue is held, so a large + /// CoinJoin wallet must not pay for a dozen full-length `filter`/`map` + /// allocations, and nothing here retains a per-row array. + /// - Parameter candidateCountOverride: reported instead of the number of + /// candidates walked. For the errored path, where classifying each row + /// would fault a relationship per row to describe a load that is being + /// discarded: the caller passes the row count it already has and an + /// empty sequence, so `candidate_count` stays truthful and every other + /// counter is honestly zero. + static func summarizeRestoreBuffer( + candidates: S, + emittedCount: Int, + errored: Bool, + candidateCountOverride: Int? = nil + ) -> RestoreBufferSummary where S.Element == RestoreCandidate { + // Rust is handed the first `emittedCount` rows that passed validation, + // in order; an errored build deallocated the whole buffer, so none of + // them reached it. + let emissionLimit = errored ? 0 : max(0, emittedCount) + + var candidateCount = 0 + var candidateValue: UInt64 = 0 + var candidateBip44Count = 0 + var candidateBip44Value: UInt64 = 0 + var candidateCoinJoinCount = 0 + var candidateCoinJoinValue: UInt64 = 0 + var emitted = 0 + var emittedValue: UInt64 = 0 + var emittedBip44Count = 0 + var emittedBip44Value: UInt64 = 0 + var emittedCoinJoinCount = 0 + var emittedCoinJoinValue: UInt64 = 0 + var missingAccountCount = 0 + var invalidTxidCount = 0 + var invalidAccountTypeCount = 0 + var validSeen = 0 + + for candidate in candidates { + candidateCount += 1 + let isBip44 = candidate.accountType == 0 && candidate.standardTag == 0 + let isCoinJoin = candidate.accountType == 1 + candidateValue = diagnosticSaturatingAdd(candidateValue, candidate.amount) + if isBip44 { + candidateBip44Count += 1 + candidateBip44Value = diagnosticSaturatingAdd( + candidateBip44Value, + candidate.amount + ) + } + if isCoinJoin { + candidateCoinJoinCount += 1 + candidateCoinJoinValue = diagnosticSaturatingAdd( + candidateCoinJoinValue, + candidate.amount + ) + } + + switch candidate.rejectionReason { + case .missingAccount: + missingAccountCount += 1 + case .invalidTxid: + invalidTxidCount += 1 + case .invalidAccountType: + invalidAccountTypeCount += 1 + case nil: + let emissionIndex = validSeen + validSeen += 1 + guard emissionIndex < emissionLimit else { continue } + emitted += 1 + emittedValue = diagnosticSaturatingAdd(emittedValue, candidate.amount) + if isBip44 { + emittedBip44Count += 1 + emittedBip44Value = diagnosticSaturatingAdd( + emittedBip44Value, + candidate.amount + ) + } + if isCoinJoin { + emittedCoinJoinCount += 1 + emittedCoinJoinValue = diagnosticSaturatingAdd( + emittedCoinJoinValue, + candidate.amount + ) + } + } + } + + return RestoreBufferSummary( + candidateCount: candidateCountOverride ?? candidateCount, + candidateValueDuffs: candidateValue, + candidateBip44Count: candidateBip44Count, + candidateBip44ValueDuffs: candidateBip44Value, + candidateCoinJoinCount: candidateCoinJoinCount, + candidateCoinJoinValueDuffs: candidateCoinJoinValue, + builtCount: emittedCount, + emittedCount: emitted, + emittedValueDuffs: emittedValue, + emittedBip44Count: emittedBip44Count, + emittedBip44ValueDuffs: emittedBip44Value, + emittedCoinJoinCount: emittedCoinJoinCount, + emittedCoinJoinValueDuffs: emittedCoinJoinValue, + missingAccountCount: missingAccountCount, + invalidTxidCount: invalidTxidCount, + invalidAccountTypeCount: invalidAccountTypeCount + ) + } + + /// Persistent facts used to detect malformed or contradictory TXO rows. + struct DatabaseTxoAuditRow: Sendable { + let txo: CoreWalletDatabaseDiagnosticSnapshot.Txo + let hasParentTransaction: Bool + let walletIdMismatch: Bool + let isSpent: Bool + let hasSpendingTransaction: Bool + /// Whether the linked spending transaction has reached a confirmed + /// context. `nil` when there is no spending transaction to ask. + /// Load-bearing: a linked-but-unconfirmed spender with `isSpent == + /// false` is the normal in-flight send, not an anomaly. + let spendingTransactionIsInBlock: Bool? + + init( + txo: CoreWalletDatabaseDiagnosticSnapshot.Txo, + hasParentTransaction: Bool, + walletIdMismatch: Bool, + isSpent: Bool, + hasSpendingTransaction: Bool, + spendingTransactionIsInBlock: Bool? = nil + ) { + self.txo = txo + self.hasParentTransaction = hasParentTransaction + self.walletIdMismatch = walletIdMismatch + self.isSpent = isSpent + self.hasSpendingTransaction = hasSpendingTransaction + self.spendingTransactionIsInBlock = spendingTransactionIsInBlock + } + } + + /// A database anomaly whose raw TXO identity is later hashed by the logger. + struct DatabaseTxoAnomaly: Sendable { + let txo: CoreWalletDatabaseDiagnosticSnapshot.Txo + let reason: String + } + + /// Complete database anomaly set and its per-reason bounded projection. + struct DatabaseTxoAnomalyResult: Sendable { + let details: [DatabaseTxoAnomaly] + let emittedDetails: [DatabaseTxoAnomaly] + let truncatedCount: Int + /// Per-reason totals, computed once from the grouping the truncation + /// already needed: the summary asks eight times, on the held queue. + let countsByReason: [String: Int] + + func count(reason: String) -> Int { + countsByReason[reason] ?? 0 + } + } + + /// Derives all applicable anomaly reasons for every supplied row. + static func databaseTxoAnomalies( + _ rows: [DatabaseTxoAuditRow] + ) -> DatabaseTxoAnomalyResult { + var details: [DatabaseTxoAnomaly] = [] + for row in rows { + if row.txo.account == nil { + details.append(.init(txo: row.txo, reason: "missing_account")) + } + if !row.hasParentTransaction { + details.append(.init(txo: row.txo, reason: "missing_parent_transaction")) + } + if row.walletIdMismatch { + details.append(.init(txo: row.txo, reason: "wallet_id_mismatch")) + } + if row.isSpent && !row.hasSpendingTransaction { + details.append(.init( + txo: row.txo, + reason: "spent_without_spending_transaction" + )) + } + // `reconcileSpendObservation` deliberately links a mempool + // spender while leaving `isSpent == false`, because a sighting + // alone is reversible by RBF or eviction. Only a spender that has + // landed in a block contradicts an unspent row; flagging the + // mempool case would put one warning per output on every wallet + // with an unconfirmed outgoing transaction and bury the real + // anomalies this export exists to surface. + if !row.isSpent && row.spendingTransactionIsInBlock == true { + details.append(.init( + txo: row.txo, + reason: "unspent_with_confirmed_spending_transaction" + )) + } + if row.txo.outpoint.count != 36 { + details.append(.init(txo: row.txo, reason: "invalid_outpoint_length")) + } + if row.txo.scriptPubKey.isEmpty { + details.append(.init(txo: row.txo, reason: "empty_script_pubkey")) + } + } + details.sort { + if $0.reason != $1.reason { return $0.reason < $1.reason } + return $0.txo.outpoint.lexicographicallyPrecedes($1.txo.outpoint) + } + let grouped = Dictionary(grouping: details, by: \.reason) + var emitted: [DatabaseTxoAnomaly] = [] + var truncated = 0 + for reason in grouped.keys.sorted() { + let rows = grouped[reason] ?? [] + emitted.append(contentsOf: rows.prefix(CoreDiagnosticConstants.detailLimit)) + truncated += max(0, rows.count - CoreDiagnosticConstants.detailLimit) + } + return .init( + details: details, + emittedDetails: emitted, + truncatedCount: truncated, + countsByReason: grouped.mapValues(\.count) + ) + } + + /// Value and spent state of a shielded note; identifiers are unnecessary + /// for the aggregate store diagnostic. + struct ShieldedNote: Sendable { + let value: UInt64 + let isSpent: Bool + } + + /// Aggregate shielded persistence state used by the exported snapshot. + struct ShieldedStoreSummary: Sendable { + let noteCount: Int + let spentNoteCount: Int + let spentValueCredits: UInt64 + let unspentNoteCount: Int + let unspentValueCredits: UInt64 + let outgoingNoteCount: Int + let activityCount: Int + let activityPendingCount: Int + let activityFailedCount: Int + let viewingKeyCount: Int + let subwalletSyncStateCount: Int + let maximumSyncWatermark: UInt64 + } + + /// Aggregates shielded note values, activity state, keys, and watermarks + /// without retaining any note or viewing-key identifiers. + static func summarizeShieldedStore( + notes: [ShieldedNote], + outgoingNoteCount: Int, + activityStatuses: [Int], + viewingKeyCount: Int, + syncWatermarks: [UInt64] + ) -> ShieldedStoreSummary { + let spent = notes.filter(\.isSpent) + let unspent = notes.filter { !$0.isSpent } + return ShieldedStoreSummary( + noteCount: notes.count, + spentNoteCount: spent.count, + spentValueCredits: diagnosticSaturatingSum(spent.map(\.value)), + unspentNoteCount: unspent.count, + unspentValueCredits: diagnosticSaturatingSum(unspent.map(\.value)), + outgoingNoteCount: outgoingNoteCount, + activityCount: activityStatuses.count, + activityPendingCount: activityStatuses.filter { $0 == 0 }.count, + activityFailedCount: activityStatuses.filter { $0 == 2 }.count, + viewingKeyCount: viewingKeyCount, + subwalletSyncStateCount: syncWatermarks.count, + maximumSyncWatermark: syncWatermarks.max() ?? 0 + ) + } + + private static func limitedTxoDetails( + _ details: [TxoDiffDetail] + ) -> (emitted: [TxoDiffDetail], truncated: Int) { + let grouped = Dictionary(grouping: details, by: \.reason) + var emitted: [TxoDiffDetail] = [] + var truncated = 0 + for reason in grouped.keys.sorted() { + let rows = (grouped[reason] ?? []).sorted(by: txoDetailOrder) + emitted.append(contentsOf: rows.prefix(CoreDiagnosticConstants.detailLimit)) + truncated += max(0, rows.count - CoreDiagnosticConstants.detailLimit) + } + return (emitted, truncated) + } + + private static func limitedAssetLockDetails( + _ details: [AssetLockDiffDetail] + ) -> (emitted: [AssetLockDiffDetail], truncated: Int) { + let grouped = Dictionary(grouping: details, by: \.reason) + var emitted: [AssetLockDiffDetail] = [] + var truncated = 0 + for reason in grouped.keys.sorted() { + let rows = (grouped[reason] ?? []).sorted(by: assetLockDetailOrder) + emitted.append(contentsOf: rows.prefix(CoreDiagnosticConstants.detailLimit)) + truncated += max(0, rows.count - CoreDiagnosticConstants.detailLimit) + } + return (emitted, truncated) + } + + private static func txoOrder( + _ lhs: CoreWalletDatabaseDiagnosticSnapshot.Txo, + _ rhs: CoreWalletDatabaseDiagnosticSnapshot.Txo + ) -> Bool { + if lhs.outpoint != rhs.outpoint { + return lhs.outpoint.lexicographicallyPrecedes(rhs.outpoint) + } + if lhs.amount != rhs.amount { return lhs.amount < rhs.amount } + if lhs.height != rhs.height { return lhs.height < rhs.height } + if lhs.scriptPubKey != rhs.scriptPubKey { + return lhs.scriptPubKey.lexicographicallyPrecedes(rhs.scriptPubKey) + } + if lhs.isLocked != rhs.isLocked { return !lhs.isLocked && rhs.isLocked } + let lhsAccount = lhs.account?.referenceMaterial ?? Data() + let rhsAccount = rhs.account?.referenceMaterial ?? Data() + return lhsAccount.lexicographicallyPrecedes(rhsAccount) + } + + private static func txoDetailOrder(_ lhs: TxoDiffDetail, _ rhs: TxoDiffDetail) -> Bool { + if lhs.reason != rhs.reason { return lhs.reason < rhs.reason } + return lhs.outpoint.lexicographicallyPrecedes(rhs.outpoint) + } + + private static func assetLockOrder( + _ lhs: CoreWalletDatabaseDiagnosticSnapshot.AssetLock, + _ rhs: CoreWalletDatabaseDiagnosticSnapshot.AssetLock + ) -> Bool { + lhs.outpointDisplay < rhs.outpointDisplay + } + + private static func assetLockDetailOrder( + _ lhs: AssetLockDiffDetail, + _ rhs: AssetLockDiffDetail + ) -> Bool { + if lhs.reason != rhs.reason { return lhs.reason < rhs.reason } + return lhs.outpointDisplay < rhs.outpointDisplay + } +} diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift index dbac9df6306..aa18440acf3 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift @@ -607,6 +607,40 @@ public class PlatformWalletManager: ObservableObject { /// data (for create, the caller would roll back its mnemonic and /// orphan the persisted rows). private var activeNativeOpCount = 0 + /// Read-only Core diagnostics have their own admission count. They must + /// keep the manager handle alive until their FFI reads finish, but they do + /// not make synchronous create/load/delete operations unsafe and therefore + /// must not participate in `ensureSyncNativeOpAllowed`. + private var activeCoreDiagnosticsNativeOpCount = 0 + /// Set by `shutdown()` before it drains `activeCoreDiagnosticsNativeOpCount`. + /// A diagnostic pass checks it before every stage, so the drain waits for + /// at most the one stage already in flight — never for the rest of an + /// export — and a support export can never outlive the process it is + /// diagnosing. Never reset: a manager is shut down once, and a pass that + /// has been told to stop must not be able to un-tell itself. + /// + /// Which is why `shutdown()` raises it only when there is a real teardown + /// or a pass to stop — see the call site. A never-configured manager takes + /// an UNCACHED no-op return from `shutdown()` precisely so it can still be + /// configured afterwards, and a one-way latch set on that path would + /// silence diagnostics for the rest of a perfectly live manager's life. + let coreDiagnosticsCancellation = CoreDiagnosticsCancellation() + + /// Diagnostic passes in flight, including the SwiftData half that runs + /// with no handle at all. `activeCoreDiagnosticsNativeOpCount` cannot + /// stand in for this: it counts only admitted FFI work, so it is zero in + /// exactly the unconfigured case whose database half still needs to be + /// told to stop. + private var activeCoreDiagnosticsPassCount = 0 + + func beginCoreDiagnosticsPass() { + activeCoreDiagnosticsPassCount += 1 + } + + func endCoreDiagnosticsPass() { + guard activeCoreDiagnosticsPassCount > 0 else { return } + activeCoreDiagnosticsPassCount -= 1 + } private var nativeOpDrainContinuations: [CheckedContinuation] = [] /// Admission + bookkeeping shared by the async native entrypoints: @@ -624,11 +658,46 @@ public class PlatformWalletManager: ObservableObject { private func finishNativeOp() { activeNativeOpCount -= 1 - if activeNativeOpCount == 0, !nativeOpDrainContinuations.isEmpty { - let waiters = nativeOpDrainContinuations - nativeOpDrainContinuations.removeAll() - waiters.forEach { $0.resume() } + resumeNativeOpDrainIfIdle() + } + + /// Reserves the native manager handle for one read-only diagnostic pass. + /// Shutdown drains this counter, while synchronous wallet operations + /// intentionally ignore it because Rust serializes its own wallet state. + /// Every successful admission must be balanced by + /// ``finishCoreDiagnosticsNativeOp()``. + func admitCoreDiagnosticsNativeOp() throws { + guard !shutdownRequested else { + throw PlatformWalletError.invalidHandle( + "manager shutdown is in progress; coreWalletDiagnostics rejected") + } + activeCoreDiagnosticsNativeOpCount += 1 + } + + /// Releases a successful diagnostic admission. The guard keeps a future + /// shutdown from observing a negative counter if an internal caller ever + /// violates the admission/defer contract. + func finishCoreDiagnosticsNativeOp() { + guard activeCoreDiagnosticsNativeOpCount > 0 else { + SDKLogger.event( + "core_diagnostics_native_op_counter_underflow", + category: .lifecycle, + severity: .error + ) + return } + activeCoreDiagnosticsNativeOpCount -= 1 + resumeNativeOpDrainIfIdle() + } + + private func resumeNativeOpDrainIfIdle() { + guard activeNativeOpCount == 0, + activeCoreDiagnosticsNativeOpCount == 0, + !nativeOpDrainContinuations.isEmpty + else { return } + let waiters = nativeOpDrainContinuations + nativeOpDrainContinuations.removeAll() + waiters.forEach { $0.resume() } } /// Test seam for the individual native calls. Production keeps `.live`; @@ -720,6 +789,20 @@ public class PlatformWalletManager: ObservableObject { qos: .userInitiated ) + /// Queue for the read-only Core diagnostic FFI reads. Deliberately NOT + /// [`destroyQueue`]: those reads are neither lifecycle operations nor + /// short, and putting them on the lifecycle queue would make one export + /// delay every manager's create/teardown, while a slow create or destroy + /// elsewhere would stall the diagnostics continuation — holding + /// `activeCoreDiagnosticsNativeOpCount` and so the next `shutdown()` drain. + /// Handle safety comes from that admission counter, not from FIFO ordering + /// with teardown, so a separate queue costs nothing. `.utility` because a + /// support export must never outrank the user's own wallet work. + nonisolated static let coreDiagnosticsQueue = DispatchQueue( + label: "org.dash.platform-wallet.core-diagnostics", + qos: .utility + ) + // MARK: - Init /// Empty init for `@StateObject` usage. Call [`configure`] before @@ -815,6 +898,23 @@ public class PlatformWalletManager: ObservableObject { if let task = shutdownTask { return await task.value } + // Before the handle guard, not after it: a diagnostic pass can be + // running on the persistence queue for a manager that was never + // configured (`emitCoreWalletDiagnostics` runs its database half + // with no handle), and the early return below would leave it with + // no way to be told to stop — holding the queue, and every Rust + // persister callback entering through it, across teardown. + // + // But only as conditionally as the shutdown state it accompanies: + // the guard's no-op return is deliberately UNCACHED so a manager + // built and shut down before `configure()` can still be configured + // later, and this latch is one-way, so raising it there would leave + // that live manager unable to produce a support export ever again. + // A real teardown, or a pass actually in flight, is the whole set + // of cases with something to cancel. + if handle != NULL_HANDLE || activeCoreDiagnosticsPassCount > 0 { + coreDiagnosticsCancellation.cancel() + } guard handle != NULL_HANDLE else { // Never configured (or a test double without a handle): // nothing to tear down. Do not cache this no-op: a manager @@ -826,13 +926,14 @@ public class PlatformWalletManager: ObservableObject { ranOffMainThread: false) } shutdownRequested = true + coreDiagnosticsCancellation.cancel() // No new poll tick from here on: both loops check cancellation // before every tick, and `beginPollTick` refuses once the handle // is taken below. A tick already dispatched completes on its own // queue against the registry (see `startProgressPolling`). progressPollTask?.cancel() walletPollTask?.cancel() - if activeNativeOpCount == 0 { break } + if activeNativeOpCount == 0, activeCoreDiagnosticsNativeOpCount == 0 { break } await withCheckedContinuation { continuation in nativeOpDrainContinuations.append(continuation) } @@ -2673,50 +2774,56 @@ public class PlatformWalletManager: ObservableObject { return [] } + switch Self.readAccountBalances(handle: handle, walletId: walletId) { + case .success(let balances): + return balances + case .failure(let error): + self.lastError = error + return [] + } + } + + + /// The one place `platform_wallet_manager_get_account_balances` is called + /// and its entries copied out. `nonisolated static` so the read-only + /// diagnostics, which run off the main actor, share it instead of carrying + /// a second copy that would report stale fields the day + /// `AccountBalanceEntryFFI` gains one. Frees the Rust allocation on every + /// successful non-empty path. + nonisolated static func readAccountBalances( + handle: Handle, + walletId: Data + ) -> Result<[AccountBalance], PlatformWalletError> { var outEntries: UnsafePointer? var outCount: UInt = 0 - - let ffiResult = walletId.withUnsafeBytes { (raw: UnsafeRawBufferPointer) -> PlatformWalletFFIResult in - let base = raw.baseAddress?.assumingMemoryBound(to: UInt8.self) - return platform_wallet_manager_get_account_balances( + let ffi = walletId.withUnsafeBytes { raw in + platform_wallet_manager_get_account_balances( handle, - base, + raw.baseAddress?.assumingMemoryBound(to: UInt8.self), &outEntries, &outCount ) } - - let result = PlatformWalletResult(ffiResult) - - - guard result.isSuccess else { - self.lastError = PlatformWalletError(result: result) - return [] - } - - guard let entries = outEntries, outCount > 0 else { - return [] - } - + let result = PlatformWalletResult(ffi) + guard result.isSuccess else { return .failure(PlatformWalletError(result: result)) } + guard let entries = outEntries, outCount > 0 else { return .success([]) } defer { platform_wallet_manager_free_account_balances( - UnsafeMutablePointer(mutating: entries), - outCount + UnsafeMutablePointer(mutating: entries), outCount ) } - - return (0.. Data { + if value.count == 32 { return value } + if value.count > 32 { return Data(value.prefix(32)) } + var padded = Data(value) + padded.append(Data(repeating: 0, count: 32 - value.count)) + return padded + } + } + + /// Minimal owned-output representation required for deterministic diffing. + struct Txo: Sendable { + let outpoint: Data + let amount: UInt64 + let height: UInt32 + let scriptPubKey: Data + let isLocked: Bool + let account: AccountKey? + } + + /// Comparable subset of one tracked AssetLock; no transaction or proof + /// bytes cross the SwiftData queue boundary. + struct AssetLock: Sendable { + let outpointDisplay: String + let fundingType: Int + let status: Int + let accountIndex: UInt32 + let registrationIndex: UInt32 + /// `nil` represents a corrupt negative value in the signed legacy + /// SwiftData column; a valid in-memory `UInt64` can never equal it. + let amountDuffs: UInt64? + let hasProof: Bool + } + + let walletId: Data + let accounts: [AccountKey] + let unspentTxos: [Txo] + let assetLocks: [AssetLock] + let assetLocksAvailable: Bool +} + +enum CoreDiagnosticConstants { + static let detailLimit = 25 +} + +/// One-way flag from `PlatformWalletManager.shutdown()` to a diagnostic pass +/// running off the main actor. Checked before every FFI read; once set, the +/// pass reports the reads it skipped and returns, releasing its admission. +final class CoreDiagnosticsCancellation: @unchecked Sendable { + private let lock = NSLock() + private var cancelled = false + + var isCancelled: Bool { + lock.withLock { cancelled } + } + + func cancel() { + lock.withLock { cancelled = true } + } +} + +/// Ceilings on what one support export may materialize at once. +/// +/// The exact #4438 audit needs every TXO and every transaction cross-wallet — +/// an output absent from this wallet may be `wrong_wallet`, not `missing_txo`, +/// and only the full table can say which. That pass runs inside the +/// persistence serial queue, so on a heavily mixed wallet it stalls every +/// Rust persister callback and the main thread behind them until it finishes: +/// a watchdog kill, and no export artifact, on exactly the wallet support asked +/// about. A fetch limit is the wrong tool because a truncated table silently +/// misclassifies. Counting first and declining above a ceiling keeps the +/// distinction exact wherever it is computable and refuses honestly where it +/// is not. The figures are a cap on materialized objects, not a tuned number. +struct CoreDiagnosticRowLimits: Sendable { + /// Above this many `PersistentTxo` rows table-wide, only this wallet's rows + /// are fetched and the cross-wallet audit is declined. + let crossWalletTxoRows: Int + /// Above this many `PersistentTransaction` rows table-wide, transaction + /// bodies are not materialized and the exact audit is declined. + let exactAuditTransactionRows: Int + + /// The transaction ceiling is the one that matters: `walletOwnsTransaction` + /// faults four relationships per transaction cross-wallet, each a query + /// under the coordinator lock, so it — not decoding — dominates the time + /// the persistence queue is held. + static let production = CoreDiagnosticRowLimits( + crossWalletTxoRows: 100_000, + exactAuditTransactionRows: 10_000 + ) +} + +private extension Data { + mutating func appendLittleEndian(_ value: T) { + var littleEndian = value.littleEndian + Swift.withUnsafeBytes(of: &littleEndian) { append(contentsOf: $0) } + } +} + +/// Adds one diagnostic value without allowing corrupt data to trap the +/// exporter. The single saturating rule every diagnostic total shares. +func diagnosticSaturatingAdd(_ partial: UInt64, _ value: UInt64) -> UInt64 { + let (sum, overflow) = partial.addingReportingOverflow(value) + return overflow ? UInt64.max : sum +} + +/// Adds diagnostic values without allowing corrupt data to trap the exporter. +func diagnosticSaturatingSum(_ values: S) -> UInt64 +where S.Element == UInt64 { + values.reduce(0, diagnosticSaturatingAdd) +} + +private func diagnosticSignedSaturatingSum(_ values: S) -> Int64 +where S.Element == Int64 { + values.reduce(0) { partial, value in + let (sum, overflow) = partial.addingReportingOverflow(value) + if !overflow { return sum } + return value >= 0 ? Int64.max : Int64.min + } +} + +/// Hashes length-delimited canonical records after sorting, making the result +/// stable across SwiftData/Rust iteration order without logging raw records. +func diagnosticFingerprint(_ records: [Data]) -> Data { + var hasher = SHA256() + for record in records.sorted(by: { $0.lexicographicallyPrecedes($1) }) { + var length = UInt64(record.count).littleEndian + Swift.withUnsafeBytes(of: &length) { hasher.update(bufferPointer: $0) } + hasher.update(data: record) + } + return Data(hasher.finalize()) +} + +/// Canonical binary representation of every TXO field compared by the +/// database↔memory analyzer. The caller hashes this value before logging it. +func diagnosticTxoFingerprint( + outpoint: Data, + amount: UInt64, + height: UInt32, + scriptPubKey: Data, + isLocked: Bool, + account: CoreWalletDatabaseDiagnosticSnapshot.AccountKey? +) -> Data { + var data = Data() + data.appendLittleEndian(UInt64(outpoint.count)) + data.append(outpoint) + data.appendLittleEndian(amount) + data.appendLittleEndian(height) + data.append(isLocked ? 1 : 0) + data.appendLittleEndian(UInt64(scriptPubKey.count)) + data.append(scriptPubKey) + if let account { + data.append(1) + data.appendLittleEndian(UInt64(account.referenceMaterial.count)) + data.append(account.referenceMaterial) + } else { + data.append(0) + } + return data +} + +extension PlatformWalletPersistenceHandler { + /// Main-actor-friendly entry point used by manual log export. The handler's + /// serial queue owns the ModelContext; only a Sendable value snapshot is + /// resumed across the continuation. + func emitCoreWalletDatabaseDiagnostics( + walletId: Data, + limits: CoreDiagnosticRowLimits = .production, + cancellation: CoreDiagnosticsCancellation? = nil + ) async -> CoreWalletDatabaseDiagnosticSnapshot? { + await withCheckedContinuation { continuation in + serialQueue.async { [self] in + let snapshot = autoreleasepool { () -> CoreWalletDatabaseDiagnosticSnapshot? in + // A scratch context, still on the serial queue. Two things + // the handler's own context could not give: it sees only + // COMMITTED state — a Rust `store()` round is one changeset + // spread across several separate `sync` blocks, and this + // block can land between two of them, where the handler's + // context holds pending rows that `endChangeset` may still + // roll back — and it is dropped with this block, so the up + // to ~110k objects the pass registers do not stay resident + // for the life of the process. The queue still guarantees + // no save lands mid-pass. + let context = ModelContext(modelContainer) + context.autosaveEnabled = false + return emitCoreWalletDatabaseDiagnosticsOnQueue( + walletId: walletId, + context: context, + limits: limits, + cancellation: cancellation + ) + } + continuation.resume(returning: snapshot) + } + } + } + + /// Queue-confined implementation behind the async export API. Callers must + /// already own `serialQueue` and hand in a context confined to it; it + /// performs the full exact audit whenever the tables fit under `limits` + /// and returns only Sendable value copies. Between its stages it asks + /// `cancellation` whether shutdown has begun and, if so, says what it + /// skipped and stops — the drain in `shutdown()` covers this pass, and + /// must never wait for a whole cross-wallet scan. + @discardableResult + func emitCoreWalletDatabaseDiagnosticsOnQueue( + walletId: Data, + context: ModelContext, + limits: CoreDiagnosticRowLimits = .production, + cancellation: CoreDiagnosticsCancellation? = nil + ) -> CoreWalletDatabaseDiagnosticSnapshot? { + // This whole pass is export-only: the launch restore path takes + // `logCoreRestoreBufferSnapshotOnQueue` and never comes here, so the + // checkpoint every event below carries is a constant, not a parameter. + let checkpoint = CoreWalletDiagnosticCheckpoint.preExport + func shutdownBegan(before stage: String) -> Bool { + guard let cancellation, cancellation.isCancelled else { return false } + SDKLogger.event( + "core_diagnostics_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("shutdown_requested"), + "skipped_from_stage": .publicText(stage), + "wallet_reference": .reference(walletId), + ] + ) + return true + } + do { + let walletDescriptor = FetchDescriptor( + predicate: PersistentWallet.predicate(walletId: walletId) + ) + guard let wallet = try context.fetch(walletDescriptor).first else { + SDKLogger.event( + "core_diagnostics_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("wallet_not_found"), + "wallet_reference": .reference(walletId), + ] + ) + return nil + } + + // Exact #4438 classification needs a complete cross-wallet pass: + // an output absent from this wallet may be `wrong_wallet`, not + // `missing_txo`. Count before materializing (see + // `CoreDiagnosticRowLimits`): under the ceiling the whole table is + // held and the audit is exact; over it only this wallet's rows are + // fetched, and the snapshot says so, because a relationship-only + // row or a cross-wallet duplicate is then invisible to it. A + // streaming pass would lift the ceiling without losing the + // distinction and remains the follow-up. + if shutdownBegan(before: "txo_fetch") { return nil } + let txoRowCount = try context.fetchCount(FetchDescriptor()) + let crossWalletTxoScan = txoRowCount <= limits.crossWalletTxoRows + let allTxos: [PersistentTxo] + if crossWalletTxoScan { + allTxos = try context.fetch(FetchDescriptor()) + } else { + allTxos = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + } + let walletTxos = allTxos.filter { + $0.walletId == walletId || Self.relationshipWalletId(of: $0) == walletId + } + if shutdownBegan(before: "transaction_fetch") { return nil } + // Walking every transaction relationship is deliberately export-only. + // A heavily mixed wallet can have enough history for this traversal to + // stall restore, which is precisely the failure this instrumentation is + // intended to diagnose rather than reproduce. + let allTransactions: [PersistentTransaction]? + let walletTransactions: [PersistentTransaction]? + do { + let transactionRowCount = try context.fetchCount( + FetchDescriptor() + ) + // Both tables must fit: the audit resolves each decoded + // output against `txoByOutpoint`, so a wallet-only TXO + // scan would turn every foreign row into `missing_txo`. + if crossWalletTxoScan, + transactionRowCount <= limits.exactAuditTransactionRows { + allTransactions = try context.fetch( + FetchDescriptor() + ) + // Through the accounts' inverse relationship, not + // `walletOwnsTransaction` over every row: that faults four + // relationships per transaction and dominated the time the + // queue is held, to feed two counts. This is the + // `involvedAccounts` route only — a row tied to the wallet + // solely through a TXO is not counted, which the field + // names (`involved_…`) say. + var seen = Set() + walletTransactions = wallet.accounts + .flatMap(\.involvedTransactions) + .filter { seen.insert(ObjectIdentifier($0)).inserted } + } else { + allTransactions = nil + walletTransactions = nil + SDKLogger.event( + "core_owned_output_audit_summary", + category: .persistence, + severity: .warning, + fields: [ + "audit_incomplete": .boolean(true), + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("tables_too_large_for_exact_audit"), + "transaction_row_count": .integer(Int64(transactionRowCount)), + "transaction_row_limit": .integer( + Int64(limits.exactAuditTransactionRows) + ), + "txo_row_count": .integer(Int64(txoRowCount)), + "txo_row_limit": .integer(Int64(limits.crossWalletTxoRows)), + "wallet_reference": .reference(walletId), + ] + ) + } + } catch { + allTransactions = nil + walletTransactions = nil + SDKLogger.event( + "core_owned_output_audit_summary", + category: .persistence, + severity: .warning, + fields: [ + "audit_incomplete": .boolean(true), + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("transaction_fetch_failed"), + "wallet_reference": .reference(walletId), + ] + ) + } + let pending: [PersistentPendingInput]? + do { + pending = try context.fetch( + FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + ) + ) + } catch { + pending = nil + } + + let confirmed = walletTxos.filter(\.isConfirmed) + let unconfirmed = walletTxos.filter { !$0.isConfirmed } + let spent = walletTxos.filter(\.isSpent) + let unspent = walletTxos.filter { !$0.isSpent } + let locked = walletTxos.filter(\.isLocked) + // Every stage from here on is O(rows) or O(accounts × rows) on its + // own, so the drain's documented "at most one stage in flight" only + // holds if each is gated. The cost is one atomic read per stage. + if shutdownBegan(before: "wallet_fingerprint") { return nil } + let txoFingerprint = diagnosticFingerprint(walletTxos.map { + diagnosticTxoFingerprint( + outpoint: $0.outpoint, + amount: $0.amount, + height: $0.height, + scriptPubKey: $0.scriptPubKey, + isLocked: $0.isLocked, + account: Self.diagnosticAccountKey($0.account) + ) + }) + let now = Date() + let oldestPendingAge: Int64 + if let pending { + oldestPendingAge = pending.compactMap { row -> Int64? in + let interval = now.timeIntervalSince(row.createdAt) + guard interval.isFinite else { return nil } + if interval <= 0 { return 0 } + if interval >= Double(Int64.max) { return Int64.max } + return Int64(interval) + }.max() ?? 0 + } else { + oldestPendingAge = -1 + } + + SDKLogger.event( + "core_db_wallet_snapshot", + category: .persistence, + fields: [ + "account_count": .integer(Int64(wallet.accounts.count)), + "birth_height": .unsignedInteger(UInt64(wallet.birthHeight)), + "checkpoint": .publicText(checkpoint.rawValue), + "confirmed_count": .integer(Int64(confirmed.count)), + "confirmed_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(confirmed.map(\.amount)) + ), + "locked_count": .integer(Int64(locked.count)), + "locked_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(locked.map(\.amount)) + ), + "oldest_pending_input_age_seconds": .integer(oldestPendingAge), + "pending_input_count": .integer(pending.map { Int64($0.count) } ?? -1), + "pending_query_available": .boolean(pending != nil), + "spent_count": .integer(Int64(spent.count)), + "spent_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(spent.map(\.amount)) + ), + "synced_height": .unsignedInteger(UInt64(wallet.syncedHeight)), + "involved_transaction_count": .integer( + walletTransactions.map { Int64($0.count) } ?? -1 + ), + "transaction_scan_available": .boolean(walletTransactions != nil), + "txo_count": .integer(Int64(walletTxos.count)), + "txo_fingerprint": .reference(txoFingerprint), + "txo_row_count": .integer(Int64(txoRowCount)), + "txo_scan_scope": .publicText( + crossWalletTxoScan ? "cross_wallet" : "wallet_id_only" + ), + "unconfirmed_count": .integer(Int64(unconfirmed.count)), + "unconfirmed_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(unconfirmed.map(\.amount)) + ), + "unspent_count": .integer(Int64(unspent.count)), + "unspent_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(unspent.map(\.amount)) + ), + "wallet_reference": .reference(walletId), + ] + ) + + let sortedAccounts = wallet.accounts.sorted(by: Self.accountOrder) + // One pass over the wallet's rows, grouped by account, instead of an + // identity filter per account (O(accounts × rows)) followed by five + // more filters per account — all on the held persistence queue. + struct AccountTally { + var spentCount = 0, spentValue: UInt64 = 0 + var unspentCount = 0, unspentValue: UInt64 = 0 + var confirmedCount = 0, confirmedValue: UInt64 = 0 + var unconfirmedCount = 0, unconfirmedValue: UInt64 = 0 + var lockedCount = 0, lockedValue: UInt64 = 0 + var fingerprintMaterial: [Data] = [] + } + var tallies: [ObjectIdentifier: AccountTally] = [:] + var accountKeys: [ObjectIdentifier: CoreWalletDatabaseDiagnosticSnapshot.AccountKey] = [:] + for txo in walletTxos { + // Rows without an account are `missing_account` anomalies, + // reported by `logTxoAnomalies`; no account snapshot owns them. + guard let account = txo.account else { continue } + let id = ObjectIdentifier(account) + let key: CoreWalletDatabaseDiagnosticSnapshot.AccountKey + if let known = accountKeys[id] { + key = known + } else { + key = Self.diagnosticAccountKey(account)! + accountKeys[id] = key + } + var tally = tallies[id, default: AccountTally()] + if txo.isSpent { + tally.spentCount += 1 + tally.spentValue = diagnosticSaturatingAdd(tally.spentValue, txo.amount) + } else { + tally.unspentCount += 1 + tally.unspentValue = diagnosticSaturatingAdd(tally.unspentValue, txo.amount) + } + if txo.isConfirmed { + tally.confirmedCount += 1 + tally.confirmedValue = diagnosticSaturatingAdd(tally.confirmedValue, txo.amount) + } else { + tally.unconfirmedCount += 1 + tally.unconfirmedValue = diagnosticSaturatingAdd(tally.unconfirmedValue, txo.amount) + } + if txo.isLocked { + tally.lockedCount += 1 + tally.lockedValue = diagnosticSaturatingAdd(tally.lockedValue, txo.amount) + } + tally.fingerprintMaterial.append(diagnosticTxoFingerprint( + outpoint: txo.outpoint, + amount: txo.amount, + height: txo.height, + scriptPubKey: txo.scriptPubKey, + isLocked: txo.isLocked, + account: key + )) + tallies[id] = tally + } + + if shutdownBegan(before: "account_snapshots") { return nil } + for account in sortedAccounts { + let key = Self.diagnosticAccountKey(account)! + let tally = tallies[ObjectIdentifier(account)] ?? AccountTally() + var externalAddressCount = 0 + var internalAddressCount = 0 + var usedAddressCount = 0 + for address in account.coreAddresses { + if address.poolTypeTag == 0 { externalAddressCount += 1 } + if address.poolTypeTag == 1 { internalAddressCount += 1 } + if address.isUsed { usedAddressCount += 1 } + } + SDKLogger.event( + "core_db_account_snapshot", + category: .persistence, + fields: [ + "account_index": .unsignedInteger(UInt64(account.accountIndex)), + "account_reference": .reference(key.referenceMaterial), + "account_type": .unsignedInteger(UInt64(account.accountType)), + "checkpoint": .publicText(checkpoint.rawValue), + "confirmed_count": .integer(Int64(tally.confirmedCount)), + "confirmed_value_duffs": .unsignedInteger(tally.confirmedValue), + "external_address_count": .integer(Int64(externalAddressCount)), + "external_highest_used": .integer(Int64(account.externalHighestUsed)), + "internal_address_count": .integer(Int64(internalAddressCount)), + "internal_highest_used": .integer(Int64(account.internalHighestUsed)), + "locked_count": .integer(Int64(tally.lockedCount)), + "locked_value_duffs": .unsignedInteger(tally.lockedValue), + "registration_index": .unsignedInteger(UInt64(account.registrationIndex)), + "spent_count": .integer(Int64(tally.spentCount)), + "spent_value_duffs": .unsignedInteger(tally.spentValue), + "standard_tag": .unsignedInteger(UInt64(account.standardTag)), + "txo_fingerprint": .reference(diagnosticFingerprint(tally.fingerprintMaterial)), + "unconfirmed_count": .integer(Int64(tally.unconfirmedCount)), + "unconfirmed_value_duffs": .unsignedInteger(tally.unconfirmedValue), + "unspent_count": .integer(Int64(tally.unspentCount)), + "unspent_value_duffs": .unsignedInteger(tally.unspentValue), + "used_address_count": .integer(Int64(usedAddressCount)), + "wallet_reference": .reference(walletId), + ] + ) + } + + // Faults `transaction`, `account.wallet` and `spendingTransaction` + // for every row it inspects — the single most expensive stage after + // the audit itself. + if shutdownBegan(before: "txo_anomalies") { return nil } + Self.logTxoAnomalies( + walletId: walletId, + checkpoint: checkpoint, + txos: walletTxos + ) + // Decoding a heavily mixed wallet's full transaction history can + // be expensive. The exact #4438 audit is needed for the manually + // exported artifact, not for restoring Rust, so keep startup's + // persistence queue limited to lightweight summaries. + // Not nested inside `if let allTransactions`: the stages that + // follow run whether or not the audit was declined for table size, + // so gating the check on the audit's input skipped it exactly when + // the pass was already refusing to do the cheap thing. + if shutdownBegan(before: "owned_output_audit") { + if let allTransactions { + SDKLogger.event( + "core_owned_output_audit_summary", + category: .persistence, + severity: .warning, + fields: [ + "audit_incomplete": .boolean(true), + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("shutdown_requested"), + "transaction_row_count": .integer(Int64(allTransactions.count)), + "wallet_reference": .reference(walletId), + ] + ) + } + return nil + } + if let allTransactions { + let gaveUp = Self.auditCoinJoinOwnedBip44Outputs( + wallet: wallet, + walletId: walletId, + checkpoint: checkpoint, + allTxos: allTxos, + allTransactions: allTransactions, + isCancelled: { cancellation?.isCancelled == true } + ) + // It stopped mid-loop rather than between stages, so the rest + // of the pass is abandoned the same way every other stage + // abandons it — the summary it did not emit is the signal. + if gaveUp, shutdownBegan(before: "owned_output_audit_interrupted") { + return nil + } + } + + if shutdownBegan(before: "asset_lock_snapshot") { return nil } + let assetLocks: [CoreWalletDatabaseDiagnosticSnapshot.AssetLock] + let assetLocksAvailable: Bool + do { + assetLocks = try Self.logAssetLockDatabaseSnapshot( + context: context, + walletId: walletId, + checkpoint: checkpoint, + walletTransactions: walletTransactions + ) + assetLocksAvailable = true + } catch { + assetLocks = [] + assetLocksAvailable = false + SDKLogger.event( + "asset_lock_db_snapshot", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "query_available": .boolean(false), + "wallet_reference": .reference(walletId), + ] + ) + } + if shutdownBegan(before: "shielded_snapshot") { return nil } + do { + try Self.logShieldedStoreSnapshot( + context: context, + walletId: walletId, + checkpoint: checkpoint + ) + } catch { + SDKLogger.event( + "shielded_store_snapshot", + category: .shielded, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "query_available": .boolean(false), + "wallet_reference": .reference(walletId), + ] + ) + } + + let snapshot = CoreWalletDatabaseDiagnosticSnapshot( + walletId: walletId, + accounts: sortedAccounts.compactMap(Self.diagnosticAccountKey), + unspentTxos: unspent.map { + CoreWalletDatabaseDiagnosticSnapshot.Txo( + outpoint: $0.outpoint, + amount: $0.amount, + height: $0.height, + scriptPubKey: $0.scriptPubKey, + isLocked: $0.isLocked, + account: Self.diagnosticAccountKey($0.account) + ) + }, + assetLocks: assetLocks, + assetLocksAvailable: assetLocksAvailable + ) + return snapshot + } catch { + SDKLogger.event( + "core_diagnostics_unavailable", + category: .persistence, + severity: .error, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("swiftdata_fetch_failed"), + "wallet_reference": .reference(walletId), + ] + ) + return nil + } + } + + /// Logs the exact UTXO slice handed to Rust, independently of the broader + /// database snapshot. This sits after compact-write, so `emitted_count` + /// cannot be confused with the number of fetched candidates. + /// + /// - Parameters: + /// - rows: the rows the restore buffer was built from, in build order. + /// - accountLessRows: rows the restore fetch matched to this wallet by its + /// denormalized id but which carry no account, so they never reached FFI + /// marshalling. Passed separately rather than merged upstream: they are + /// rare, and keeping them out of `rows` avoids a second full-length copy + /// of every unspent row on the launch path. + func logCoreRestoreBufferSnapshotOnQueue( + walletId: Data, + rows: [PersistentTxo], + accountLessRows: [PersistentTxo] = [], + emittedCount: Int, + errored: Bool + ) { + func candidate(_ row: PersistentTxo) + -> CoreWalletDiagnosticAnalyzer.RestoreCandidate { + let rejection: CoreWalletDiagnosticAnalyzer.RestoreCandidate.RejectionReason? + if row.account == nil { + rejection = .missingAccount + } else if row.txid.count != 32 { + rejection = .invalidTxid + } else if let account = row.account, + UInt8(exactly: account.accountType) == nil { + rejection = .invalidAccountType + } else { + rejection = nil + } + return CoreWalletDiagnosticAnalyzer.RestoreCandidate( + amount: row.amount, + accountType: row.account?.accountType, + standardTag: row.account?.standardTag, + rejectionReason: rejection + ) + } + // A validation error deallocates the compact buffer and aborts the + // whole callback, so zero rows were actually handed to Rust even if + // some valid rows preceded the corrupt one. + // + // It also means classifying the rows would cost more than it is worth: + // `buildUtxoRestoreBuffer` can bail at row 0 having faulted nothing, + // and `candidate` touches `account` and `txid` — to-one relationships + // — on every row, so the walk would issue a fault per row, at launch, + // with the persistence queue held, to describe a load that is about to + // be discarded. The row that failed is already named with its reason + // by `persistence_wallet_load_validation_failed`; here the count is + // what is left to say, and the positional emission window the walk + // exists for is moot at zero emitted. + let summary: CoreWalletDiagnosticAnalyzer.RestoreBufferSummary + if errored { + summary = CoreWalletDiagnosticAnalyzer.summarizeRestoreBuffer( + candidates: EmptyCollection< + CoreWalletDiagnosticAnalyzer.RestoreCandidate + >(), + emittedCount: 0, + errored: true, + candidateCountOverride: rows.count + accountLessRows.count + ) + } else { + // Lazily, and with the built rows first: the summary's emission + // window is positional, so the rejected rows must not shift it. + // Nothing here materializes a per-row array — this runs while the + // launch restore holds the persistence queue, and every row it + // touches was already faulted by the build it is reconciling. + summary = CoreWalletDiagnosticAnalyzer.summarizeRestoreBuffer( + candidates: [rows, accountLessRows].lazy.flatMap { $0 }.map(candidate), + emittedCount: emittedCount, + errored: false + ) + } + let hasRejectedRows = summary.missingAccountCount > 0 + || summary.invalidTxidCount > 0 + || summary.invalidAccountTypeCount > 0 + + SDKLogger.event( + "core_restore_buffer_snapshot", + category: .persistence, + severity: errored ? .error : (hasRejectedRows ? .warning : .info), + fields: [ + "candidate_count": .integer(Int64(summary.candidateCount)), + "candidate_bip44_count": .integer(Int64(summary.candidateBip44Count)), + "candidate_bip44_value_duffs": .unsignedInteger( + summary.candidateBip44ValueDuffs + ), + "candidate_coinjoin_count": .integer(Int64(summary.candidateCoinJoinCount)), + "candidate_coinjoin_value_duffs": .unsignedInteger( + summary.candidateCoinJoinValueDuffs + ), + "candidate_value_duffs": .unsignedInteger(summary.candidateValueDuffs), + "built_count": .integer(Int64(summary.builtCount)), + "checkpoint": .publicText(CoreWalletDiagnosticCheckpoint.restoreBuffer.rawValue), + "emitted_count": .integer(Int64(summary.emittedCount)), + "emitted_bip44_count": .integer(Int64(summary.emittedBip44Count)), + "emitted_bip44_value_duffs": .unsignedInteger( + summary.emittedBip44ValueDuffs + ), + "emitted_coinjoin_count": .integer(Int64(summary.emittedCoinJoinCount)), + "emitted_coinjoin_value_duffs": .unsignedInteger( + summary.emittedCoinJoinValueDuffs + ), + "emitted_value_duffs": .unsignedInteger(summary.emittedValueDuffs), + "errored": .boolean(errored), + "skipped_invalid_account_type_count": .integer( + Int64(summary.invalidAccountTypeCount) + ), + "skipped_invalid_txid_count": .integer(Int64(summary.invalidTxidCount)), + "skipped_missing_account_count": .integer(Int64(summary.missingAccountCount)), + "wallet_reference": .reference(walletId), + ] + ) + } + + private static func diagnosticAccountKey( + _ account: PersistentAccount? + ) -> CoreWalletDatabaseDiagnosticSnapshot.AccountKey? { + guard let account else { return nil } + return CoreWalletDatabaseDiagnosticSnapshot.AccountKey( + typeTag: account.accountType, + standardTag: account.standardTag, + index: account.accountIndex, + registrationIndex: account.registrationIndex, + keyClass: account.keyClass, + userIdentityId: account.userIdentityId, + friendIdentityId: account.friendIdentityId + ) + } + + /// The one ordering every per-account pass uses, so anything that picks + /// "the first account" picks the same one on every export. + private static func accountOrder(_ lhs: PersistentAccount, _ rhs: PersistentAccount) -> Bool { + (lhs.accountType, lhs.standardTag, lhs.accountIndex, lhs.registrationIndex, lhs.keyClass) + < (rhs.accountType, rhs.standardTag, rhs.accountIndex, rhs.registrationIndex, rhs.keyClass) + } + + /// Read the relationship-owned wallet independently of the denormalized + /// `PersistentTxo.walletId`. Diagnostics must compare the two sources; + /// `resolvedWalletId(of:)` deliberately prefers the denormalized value and + /// would therefore hide exactly the corruption we are trying to expose. + private static func relationshipWalletId(of txo: PersistentTxo) -> Data? { + let account: PersistentAccount? = txo.account + guard let account else { return nil } + let wallet: PersistentWallet? = account.wallet + return wallet?.walletId + } + + private static func logTxoAnomalies( + walletId: Data, + checkpoint: CoreWalletDiagnosticCheckpoint, + txos: [PersistentTxo] + ) { + let result = CoreWalletDiagnosticAnalyzer.databaseTxoAnomalies(txos.map { txo in + let relationshipWalletId = relationshipWalletId(of: txo) + return CoreWalletDiagnosticAnalyzer.DatabaseTxoAuditRow( + txo: CoreWalletDatabaseDiagnosticSnapshot.Txo( + outpoint: txo.outpoint, + amount: txo.amount, + height: txo.height, + scriptPubKey: txo.scriptPubKey, + isLocked: txo.isLocked, + account: diagnosticAccountKey(txo.account) + ), + hasParentTransaction: txo.transaction != nil, + walletIdMismatch: !txo.walletId.isEmpty + && relationshipWalletId != nil + && txo.walletId != relationshipWalletId, + isSpent: txo.isSpent, + hasSpendingTransaction: txo.spendingTransaction != nil, + spendingTransactionIsInBlock: txo.spendingTransaction.map(spendIsInBlock) + ) + }) + SDKLogger.event( + "core_db_anomaly_summary", + category: .persistence, + severity: result.details.isEmpty ? .info : .warning, + fields: [ + "anomaly_count": .integer(Int64(result.details.count)), + "checkpoint": .publicText(checkpoint.rawValue), + "detail_count": .integer(Int64(result.emittedDetails.count)), + "empty_script_count": .integer(Int64(result.count(reason: "empty_script_pubkey"))), + "invalid_outpoint_count": .integer(Int64( + result.count(reason: "invalid_outpoint_length") + )), + "missing_account_count": .integer(Int64(result.count(reason: "missing_account"))), + "missing_parent_transaction_count": .integer(Int64( + result.count(reason: "missing_parent_transaction") + )), + "spent_relation_mismatch_count": .integer(Int64( + result.count(reason: "spent_without_spending_transaction") + + result.count(reason: "unspent_with_confirmed_spending_transaction") + )), + "truncated_count": .integer(Int64(result.truncatedCount)), + "wallet_mismatch_count": .integer(Int64( + result.count(reason: "wallet_id_mismatch") + )), + "wallet_reference": .reference(walletId), + ] + ) + for detail in result.emittedDetails { + SDKLogger.event( + "core_db_txo_anomaly", + category: .persistence, + severity: .warning, + fields: [ + "amount_duffs": .unsignedInteger(detail.txo.amount), + "checkpoint": .publicText(checkpoint.rawValue), + "height": .unsignedInteger(UInt64(detail.txo.height)), + "outpoint_reference": .reference(detail.txo.outpoint), + "reason": .publicText(detail.reason), + "wallet_reference": .reference(walletId), + ] + ) + } + } + + /// Exact detector for dashpay/platform#4438. It does not trust the + /// transaction's persisted role: it decodes inputs, proves at least one + /// spends a known CoinJoin TXO, then checks every decoded output against + /// the persisted BIP44 address pool and the TXO table. + /// + /// Ownership is decided by the persisted `PersistentCoreAddress` pool, so + /// `coinjoin_to_bip44_missing_count == 0` proves only that every output the + /// audit could attribute is persisted. An output beyond the derived pool, + /// or one whose address row was never written, is attributable to nobody + /// and lands in `unattributed_output_count` instead — which is why the + /// summary reports that counter and the pool size next to the verdict. + /// - Parameter isCancelled: polled inside the long loops, not only around + /// them. This is the most expensive stage in the pass — up to + /// `exactAuditTransactionRows` decodes plus per-output relationship work + /// — so a `shutdown()` that arrives mid-loop must not wait for the whole + /// of it. Returns true if the audit gave up, and the caller says so. + private static func auditCoinJoinOwnedBip44Outputs( + wallet: PersistentWallet, + walletId: Data, + checkpoint: CoreWalletDiagnosticCheckpoint, + allTxos: [PersistentTxo], + allTransactions: [PersistentTransaction], + isCancelled: () -> Bool + ) -> Bool { + // Match the wallet exactly as `walletTxos` does. Accepting only the + // relationship would drop a CoinJoin row whose `account.wallet` link is + // broken — the very corruption this audit exists to expose — and its + // spending transaction would never even become a candidate. + let coinJoinOutpoints = Set(allTxos.compactMap { txo -> Data? in + guard txo.account?.accountType == 1, + txo.walletId == walletId || relationshipWalletId(of: txo) == walletId + else { return nil } + return txo.outpoint + }) + // `wallet.accounts` is unordered. Two BIP44 accounts holding a row for + // the same address would otherwise make `expectedAccount` — and so + // `wrong_account` — depend on which faulted first; the same ordering + // the account snapshots use makes the winner the same on every run. + // + // Both kinds of account we can own an output on. A mixed send consumes + // a CoinJoin output and typically pays CoinJoin change back to the + // wallet: that output has a persisted address row, so booking it as + // unattributed both overstated "paid to someone else" and hid the case + // where it is the CoinJoin-side output that went missing from + // `PersistentTxo` — a third route to the false all-clear. + // + // BIP44 first, so a shared address keeps the BIP44 attribution this + // audit has always given it. + var ownedAddresses: [String: PersistentAccount] = [:] + let bip44Accounts = wallet.accounts + .filter { $0.accountType == 0 && $0.standardTag == 0 } + .sorted(by: Self.accountOrder) + let coinJoinAccounts = wallet.accounts + .filter { $0.accountType == 1 } + .sorted(by: Self.accountOrder) + for account in bip44Accounts + coinJoinAccounts { + for coreAddress in account.coreAddresses where ownedAddresses[coreAddress.address] == nil { + ownedAddresses[coreAddress.address] = account + } + } + let bip44AddressCount = bip44Accounts.reduce(0) { $0 + $1.coreAddresses.count } + let coinJoinAddressCount = coinJoinAccounts.reduce(0) { $0 + $1.coreAddresses.count } + let txoByOutpoint = Dictionary(grouping: allTxos, by: \.outpoint) + + var candidateCount = 0 + var decodeFailureCount = 0 + var transactionBytesMissingCount = 0 + var ownedOutputCount = 0 + var ownedOutputValue: UInt64 = 0 + var ownedCoinJoinOutputCount = 0 + var ownedCoinJoinOutputValue: UInt64 = 0 + var unattributedOutputCount = 0 + var undecodableAddressOutputCount = 0 + var validCount = 0 + var anomalies: [(tx: PersistentTransaction, vout: UInt32, amount: UInt64, + outpoint: Data, reason: String, outputIsCoinJoin: Bool)] = [] + + guard let network = wallet.network else { + SDKLogger.event( + "core_owned_output_audit_summary", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("wallet_network_unknown"), + "wallet_reference": .reference(walletId), + ] + ) + return false + } + + for transaction in allTransactions { + if isCancelled() { return true } + // A stub row with no consensus bytes is a real production state + // (an orphaned upsert reads back as empty) — and the half-written + // persistence #4438 is about. It cannot be decoded, so it cannot + // become a candidate; it must be counted rather than skipped, or + // the export reports a clean wallet with `audit_incomplete=false`. + // Stubs are rare, so the ownership check is cheap here even though + // it is the expensive one. + if transaction.transactionData.isEmpty { + if walletOwnsTransaction(walletId: walletId, transaction: transaction) { + transactionBytesMissingCount += 1 + } + continue + } + let decoded: DecodedTransaction + do { + decoded = try TransactionDecoder.decode(transaction.transactionData, network: network) + } catch { + // Only count decode failures for rows already associated with + // this wallet; unrelated-wallet corruption must not pollute + // this wallet's audit result. + if walletOwnsTransaction(walletId: walletId, transaction: transaction) { + decodeFailureCount += 1 + } + continue + } + let spendsCoinJoin = decoded.inputs.contains { input in + coinJoinOutpoints.contains( + PersistentTxo.makeOutpoint(txid: input.prevTxid, vout: input.prevVout) + ) + } + guard spendsCoinJoin else { continue } + candidateCount += 1 + + for (index, output) in decoded.outputs.enumerated() { + if isCancelled() { return true } + guard let address = output.address else { + // Non-P2PKH/P2SH scriptPubKey: nothing to match against the + // address pool, so it is unclassified rather than foreign. + undecodableAddressOutputCount += 1 + continue + } + guard let expectedAccount = ownedAddresses[address] else { + // Either a genuine payment to someone else or one of our + // own change addresses with no persisted row. The audit + // cannot tell them apart, so it counts rather than clears. + unattributedOutputCount += 1 + continue + } + let outputIsCoinJoin = expectedAccount.accountType == 1 + if outputIsCoinJoin { + ownedCoinJoinOutputCount += 1 + ownedCoinJoinOutputValue = diagnosticSaturatingAdd( + ownedCoinJoinOutputValue, output.valueDuffs) + } else { + ownedOutputCount += 1 + ownedOutputValue = diagnosticSaturatingAdd(ownedOutputValue, output.valueDuffs) + } + let vout = UInt32(index) + let outpoint = PersistentTxo.makeOutpoint(txid: decoded.txid, vout: vout) + guard let row = representativeTxo( + rows: txoByOutpoint[outpoint], + walletId: walletId + ) else { + anomalies.append((transaction, vout, output.valueDuffs, outpoint, "missing_txo", outputIsCoinJoin)) + continue + } + // Same admission rule as `walletTxos` and the candidate set: + // the denormalized id OR the relationship may name this + // wallet. Judging by the relationship alone here would report + // this wallet's own row with a broken link as `wrong_wallet`. + let rowRelationshipWallet = relationshipWalletId(of: row) + let denormalizedNamesWallet = row.walletId == walletId + let relationshipNamesWallet = rowRelationshipWallet == walletId + guard denormalizedNamesWallet || relationshipNamesWallet else { + anomalies.append((transaction, vout, output.valueDuffs, outpoint, "wrong_wallet", outputIsCoinJoin)) + continue + } + guard relationshipNamesWallet else { + // Ours by id, but the relationship says otherwise. Reuse + // the vocabulary `logTxoAnomalies` already emits for the + // same two facts, so the analyst sees one story. + let reason = rowRelationshipWallet == nil + ? "relationship_missing" : "wallet_id_mismatch" + anomalies.append((transaction, vout, output.valueDuffs, outpoint, reason, outputIsCoinJoin)) + continue + } + // Against the account the address pool named, not a hardcoded + // BIP44 shape: the same check now serves CoinJoin-owned + // outputs, whose account carries type 1. + guard row.account === expectedAccount, + row.account?.accountType == expectedAccount.accountType, + row.account?.standardTag == expectedAccount.standardTag + else { + anomalies.append((transaction, vout, output.valueDuffs, outpoint, "wrong_account", outputIsCoinJoin)) + continue + } + guard row.amount == output.valueDuffs else { + anomalies.append((transaction, vout, output.valueDuffs, outpoint, "amount_mismatch", outputIsCoinJoin)) + continue + } + guard row.scriptPubKey == output.scriptPubkey else { + anomalies.append((transaction, vout, output.valueDuffs, outpoint, "script_mismatch", outputIsCoinJoin)) + continue + } + validCount += 1 + } + } + + anomalies.sort { + if $0.outpoint != $1.outpoint { + return $0.outpoint.lexicographicallyPrecedes($1.outpoint) + } + return $0.reason < $1.reason + } + let anomalyGroups = Dictionary(grouping: anomalies, by: { $0.reason }) + let truncatedAnomalyCount = anomalyGroups.values.reduce(0) { + $0 + max(0, $1.count - CoreDiagnosticConstants.detailLimit) + } + let missing = anomalies.filter { $0.reason == "missing_txo" } + let missingCount = missing.filter { !$0.outputIsCoinJoin }.count + let missingValue = diagnosticSaturatingSum(missing.compactMap { + $0.outputIsCoinJoin ? nil : $0.amount + }) + let missingCoinJoinCount = missing.filter(\.outputIsCoinJoin).count + let missingCoinJoinValue = diagnosticSaturatingSum(missing.compactMap { + $0.outputIsCoinJoin ? $0.amount : nil + }) + // With no persisted BIP44 addresses nothing can be attributed to this + // wallet: every decoded output falls into `unattributedOutputCount`, + // no output reaches the `missing_txo` check, and the summary would + // otherwise read as a clean, complete audit — on a wallet whose + // address rows are exactly what went missing. The pool being empty is + // an incompleteness of the same kind as an undecodable transaction. + // + // `unattributedOutputCount > 0` is deliberately NOT part of this: a + // CoinJoin-spending transaction pays its peers, and their outputs are + // unattributable by construction, so every healthy audit has some. + // Since the pool now covers CoinJoin accounts too, that count is + // finally only peers and rows we genuinely lack — our own CoinJoin + // change no longer inflates it. + // A partially lost pool is not distinguishable from a small one here; + // `bip44_address_pool_size` sits beside this flag for that reading. + // Per pool, and symmetrically: an empty pool is evidence only where + // there are accounts that should have filled it. Judging BIP44 + // unconditionally called a CoinJoin-only wallet incomplete although + // attribution had worked — noise rather than danger, but noise in the + // one field an analyst uses to decide whether to keep reading. + // Nothing owned at all is incomplete however few accounts exist: + // no address means nothing can be attributed. + let addressPoolEmpty = ownedAddresses.isEmpty + || (!bip44Accounts.isEmpty && bip44AddressCount == 0) + || (!coinJoinAccounts.isEmpty && coinJoinAddressCount == 0) + let auditIncomplete = decodeFailureCount > 0 + || transactionBytesMissingCount > 0 + || addressPoolEmpty + SDKLogger.event( + "core_owned_output_audit_summary", + category: .persistence, + severity: anomalies.isEmpty && !auditIncomplete ? .info : .warning, + fields: [ + "audit_incomplete": .boolean(auditIncomplete), + "address_pool_empty": .boolean(addressPoolEmpty), + "bip44_address_pool_size": .integer(Int64(bip44AddressCount)), + "coinjoin_address_pool_size": .integer(Int64(coinJoinAddressCount)), + "coinjoin_to_coinjoin_missing_count": .integer(Int64(missingCoinJoinCount)), + "coinjoin_to_coinjoin_missing_value_duffs": .unsignedInteger(missingCoinJoinValue), + "candidate_transaction_count": .integer(Int64(candidateCount)), + "checkpoint": .publicText(checkpoint.rawValue), + "coinjoin_to_bip44_missing_count": .integer(Int64(missingCount)), + "coinjoin_to_bip44_missing_value_duffs": .unsignedInteger(missingValue), + "decode_failure_count": .integer(Int64(decodeFailureCount)), + "output_address_undecodable_count": .integer( + Int64(undecodableAddressOutputCount) + ), + "owned_bip44_output_count": .integer(Int64(ownedOutputCount)), + "owned_bip44_output_value_duffs": .unsignedInteger(ownedOutputValue), + "owned_coinjoin_output_count": .integer(Int64(ownedCoinJoinOutputCount)), + "owned_coinjoin_output_value_duffs": .unsignedInteger(ownedCoinJoinOutputValue), + "persisted_valid_count": .integer(Int64(validCount)), + "total_anomaly_count": .integer(Int64(anomalies.count)), + "transaction_bytes_missing_count": .integer(Int64(transactionBytesMissingCount)), + "truncated_count": .integer(Int64(truncatedAnomalyCount)), + "unattributed_output_count": .integer(Int64(unattributedOutputCount)), + "wallet_reference": .reference(walletId), + ] + ) + for reason in anomalyGroups.keys.sorted() { + for anomaly in (anomalyGroups[reason] ?? []).prefix(CoreDiagnosticConstants.detailLimit) { + SDKLogger.event( + "core_owned_output_anomaly", + category: .persistence, + severity: .warning, + fields: [ + "amount_duffs": .unsignedInteger(anomaly.amount), + "block_height": .unsignedInteger(UInt64(anomaly.tx.blockHeight)), + "checkpoint": .publicText(checkpoint.rawValue), + "input_account_kind": .publicText("coinjoin"), + "outpoint_reference": .reference(anomaly.outpoint), + "output_account_kind": .publicText( + anomaly.outputIsCoinJoin ? "coinjoin" : "bip44" + ), + "reason": .publicText(reason), + "transaction_context": .unsignedInteger(UInt64(anomaly.tx.context)), + "transaction_reference": .reference(anomaly.tx.txid), + "vout": .unsignedInteger(UInt64(anomaly.vout)), + "wallet_reference": .reference(walletId), + ] + ) + } + } + return false + } + + /// Picks the row that represents one outpoint when the table holds more + /// than one. + /// + /// A duplicated outpoint split across wallets is precisely the `wrong_wallet` + /// corruption this audit names, so the choice must not depend on SwiftData's + /// fetch order — the same database would otherwise report `wrong_wallet` on + /// one run and a clean count on the next. A row this wallet owns wins (the + /// output IS persisted here, whatever else shares the outpoint); otherwise a + /// deterministic representative is chosen the way `compareTxos` resolves + /// duplicates before comparing. + static func representativeTxo( + rows: [PersistentTxo]?, + walletId: Data + ) -> PersistentTxo? { + guard let rows, !rows.isEmpty else { return nil } + if rows.count == 1 { return rows[0] } + let ordered = rows.sorted { + duplicateResolutionKey($0).lexicographicallyPrecedes( + duplicateResolutionKey($1) + ) + } + return ordered.first { + $0.walletId == walletId || relationshipWalletId(of: $0) == walletId + } ?? ordered[0] + } + + /// Total order over rows sharing an outpoint. Uses only persisted bytes, so + /// two runs over the same database agree. + /// + /// A single `0` byte separates the two wallet ids. That is unambiguous only + /// because each is exactly 32 bytes or empty — a variable-width id would + /// need length prefixes, as `diagnosticTxoFingerprint` uses. + private static func duplicateResolutionKey(_ txo: PersistentTxo) -> Data { + var key = Data() + key.append(txo.walletId) + key.append(0) + key.append(relationshipWalletId(of: txo) ?? Data()) + key.append(0) + withUnsafeBytes(of: txo.amount.littleEndian) { key.append(contentsOf: $0) } + // Two rows ours by id with the same amount and script but different + // accounts (BIP44 vs CoinJoin) must not tie: `sorted` is not stable, + // so a tie would make `wrong_account` depend on fetch order. + if let account = diagnosticAccountKey(txo.account) { + key.append(1) + key.append(account.referenceMaterial) + } else { + key.append(0) + } + key.append(0) + key.append(txo.scriptPubKey) + return key + } + + private static func logAssetLockDatabaseSnapshot( + context: ModelContext, + walletId: Data, + checkpoint: CoreWalletDiagnosticCheckpoint, + walletTransactions: [PersistentTransaction]? + ) throws -> [CoreWalletDatabaseDiagnosticSnapshot.AssetLock] { + let rows = try context.fetch( + FetchDescriptor( + predicate: PersistentAssetLock.predicate(walletId: walletId) + ) + ) + SDKLogger.event( + "asset_lock_db_snapshot", + category: .persistence, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "involved_type_8_transaction_count": .integer( + walletTransactions.map { Int64($0.filter(\.isAssetLock).count) } ?? -1 + ), + "core_transaction_scan_available": .boolean(walletTransactions != nil), + "lock_count": .integer(Int64(rows.count)), + "proof_present_count": .integer(Int64(rows.filter { + $0.proofBytes?.isEmpty == false + }.count)), + "query_available": .boolean(true), + "shielded_funding_count": .integer(Int64(rows.filter { + $0.fundingTypeRaw == 5 + }.count)), + "transaction_bytes_present_count": .integer(Int64(rows.filter { + !$0.transactionBytes.isEmpty + }.count)), + "wallet_reference": .reference(walletId), + ] + ) + + let groups = Dictionary(grouping: rows) { + "\($0.fundingTypeRaw):\($0.statusRaw)" + } + for key in groups.keys.sorted() { + guard let group = groups[key], let first = group.first else { continue } + SDKLogger.event( + "asset_lock_db_group", + category: .persistence, + fields: [ + "amount_duffs": .integer( + diagnosticSignedSaturatingSum(group.map(\.amountDuffs)) + ), + "checkpoint": .publicText(checkpoint.rawValue), + "count": .integer(Int64(group.count)), + "funding_type": .integer(Int64(first.fundingTypeRaw)), + "status": .integer(Int64(first.statusRaw)), + "wallet_reference": .reference(walletId), + ] + ) + } + return rows.map { + CoreWalletDatabaseDiagnosticSnapshot.AssetLock( + outpointDisplay: $0.outPointHex, + fundingType: $0.fundingTypeRaw, + status: $0.statusRaw, + accountIndex: UInt32(bitPattern: $0.accountIndexRaw), + registrationIndex: UInt32(bitPattern: $0.identityIndexRaw), + amountDuffs: UInt64(exactly: $0.amountDuffs), + hasProof: $0.proofBytes?.isEmpty == false + ) + } + } + + private static func logShieldedStoreSnapshot( + context: ModelContext, + walletId: Data, + checkpoint: CoreWalletDiagnosticCheckpoint + ) throws { + let notes = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + let outgoing = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + let states = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + let activity = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + let viewingKeys = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.walletId == walletId } + )) + let summary = CoreWalletDiagnosticAnalyzer.summarizeShieldedStore( + notes: notes.map { .init(value: $0.value, isSpent: $0.isSpent) }, + outgoingNoteCount: outgoing.count, + activityStatuses: activity.map(\.status), + viewingKeyCount: viewingKeys.count, + syncWatermarks: states.map(\.lastSyncedIndex) + ) + SDKLogger.event( + "shielded_store_snapshot", + category: .shielded, + fields: [ + "activity_count": .integer(Int64(summary.activityCount)), + "activity_failed_count": .integer(Int64(summary.activityFailedCount)), + "activity_pending_count": .integer(Int64(summary.activityPendingCount)), + "checkpoint": .publicText(checkpoint.rawValue), + "maximum_sync_watermark": .unsignedInteger(summary.maximumSyncWatermark), + "note_count": .integer(Int64(summary.noteCount)), + "outgoing_note_count": .integer(Int64(summary.outgoingNoteCount)), + "query_available": .boolean(true), + "spent_note_count": .integer(Int64(summary.spentNoteCount)), + "spent_value_credits": .unsignedInteger(summary.spentValueCredits), + "subwallet_sync_state_count": .integer(Int64(summary.subwalletSyncStateCount)), + "unspent_note_count": .integer(Int64(summary.unspentNoteCount)), + "unspent_value_credits": .unsignedInteger(summary.unspentValueCredits), + "viewing_key_count": .integer(Int64(summary.viewingKeyCount)), + "wallet_reference": .reference(walletId), + ] + ) + } +} + +// MARK: - Rust memory comparison + +@MainActor +extension PlatformWalletManager { + /// Emit a best-effort, read-only snapshot immediately before a diagnostic + /// export. The method intentionally never throws: a failed sub-query is a + /// diagnostic fact and is logged as `unavailable`, not reported as zero. + /// + /// Cost, so hosts do not trigger this mid-sync: the SwiftData half runs on + /// the persistence serial queue and holds it for its whole duration, which + /// blocks every Rust persister and SPV callback (they enter through + /// `serialQueue.sync`) and any main-thread persistence access until it + /// returns. Under `CoreDiagnosticRowLimits` that includes materializing + /// the full TXO and transaction tables for the exact #4438 audit; above + /// them the export narrows to this wallet's rows, declines the audit, and + /// says so in `core_owned_output_audit_summary`. A paged variant that + /// lifts the ceilings without losing the classification is tracked as a + /// follow-up. + /// + /// Coordinates the queue-owned SwiftData snapshot with read-only Rust FFI + /// queries. Admission happens after the database await, then keeps the + /// native handle alive until the off-main worker finishes. + /// + /// No caller inside this repository, by design: the artifact this produces + /// is a support export, and the only screen that asks for one lives in the + /// host app (dashpay/dashwallet-ios#1105 wires Contact Support to it). + /// `SwiftExampleApp` deliberately does not — it has no support flow, and a + /// demo button would make a pass documented as holding the persistence + /// queue look like something to press casually. Everything under the + /// `preExport` checkpoint is therefore reachable only through a host; the + /// `restoreBuffer` summary and `core_store_open_result` are the parts that + /// run unprompted. + public func emitCoreWalletDiagnostics(for walletId: Data) async { + let checkpoint = CoreWalletDiagnosticCheckpoint.preExport + guard walletId.count == 32, let handler = persistence else { + SDKLogger.event( + "core_diagnostics_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("invalid_wallet_or_persistence_disabled"), + "wallet_reference": .reference(walletId), + ] + ) + return + } + let cancellation = coreDiagnosticsCancellation + // Counted for the whole pass, both halves, so a `shutdown()` that + // finds no handle can still tell a running database half to stop — + // and so one that finds neither leaves the latch down. + beginCoreDiagnosticsPass() + defer { endCoreDiagnosticsPass() } + + // The database half may come back empty — wallet row missing, fetch + // failed — and those are exactly the "coins gone from the database" + // reports this exists for. The Rust half needs only the wallet id, so + // it runs regardless and marks its diffs as one-sided. + // + // Deliberately OUTSIDE the native-op admission. That admission exists + // for one thing: keeping `handle` alive while an FFI read is in + // flight. This half makes no FFI call at all — it reads SwiftData on + // the persistence serial queue — so covering it buys the handle + // nothing and costs `shutdown()`'s drain everything: the drain would + // then wait on a block whose progress depends on that queue, and a + // wedged persister round (the failure this export exists to + // investigate) is exactly when the queue does not advance. There is + // no deadline on the drain, so that wait would be unbounded. + // + // Ordering is still safe without the drain, because it does not come + // from the drain: native teardown runs on `destroyQueue`, and the Rust + // destroy's persister callbacks enter through `serialQueue.sync`, so + // they queue BEHIND this block rather than racing it — off the main + // thread, and bounded by the cancellation flag this block polls + // between stages. ARC covers the rest: the block holds the handler and + // its container, so neither can be deallocated under the read. + let database = await handler.emitCoreWalletDatabaseDiagnostics( + walletId: walletId, + cancellation: cancellation + ) + + // Admission covers the FFI half only, which runs on + // `coreDiagnosticsQueue` and polls cancellation between reads — so the + // drain's "at most one stage in flight" is bounded by a stage this + // manager owns, not by whatever is holding the persistence queue. + guard isConfigured, handle != NULL_HANDLE else { + SDKLogger.event( + "core_memory_snapshot_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("manager_not_configured"), + "wallet_reference": .reference(walletId), + ] + ) + return + } + do { + try admitCoreDiagnosticsNativeOp() + } catch { + SDKLogger.event( + "core_memory_snapshot_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("manager_shutdown_in_progress"), + "wallet_reference": .reference(walletId), + ] + ) + return + } + defer { finishCoreDiagnosticsNativeOp() } + + let managerHandle = handle + let managedWallet = wallets[walletId] + await withCheckedContinuation { continuation in + Self.coreDiagnosticsQueue.async { + Self.emitCoreMemoryDiagnostics( + managerHandle: managerHandle, + managedWallet: managedWallet, + walletId: walletId, + database: database, + checkpoint: checkpoint, + cancellation: cancellation + ) + continuation.resume() + } + } + } + + /// Runs all Rust-memory reads on `coreDiagnosticsQueue`. Each subsystem + /// reports its own unavailable state so one failed query does not hide the + /// others. + private nonisolated static func emitCoreMemoryDiagnostics( + managerHandle: Handle, + managedWallet: ManagedPlatformWallet?, + walletId: Data, + database: CoreWalletDatabaseDiagnosticSnapshot?, + checkpoint: CoreWalletDiagnosticCheckpoint, + cancellation: CoreDiagnosticsCancellation + ) { + // Each FFI read below can park this thread on a Rust lock for as long + // as a wedged sync pass holds it, and `shutdown()` waits on this + // pass's admission. So before every read, ask whether shutdown has + // begun; if so, say which reads were skipped and let the drain go. + func shutdownBegan(before stage: String) -> Bool { + guard cancellation.isCancelled else { return false } + SDKLogger.event( + "core_memory_snapshot_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("shutdown_requested"), + "skipped_from_stage": .publicText(stage), + "wallet_reference": .reference(walletId), + ] + ) + return true + } + + // `compareDatabase` and `compareAssetLocks` both emit a + // `diff_incomplete=true` summary rather than nothing when their input + // is missing, because an absent summary is indistinguishable from a + // log that was cut off mid-export. Every early return out of this + // function owes the reader the same line — otherwise grepping + // `core_db_memory_diff_summary` on a failed balance read finds + // silence, which reads as truncation. + func emitAbandonedDiffSummary(reason: String) { + SDKLogger.event( + "core_db_memory_diff_summary", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "database_snapshot_available": .boolean(database != nil), + "diff_incomplete": .boolean(true), + "reason": .publicText(reason), + "wallet_reference": .reference(walletId), + ] + ) + } + + // Keep the two Rust-memory sources independent: corrupt account state + // must not suppress the AssetLock evidence that can explain a missing + // balance (and vice versa). + if shutdownBegan(before: "asset_locks") { + emitAbandonedDiffSummary(reason: "shutdown_requested") + return + } + compareAssetLocks( + database, + walletId: walletId, + managedWallet: managedWallet, + checkpoint: checkpoint + ) + if shutdownBegan(before: "account_balances") { + emitAbandonedDiffSummary(reason: "shutdown_requested") + return + } + let balanceQuery = readAccountBalances( + handle: managerHandle, + walletId: walletId + ) + guard case .success(let balances) = balanceQuery else { + SDKLogger.event( + "core_memory_snapshot_unavailable", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "reason": .publicText("account_balance_query_failed"), + "wallet_reference": .reference(walletId), + ] + ) + emitAbandonedDiffSummary(reason: "account_balance_query_failed") + return + } + + var memoryTxos: [CoreWalletDatabaseDiagnosticSnapshot.Txo] = [] + var unavailableAccounts: Set = [] + let sortedBalances = balances.sorted { + Self.diagnosticAccountKey($0).referenceMaterial.lexicographicallyPrecedes( + Self.diagnosticAccountKey($1).referenceMaterial + ) + } + for balance in sortedBalances { + if shutdownBegan(before: "account_utxos") { + emitAbandonedDiffSummary(reason: "shutdown_requested") + return + } + // One pool per account, matching `emitCoreWalletDatabaseDiagnostics`. + // libdispatch drains its own pool once per work item, and this + // whole loop is one work item: without this, every account's + // per-UTXO txid and scriptPubKey copies, its fingerprint material + // and the formatter each log event allocates all stay resident + // until the export ends, so the peak is the sum of every account + // rather than the largest one. `memoryTxos` is returned out of the + // pool on purpose — `compareDatabase` needs the whole set. + let accountTxos: [CoreWalletDatabaseDiagnosticSnapshot.Txo]? = autoreleasepool { + let key = Self.diagnosticAccountKey(balance) + let query = diagnosticAccountUtxos( + managerHandle: managerHandle, + walletId: walletId, + balance: balance + ) + guard case .success(let utxos) = query else { + unavailableAccounts.insert(key) + SDKLogger.event( + "core_memory_account_snapshot", + category: .persistence, + severity: .warning, + fields: [ + "account_reference": .reference(key.referenceMaterial), + "account_type": .unsignedInteger(UInt64(key.typeTag)), + "checkpoint": .publicText(checkpoint.rawValue), + "query_available": .boolean(false), + "wallet_reference": .reference(walletId), + ] + ) + return nil + } + let materials = utxos.map { + diagnosticTxoFingerprint( + outpoint: $0.outpoint, + amount: $0.amount, + height: $0.height, + scriptPubKey: $0.scriptPubKey, + isLocked: $0.isLocked, + account: key + ) + } + SDKLogger.event( + "core_memory_account_snapshot", + category: .persistence, + fields: [ + "account_index": .unsignedInteger(UInt64(balance.index)), + "account_reference": .reference(key.referenceMaterial), + "account_type": .unsignedInteger(UInt64(balance.typeTag)), + "checkpoint": .publicText(checkpoint.rawValue), + "confirmed_duffs": .unsignedInteger(balance.confirmed), + "immature_duffs": .unsignedInteger(balance.immature), + "locked_duffs": .unsignedInteger(balance.locked), + "query_available": .boolean(true), + "standard_tag": .unsignedInteger(UInt64(balance.standardTag)), + "unconfirmed_duffs": .unsignedInteger(balance.unconfirmed), + "utxo_count": .integer(Int64(utxos.count)), + "utxo_fingerprint": .reference(diagnosticFingerprint(materials)), + "utxo_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(utxos.map(\.amount)) + ), + "wallet_reference": .reference(walletId), + ] + ) + return utxos + } + if let accountTxos { memoryTxos.append(contentsOf: accountTxos) } + } + compareDatabase( + database, + walletId: walletId, + memoryTxos: memoryTxos, + memoryAccounts: Set(balances.map(Self.diagnosticAccountKey)), + unavailableAccounts: unavailableAccounts, + checkpoint: checkpoint + ) + } + + /// Logs the deterministic DB↔Rust UTXO diff, excluding accounts whose Rust + /// UTXO query failed instead of falsely reporting all their rows DB-only. + private nonisolated static func compareDatabase( + _ database: CoreWalletDatabaseDiagnosticSnapshot?, + walletId: Data, + memoryTxos: [CoreWalletDatabaseDiagnosticSnapshot.Txo], + memoryAccounts: Set, + unavailableAccounts: Set, + checkpoint: CoreWalletDiagnosticCheckpoint + ) { + guard let database else { + // No database side to diff against: say so, with the memory side's + // size, rather than emit nothing — an absent summary reads like a + // truncated log, and this is the case where Rust may still hold + // the funds the database lost. + SDKLogger.event( + "core_db_memory_diff_summary", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "database_snapshot_available": .boolean(false), + "diff_incomplete": .boolean(true), + "memory_account_count": .integer(Int64(memoryAccounts.count)), + "memory_txo_count": .integer(Int64(memoryTxos.count)), + "unavailable_account_count": .integer(Int64(unavailableAccounts.count)), + "wallet_reference": .reference(walletId), + ] + ) + return + } + let excludedDatabaseTxos = database.unspentTxos.filter { row in + row.account.map(unavailableAccounts.contains) ?? false + } + let comparableDatabaseTxos = database.unspentTxos.filter { row in + !(row.account.map(unavailableAccounts.contains) ?? false) + } + let result = CoreWalletDiagnosticAnalyzer.compareTxos( + database: comparableDatabaseTxos, + memory: memoryTxos, + databaseAccounts: Set(database.accounts), + memoryAccounts: memoryAccounts + ) + SDKLogger.event( + "core_db_memory_diff_summary", + category: .persistence, + severity: result.details.isEmpty + && result.databaseAccountOnlyCount == 0 + && result.memoryAccountOnlyCount == 0 + ? .info : .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "common_count": .integer(Int64(result.commonCount)), + "database_snapshot_available": .boolean(true), + "database_account_only_count": .integer( + Int64(result.databaseAccountOnlyCount) + ), + "database_only_count": .integer(Int64(result.databaseOnlyCount)), + "diff_incomplete": .boolean(!unavailableAccounts.isEmpty), + "excluded_database_txo_count": .integer(Int64(excludedDatabaseTxos.count)), + "field_mismatch_count": .integer(Int64(result.fieldMismatchCount)), + "memory_only_count": .integer(Int64(result.memoryOnlyCount)), + "memory_account_only_count": .integer(Int64(result.memoryAccountOnlyCount)), + "truncated_count": .integer(Int64(result.truncatedCount)), + "unavailable_account_count": .integer(Int64(unavailableAccounts.count)), + "wallet_reference": .reference(walletId), + ] + ) + for detail in result.emittedDetails { + logDiffItem( + walletId, + checkpoint, + detail.row, + detail.outpoint, + detail.reason + ) + } + } + + private nonisolated static func logDiffItem( + _ walletId: Data, + _ checkpoint: CoreWalletDiagnosticCheckpoint, + _ row: CoreWalletDatabaseDiagnosticSnapshot.Txo, + _ outpoint: Data, + _ reason: String + ) { + SDKLogger.event( + "core_db_memory_diff_item", + category: .persistence, + severity: .warning, + fields: [ + "amount_duffs": .unsignedInteger(row.amount), + "checkpoint": .publicText(checkpoint.rawValue), + "height": .unsignedInteger(UInt64(row.height)), + "outpoint_reference": .reference(outpoint), + "reason": .publicText(reason), + "wallet_reference": .reference(walletId), + ] + ) + } + + /// Captures the managed wallet's tracked locks and compares them with the + /// queue-safe SwiftData snapshot. Raw outpoints are only reference-hashed. + private nonisolated static func compareAssetLocks( + _ database: CoreWalletDatabaseDiagnosticSnapshot?, + walletId: Data, + managedWallet: ManagedPlatformWallet?, + checkpoint: CoreWalletDiagnosticCheckpoint + ) { + let memory: [ManagedAssetLockManager.TrackedAssetLock] + do { + guard let managedWallet else { + throw PlatformWalletError.notFound("diagnostic wallet is not loaded") + } + memory = try managedWallet.assetLockManager().listTrackedLocks() + } catch { + SDKLogger.event( + "asset_lock_memory_snapshot", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "query_available": .boolean(false), + "wallet_reference": .reference(walletId), + ] + ) + // Mirror the database-unavailable path below: an analyst greps for + // `asset_lock_db_memory_diff_summary`, and a missing line is + // indistinguishable from a truncated log. Say the diff is + // incomplete instead of saying nothing. + SDKLogger.event( + "asset_lock_db_memory_diff_summary", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "database_query_available": .boolean(database?.assetLocksAvailable ?? false), + "database_snapshot_available": .boolean(database != nil), + "diff_incomplete": .boolean(true), + "memory_query_available": .boolean(false), + "mismatch_count": .integer(0), + "truncated_count": .integer(0), + "wallet_reference": .reference(walletId), + ] + ) + return + } + SDKLogger.event( + "asset_lock_memory_snapshot", + category: .persistence, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "lock_count": .integer(Int64(memory.count)), + "locked_value_duffs": .unsignedInteger( + diagnosticSaturatingSum(memory.map(\.amount)) + ), + "proof_present_count": .integer(Int64(memory.filter(\.hasProof).count)), + "query_available": .boolean(true), + "shielded_funding_count": .integer(Int64(memory.filter { + $0.fundingType == .assetLockShieldedAddressTopUp + }.count)), + "wallet_reference": .reference(walletId), + ] + ) + + let groups = Dictionary(grouping: memory) { + "\($0.fundingType.rawValue):\($0.status.rawValue)" + } + for key in groups.keys.sorted() { + guard let group = groups[key], let first = group.first else { continue } + SDKLogger.event( + "asset_lock_memory_group", + category: .persistence, + fields: [ + "amount_duffs": .unsignedInteger( + diagnosticSaturatingSum(group.map(\.amount)) + ), + "checkpoint": .publicText(checkpoint.rawValue), + "count": .integer(Int64(group.count)), + "funding_type": .unsignedInteger(UInt64(first.fundingType.rawValue)), + "proof_present_count": .integer(Int64(group.filter(\.hasProof).count)), + "status": .unsignedInteger(UInt64(first.status.rawValue)), + "wallet_reference": .reference(walletId), + ] + ) + } + + let normalizedMemory = memory.map { row in + CoreWalletDatabaseDiagnosticSnapshot.AssetLock( + outpointDisplay: Self.assetLockOutpointDisplay(txid: row.txid, vout: row.vout), + fundingType: Int(row.fundingType.rawValue), + status: Int(row.status.rawValue), + accountIndex: row.accountIndex, + registrationIndex: row.identityIndex, + amountDuffs: row.amount, + hasProof: row.hasProof + ) + } + guard let database, database.assetLocksAvailable else { + SDKLogger.event( + "asset_lock_db_memory_diff_summary", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "database_query_available": .boolean(false), + "database_snapshot_available": .boolean(database != nil), + "diff_incomplete": .boolean(true), + "memory_query_available": .boolean(true), + "mismatch_count": .integer(0), + "truncated_count": .integer(0), + "wallet_reference": .reference(walletId), + ] + ) + return + } + let result = CoreWalletDiagnosticAnalyzer.compareAssetLocks( + database: database.assetLocks, + memory: normalizedMemory + ) + SDKLogger.event( + "asset_lock_db_memory_diff_summary", + category: .persistence, + severity: result.details.isEmpty ? .info : .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "database_query_available": .boolean(true), + "database_snapshot_available": .boolean(true), + "diff_incomplete": .boolean(false), + "memory_query_available": .boolean(true), + "mismatch_count": .integer(Int64(result.details.count)), + "truncated_count": .integer(Int64(result.truncatedCount)), + "wallet_reference": .reference(walletId), + ] + ) + for detail in result.emittedDetails { + SDKLogger.event( + "asset_lock_db_memory_diff_item", + category: .persistence, + severity: .warning, + fields: [ + "checkpoint": .publicText(checkpoint.rawValue), + "outpoint_reference": .referenceString(detail.outpointDisplay), + "reason": .publicText(detail.reason), + "wallet_reference": .reference(walletId), + ] + ) + } + } + + /// Marshals one account selector, copies its Rust-owned UTXO slice, and + /// releases the native allocation after all pointed-to scripts are copied. + private nonisolated static func diagnosticAccountUtxos( + managerHandle: Handle, + walletId: Data, + balance: AccountBalance + ) -> Result<[CoreWalletDatabaseDiagnosticSnapshot.Txo], PlatformWalletError> { + var spec = AccountSpecFFI() + spec.type_tag = balance.typeTag + spec.standard_tag = balance.standardTag + spec.index = balance.index + spec.registration_index = balance.registrationIndex + spec.key_class = balance.keyClass + _ = Swift.withUnsafeMutableBytes(of: &spec.user_identity_id) { raw in + balance.userIdentityId.copyBytes( + to: raw.bindMemory(to: UInt8.self), + count: min(32, balance.userIdentityId.count) + ) + } + _ = Swift.withUnsafeMutableBytes(of: &spec.friend_identity_id) { raw in + balance.friendIdentityId.copyBytes( + to: raw.bindMemory(to: UInt8.self), + count: min(32, balance.friendIdentityId.count) + ) + } + var outEntries: UnsafePointer? + var outCount: UInt = 0 + let ffi = walletId.withUnsafeBytes { raw in + platform_wallet_account_utxos( + managerHandle, + raw.baseAddress?.assumingMemoryBound(to: UInt8.self), + &spec, + &outEntries, + &outCount + ) + } + let result = PlatformWalletResult(ffi) + guard result.isSuccess else { return .failure(PlatformWalletError(result: result)) } + guard let entries = outEntries, outCount > 0 else { return .success([]) } + defer { + platform_wallet_account_utxos_free( + UnsafeMutablePointer(mutating: entries), outCount + ) + } + let key = Self.diagnosticAccountKey(balance) + return .success((0.. CoreWalletDatabaseDiagnosticSnapshot.AccountKey { + CoreWalletDatabaseDiagnosticSnapshot.AccountKey( + typeTag: UInt32(balance.typeTag), + standardTag: balance.standardTag, + index: balance.index, + registrationIndex: balance.registrationIndex, + keyClass: balance.keyClass, + userIdentityId: balance.userIdentityId, + friendIdentityId: balance.friendIdentityId + ) + } + + /// Renders the memory side of the AssetLock diff in the exact format the + /// database side is keyed by. + /// + /// `compareAssetLocks` matches the two sides on this string, so it must go + /// through `PersistentAssetLock.encodeOutPoint` rather than a second hex + /// loop: a hand-rolled copy agrees only by coincidence, and any later change + /// to the canonical encoder would silently make every lock report as both + /// `database_only` and `memory_only`. + nonisolated static func assetLockOutpointDisplay( + txid: Data, + vout: UInt32 + ) -> String { + let raw = PersistentTxo.makeOutpoint(txid: txid, vout: vout) + // `encodeOutPoint` traps on a malformed outpoint. Diagnostics must + // survive corrupt input, so fall back to a clearly non-matching marker + // that shows up as `memory_only` instead of taking the process down. + // Reachable only through a `txid` that is not 32 bytes: `makeOutpoint` + // appends whatever it is given, and the FFI copies a fixed 32-byte + // array, so this guards the Rust side's word, not this file's. + guard raw.count == 36 else { + return "invalid_outpoint:\(raw.count)_bytes:\(vout)" + } + return PersistentAssetLock.encodeOutPoint(rawBytes: raw) + } +} diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerSPV.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerSPV.swift index 4fa271e5bb1..a784e62c9ca 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerSPV.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerSPV.swift @@ -338,16 +338,58 @@ extension PlatformWalletManager { /// - fromHeight: the core block height to rewind the filter scan to. public func spvRescanFilters(walletId: Data, fromHeight: UInt32) throws { guard walletId.count == 32 else { + // Every other way this method fails leaves a `core_rescan_requested` + // line; a rejected request must too, or the export reads as if + // no rescan was ever asked for. + SDKLogger.event( + "core_rescan_requested", + category: .persistence, + severity: .error, + fields: [ + "from_height": .unsignedInteger(UInt64(fromHeight)), + "result": .publicText("invalid_wallet_id"), + "wallet_reference": .reference(walletId), + ] + ) throw PlatformWalletError.invalidParameter( "walletId must be exactly 32 bytes" ) } - try walletId.withUnsafeBytes { widRaw in - guard let widPtr = widRaw.baseAddress?.assumingMemoryBound(to: UInt8.self) - else { - throw PlatformWalletError.invalidParameter("walletId baseAddress is nil") + // The event records the request and whether the FFI accepted it — + // nothing about whether a rewind actually happened. Classifying that + // would need the filter-scan checkpoint, which is not the core + // wallet's synced height and is not readable without a blocking + // Rust-lock FFI call on the main actor; a label built on either + // would let an analyst rule a rewind in or out on false grounds. + do { + try walletId.withUnsafeBytes { widRaw in + guard let widPtr = widRaw.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { + throw PlatformWalletError.invalidParameter("walletId baseAddress is nil") + } + try platform_wallet_manager_spv_rescan_filters(handle, widPtr, fromHeight).check() } - try platform_wallet_manager_spv_rescan_filters(handle, widPtr, fromHeight).check() + SDKLogger.event( + "core_rescan_requested", + category: .persistence, + fields: [ + "from_height": .unsignedInteger(UInt64(fromHeight)), + "result": .publicText("accepted"), + "wallet_reference": .reference(walletId), + ] + ) + } catch { + SDKLogger.event( + "core_rescan_requested", + category: .persistence, + severity: .error, + fields: [ + "from_height": .unsignedInteger(UInt64(fromHeight)), + "result": .publicText("failed"), + "wallet_reference": .reference(walletId), + ] + ) + throw error } } } diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift index 49e97964602..123c1eefeb7 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift @@ -135,7 +135,13 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { /// entry points — both the FFI callback shims and the /// app-facing accessors — funnel through `onQueue` so the /// context is only ever touched on this queue. - private let serialQueue = DispatchQueue( + /// Internal only so the read-only diagnostics extension can enqueue its + /// export pass here without blocking the main actor. That pass runs on a + /// scratch `ModelContext` of its own — it never touches + /// `backgroundContext`, so it sees only committed state — and needs this + /// queue solely so no save can land while it reads. All mutations remain + /// in this file's persistence callbacks. + let serialQueue = DispatchQueue( label: "org.dash.platform-wallet.persistence", qos: .userInitiated ) @@ -542,7 +548,18 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { txo.lastUpdated = Date() } } catch { - print("⚠️ persistAssetLocks: stale-TXO fetch failed for \(entry.outPointHex) — failing the round so the lock status does not commit ahead of its spend flags: \(error)") + SDKLogger.event( + "persistence_asset_lock_stale_txo_fetch_failed", + category: .persistence, + severity: .error, + fields: [ + "outpoint_reference": .referenceString(entry.outPointHex), + "status": .integer(Int64(entry.statusRaw)), + "wallet_reference": .reference(walletId), + ], + error: error, + redacting: [entry.outPointHex, wireTxid.hexString] + ) allPersisted = false } } @@ -2538,7 +2555,9 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { /// persisted state reflects "still spendable from this row's POV", /// and the catch-up classifier on the next launch reloads the /// row and recognises it as ours when the block arrives. - private static func spendIsInBlock(_ tx: PersistentTransaction) -> Bool { + /// Internal rather than private so the read-only diagnostics extension + /// classifies a linked spender by the same rule that decided `isSpent`. + static func spendIsInBlock(_ tx: PersistentTransaction) -> Bool { tx.context >= TransactionContextType.inBlock.rawValue } @@ -6571,6 +6590,12 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // called on the path through `loadAllocations` after the // pointer hand-off to Rust succeeds). var unspentBuckets: [Data: [PersistentTxo]] = [:] + // Rows the restore fetch matched to a wallet but which carry no + // account, so they can never be marshalled. Kept apart from + // `unspentBuckets` — rather than duplicating every unspent row into a + // second map — so the lightweight restore summary can report why they + // were dropped at no cost to the launch path. + var accountLessBuckets: [Data: [PersistentTxo]] = [:] do { var unspentDescriptor = FetchDescriptor( predicate: #Predicate { $0.isSpent == false } @@ -6676,8 +6701,13 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { } } unspentBuckets.reserveCapacity(restorable.count) + // Rows no wallet can claim — no denormalized id and either no + // account or an account whose wallet link is broken — belong to + // no bucket and so to no per-wallet snapshot. Counted here and + // reported once, or the restore summary would say nothing was + // dropped while exactly the corruption it exists to surface was. + var unroutableRowCount = 0 for row in liveUnspent { - guard row.account != nil else { continue } let key: Data if !row.walletId.isEmpty { key = row.walletId @@ -6687,13 +6717,34 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // a relationship-store inconsistency would // crash here, so guard via Optional cast. let wallet: PersistentWallet? = account.wallet - guard let resolved = wallet else { continue } + guard let resolved = wallet else { + unroutableRowCount += 1 + continue + } key = resolved.walletId } else { + unroutableRowCount += 1 + continue + } + // Preserve the upstream restore contract: account-less rows + // are diagnostic candidates only and never enter FFI + // marshalling. + guard row.account != nil else { + accountLessBuckets[key, default: []].append(row) continue } unspentBuckets[key, default: []].append(row) } + SDKLogger.event( + "core_restore_unroutable_rows", + category: .persistence, + severity: unroutableRowCount == 0 ? .info : .warning, + fields: [ + "checkpoint": .publicText(CoreWalletDiagnosticCheckpoint.restoreBuffer.rawValue), + "scanned_row_count": .integer(Int64(liveUnspent.count)), + "unroutable_row_count": .integer(Int64(unroutableRowCount)), + ] + ) } // Allocate `entriesPtr` and the `LoadAllocation` here — past @@ -6912,10 +6963,18 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // the matching funds-bearing account by tag; rows whose // account isn't a funds variant get silently skipped on // the receiving side. + let restoreRows = unspentBuckets[w.walletId] ?? [] let (utxoBuf, utxoCount, utxoErrored) = buildUtxoRestoreBuffer( - rows: unspentBuckets[w.walletId] ?? [], + rows: restoreRows, allocation: allocation ) + logCoreRestoreBufferSnapshotOnQueue( + walletId: w.walletId, + rows: restoreRows, + accountLessRows: accountLessBuckets[w.walletId] ?? [], + emittedCount: utxoCount, + errored: utxoErrored + ) // `buildUtxoRestoreBuffer` already deallocated its own // buffer on the errored path; release everything else // we've accumulated and abort the load callback so Rust diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/AssetLockSpendVisibilityTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/AssetLockSpendVisibilityTests.swift index c24f81295da..861e8702c3b 100644 --- a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/AssetLockSpendVisibilityTests.swift +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/AssetLockSpendVisibilityTests.swift @@ -31,15 +31,23 @@ import DashSDKFFI /// Serves every read live except the one model type it is told to fault, /// and records the reads it saw so a test can prove which fetch failed. private final class FetchFaultInjector: ModelFetching, @unchecked Sendable { - struct ReadFault: Error {} + struct ReadFault: LocalizedError { + let message: String + var errorDescription: String? { message } + } private let live = LiveModelFetcher() private let faulted: ObjectIdentifier + private let faultMessage: String private let lock = NSLock() private var reads: [String] = [] - init(faulting model: any PersistentModel.Type) { + init( + faulting model: any PersistentModel.Type, + faultMessage: String = "injected SwiftData read failure" + ) { faulted = ObjectIdentifier(model) + self.faultMessage = faultMessage } /// Model names in the order they were read, the faulted one included. @@ -56,7 +64,9 @@ private final class FetchFaultInjector: ModelFetching, @unchecked Sendable { lock.lock() reads.append(String(describing: T.self)) lock.unlock() - guard ObjectIdentifier(T.self) != faulted else { throw ReadFault() } + guard ObjectIdentifier(T.self) != faulted else { + throw ReadFault(message: faultMessage) + } return try live.fetch(descriptor, in: context) } } @@ -72,6 +82,8 @@ final class AssetLockSpendVisibilityTests: XCTestCase { override func setUpWithError() throws { try super.setUpWithError() + // Whole-log assertions below must not inherit another suite's backlog. + SDKLogger.resetForTesting() container = try DashModelContainer.createInMemory() handler = PlatformWalletPersistenceHandler( modelContainer: container, @@ -85,6 +97,16 @@ final class AssetLockSpendVisibilityTests: XCTestCase { super.tearDown() } + private func temporaryLogDirectory() throws -> URL { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent( + "AssetLockSpendVisibilityTests-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return directory + } + // MARK: Fixtures /// Seeds the state an older build left behind: a restorable wallet @@ -269,19 +291,26 @@ final class AssetLockSpendVisibilityTests: XCTestCase { /// which rolls the status back and lets Rust re-emit it. func testPersistAssetLocksFailsTheRoundWhenTheStaleTxoFetchThrows() throws { let outpoint = try seedFundingTxoSpentByAMempoolAssetLock(into: container) - let injector = FetchFaultInjector(faulting: PersistentTxo.self) + let privacyTxid = Data((0..<32).map { UInt8($0) }) + let outPointRaw = PersistentTxo.makeOutpoint(txid: privacyTxid, vout: 0) + let outPointHex = PersistentAssetLock.encodeOutPoint(rawBytes: outPointRaw) + let injector = FetchFaultInjector( + faulting: PersistentTxo.self, + faultMessage: "failed for \(outPointHex), wire \(privacyTxid.hexString), at /private/user/wallet.sqlite" + ) let handler = PlatformWalletPersistenceHandler( modelContainer: container, network: .testnet, modelFetcher: injector ) - let outPointRaw = PersistentTxo.makeOutpoint(txid: lockTxid, vout: 0) + let session = try temporaryLogDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) handler.beginChangeset(walletId: walletId) let staged = handler.persistAssetLocks( walletId: walletId, upserts: [.init( - outPointHex: PersistentAssetLock.encodeOutPoint(rawBytes: outPointRaw), + outPointHex: outPointHex, transactionBytes: Data(repeating: 0x05, count: 10), fundingTypeRaw: 0, identityIndexRaw: 0, @@ -312,6 +341,25 @@ final class AssetLockSpendVisibilityTests: XCTestCase { try txoIsSpent(outpoint: outpoint), "and nothing may be left half-applied by the rolled-back round" ) + + SDKLogger.flush() + let log = try String( + contentsOf: session.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + let event = try XCTUnwrap(log.split(separator: "\n").first { + $0.contains("event=persistence_asset_lock_stale_txo_fetch_failed ") + }) + XCTAssertTrue(event.contains( + "outpoint_reference=\(SDKLogFormatter.reference(outPointHex))" + )) + XCTAssertTrue(event.contains("")) + XCTAssertFalse(event.contains(outPointHex)) + XCTAssertFalse(event.contains(privacyTxid.hexString)) + XCTAssertFalse(event.contains(Data(privacyTxid.reversed()).hexString)) + XCTAssertFalse(event.contains(outPointRaw.hexString)) + XCTAssertFalse(event.contains(walletId.hexString)) + XCTAssertFalse(event.contains("wallet.sqlite")) } /// An unreadable lock table must not be read as the positive claim diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticAnalyzerTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticAnalyzerTests.swift new file mode 100644 index 00000000000..97ef907fae7 --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticAnalyzerTests.swift @@ -0,0 +1,375 @@ +import Foundation +import XCTest +@testable import SwiftDashSDK + +final class CoreWalletDiagnosticAnalyzerTests: XCTestCase { + typealias AccountKey = CoreWalletDatabaseDiagnosticSnapshot.AccountKey + typealias AssetLock = CoreWalletDatabaseDiagnosticSnapshot.AssetLock + typealias Txo = CoreWalletDatabaseDiagnosticSnapshot.Txo + + private func account( + type: UInt32 = 0, + standardTag: UInt8 = 0, + index: UInt32 = 0 + ) -> AccountKey { + AccountKey( + typeTag: type, + standardTag: standardTag, + index: index, + registrationIndex: 0, + keyClass: 0, + userIdentityId: Data(), + friendIdentityId: Data() + ) + } + + private func outpoint(_ marker: UInt8) -> Data { + Data(repeating: marker, count: 32) + Data([0, 0, 0, 0]) + } + + private func txo( + _ marker: UInt8, + amount: UInt64 = 100, + height: UInt32 = 200, + script: Data = Data([0x51]), + locked: Bool = false, + account: AccountKey? = nil + ) -> Txo { + Txo( + outpoint: outpoint(marker), + amount: amount, + height: height, + scriptPubKey: script, + isLocked: locked, + account: account ?? self.account() + ) + } + + private func assetLock( + _ outpoint: String, + fundingType: Int = 5, + status: Int = 1, + accountIndex: UInt32 = 2, + registrationIndex: UInt32 = 3, + amount: UInt64? = 400, + hasProof: Bool = true + ) -> AssetLock { + AssetLock( + outpointDisplay: outpoint, + fundingType: fundingType, + status: status, + accountIndex: accountIndex, + registrationIndex: registrationIndex, + amountDuffs: amount, + hasProof: hasProof + ) + } + + func testTxoDiffExactDatabaseOnlyMemoryOnlyAndEveryFieldMismatch() { + let baseAccount = account() + let exact = txo(0x01, account: baseAccount) + let exactResult = CoreWalletDiagnosticAnalyzer.compareTxos( + database: [exact], + memory: [exact], + databaseAccounts: [baseAccount], + memoryAccounts: [baseAccount] + ) + XCTAssertEqual(exactResult.commonCount, 1) + XCTAssertEqual(exactResult.databaseAccountOnlyCount, 0) + XCTAssertEqual(exactResult.memoryAccountOnlyCount, 0) + XCTAssertTrue(exactResult.details.isEmpty) + + let database = [ + txo(0x10), + txo(0x20, amount: 101), + txo(0x21, height: 201), + txo(0x22, script: Data([0x52])), + txo(0x23, locked: true), + txo(0x24, account: account(type: 1)), + ] + let memory = [ + txo(0x11), + txo(0x20, amount: 102), + txo(0x21, height: 202), + txo(0x22, script: Data([0x53])), + txo(0x23, locked: false), + txo(0x24, account: account(type: 0)), + ] + let result = CoreWalletDiagnosticAnalyzer.compareTxos( + database: database, + memory: memory, + databaseAccounts: [baseAccount, account(type: 1)], + memoryAccounts: [baseAccount, account(type: 2)] + ) + + XCTAssertEqual(result.commonCount, 5) + XCTAssertEqual(result.databaseAccountOnlyCount, 1) + XCTAssertEqual(result.memoryAccountOnlyCount, 1) + XCTAssertEqual(result.databaseOnlyCount, 1) + XCTAssertEqual(result.memoryOnlyCount, 1) + XCTAssertEqual(result.fieldMismatchCount, 5) + XCTAssertEqual(Set(result.details.map(\.reason)), [ + "account_mismatch", + "amount_mismatch", + "database_only", + "height_mismatch", + "lock_mismatch", + "memory_only", + "script_mismatch", + ]) + } + + func testTxoDiffLimitsEachReasonToTwentyFiveDetails() { + let database = (0..<30).map { index in + txo(UInt8(index + 1)) + } + let result = CoreWalletDiagnosticAnalyzer.compareTxos( + database: database, + memory: [], + databaseAccounts: [], + memoryAccounts: [] + ) + + XCTAssertEqual(result.details.count, 30) + XCTAssertEqual(result.emittedDetails.count, 25) + XCTAssertEqual(result.truncatedCount, 5) + XCTAssertTrue(result.emittedDetails.allSatisfy { $0.reason == "database_only" }) + XCTAssertEqual( + result.emittedDetails.map(\.outpoint), + result.emittedDetails.map(\.outpoint).sorted { + $0.lexicographicallyPrecedes($1) + } + ) + } + + func testAssetLockDiffExactAndEveryMismatchClass() { + let exact = assetLock("exact:0") + let exactResult = CoreWalletDiagnosticAnalyzer.compareAssetLocks( + database: [exact], + memory: [exact] + ) + XCTAssertTrue(exactResult.details.isEmpty) + + let result = CoreWalletDiagnosticAnalyzer.compareAssetLocks( + database: [ + assetLock("database-only:0"), + assetLock("different:0"), + ], + memory: [ + assetLock("memory-only:0"), + assetLock( + "different:0", + fundingType: 4, + status: 2, + accountIndex: 7, + registrationIndex: 8, + amount: 401, + hasProof: false + ), + ] + ) + + XCTAssertEqual(Set(result.details.map(\.reason)), [ + "account_index_mismatch", + "amount_mismatch", + "database_only", + "funding_type_mismatch", + "memory_only", + "proof_presence_mismatch", + "registration_index_mismatch", + "status_mismatch", + ]) + XCTAssertEqual(result.emittedDetails.count, result.details.count) + XCTAssertEqual(result.truncatedCount, 0) + } + + func testMissingAccountIsDatabaseAnomalyAndRejectedFromRestoreBuffer() { + let missingAccountTxo = Txo( + outpoint: outpoint(0x30), + amount: 700, + height: 900, + scriptPubKey: Data([0x51]), + isLocked: false, + account: nil + ) + let anomalies = CoreWalletDiagnosticAnalyzer.databaseTxoAnomalies([ + .init( + txo: missingAccountTxo, + hasParentTransaction: true, + walletIdMismatch: false, + isSpent: false, + hasSpendingTransaction: false + ), + ]) + XCTAssertEqual(anomalies.count(reason: "missing_account"), 1) + + let rejected = CoreWalletDiagnosticAnalyzer.RestoreCandidate( + amount: missingAccountTxo.amount, + accountType: nil, + standardTag: nil, + rejectionReason: .missingAccount + ) + let acceptedTxo = txo(0x31, amount: 800) + let accepted = CoreWalletDiagnosticAnalyzer.RestoreCandidate( + amount: acceptedTxo.amount, + accountType: 0, + standardTag: 0, + rejectionReason: nil + ) + let summary = CoreWalletDiagnosticAnalyzer.summarizeRestoreBuffer( + candidates: [rejected, accepted], + emittedCount: 1, + errored: false + ) + + XCTAssertEqual(summary.candidateCount, 2) + XCTAssertEqual(summary.candidateValueDuffs, 1_500) + XCTAssertEqual(summary.missingAccountCount, 1) + XCTAssertEqual(summary.emittedCount, 1) + XCTAssertEqual(summary.emittedValueDuffs, 800) + XCTAssertEqual(summary.emittedBip44Count, 1) + XCTAssertEqual(summary.emittedBip44ValueDuffs, 800) + } + + /// A TXO linked to a mempool spender while still unspent is what + /// `reconcileSpendObservation` writes for every normal in-flight send, so + /// it must not be reported — otherwise one healthy unconfirmed transaction + /// buries the export in warnings. + func testUnconfirmedSpenderIsNotAnAnomalyButAConfirmedOneIs() { + let inFlightSend = CoreWalletDiagnosticAnalyzer.databaseTxoAnomalies([ + .init( + txo: txo(0x32), + hasParentTransaction: true, + walletIdMismatch: false, + isSpent: false, + hasSpendingTransaction: true, + spendingTransactionIsInBlock: false + ), + ]) + XCTAssertTrue(inFlightSend.details.isEmpty) + + let confirmedButUnspent = CoreWalletDiagnosticAnalyzer.databaseTxoAnomalies([ + .init( + txo: txo(0x33), + hasParentTransaction: true, + walletIdMismatch: false, + isSpent: false, + hasSpendingTransaction: true, + spendingTransactionIsInBlock: true + ), + ]) + XCTAssertEqual( + confirmedButUnspent.count(reason: "unspent_with_confirmed_spending_transaction"), + 1 + ) + + let spentWithoutLink = CoreWalletDiagnosticAnalyzer.databaseTxoAnomalies([ + .init( + txo: txo(0x34), + hasParentTransaction: true, + walletIdMismatch: false, + isSpent: true, + hasSpendingTransaction: false + ), + ]) + XCTAssertEqual( + spentWithoutLink.count(reason: "spent_without_spending_transaction"), + 1 + ) + } + + /// `compareAssetLocks` matches the two sides on `outpointDisplay`. The + /// database side is keyed by `PersistentAssetLock.encodeOutPoint`; the + /// memory side by `assetLockOutpointDisplay`. This pins that they are one + /// encoder, so a lock present on both sides is never reported twice. + func testAssetLockDiffKeysAgreeBetweenDatabaseAndMemorySides() { + let txid = Data((0..<32).map { UInt8($0) }) + let databaseKey = PersistentAssetLock.encodeOutPoint( + rawBytes: PersistentTxo.makeOutpoint(txid: txid, vout: 7) + ) + let memoryKey = PlatformWalletManager.assetLockOutpointDisplay(txid: txid, vout: 7) + XCTAssertEqual(databaseKey, memoryKey) + XCTAssertTrue(memoryKey.hasSuffix(":7"), memoryKey) + + let diff = CoreWalletDiagnosticAnalyzer.compareAssetLocks( + database: [assetLock(databaseKey)], + memory: [assetLock(memoryKey)] + ) + XCTAssertTrue(diff.details.isEmpty, "\(diff.details)") + + // A malformed txid must neither trap the exporter nor collide with a + // real key: it surfaces as `memory_only`. + let malformed = PlatformWalletManager.assetLockOutpointDisplay( + txid: Data([1, 2, 3]), + vout: 7 + ) + XCTAssertTrue(malformed.hasPrefix("invalid_outpoint:"), malformed) + let malformedDiff = CoreWalletDiagnosticAnalyzer.compareAssetLocks( + database: [assetLock(databaseKey)], + memory: [assetLock(malformed)] + ) + XCTAssertEqual( + malformedDiff.details.map(\.reason).sorted(), + ["database_only", "memory_only"] + ) + } + + func testShieldedStoreSummaryIncludesValuesActivityKeysAndWatermark() { + let summary = CoreWalletDiagnosticAnalyzer.summarizeShieldedStore( + notes: [ + .init(value: 7, isSpent: true), + .init(value: 8, isSpent: true), + .init(value: 20, isSpent: false), + ], + outgoingNoteCount: 2, + activityStatuses: [0, 1, 2, 0], + viewingKeyCount: 3, + syncWatermarks: [5, 99, 40] + ) + + XCTAssertEqual(summary.noteCount, 3) + XCTAssertEqual(summary.spentNoteCount, 2) + XCTAssertEqual(summary.spentValueCredits, 15) + XCTAssertEqual(summary.unspentNoteCount, 1) + XCTAssertEqual(summary.unspentValueCredits, 20) + XCTAssertEqual(summary.outgoingNoteCount, 2) + XCTAssertEqual(summary.activityCount, 4) + XCTAssertEqual(summary.activityPendingCount, 2) + XCTAssertEqual(summary.activityFailedCount, 1) + XCTAssertEqual(summary.viewingKeyCount, 3) + XCTAssertEqual(summary.subwalletSyncStateCount, 3) + XCTAssertEqual(summary.maximumSyncWatermark, 99) + } + + func testFingerprintIsStableUnderReorderAndSensitiveToEveryTxoField() { + let baseAccount = account() + let first = txo(0x40, account: baseAccount) + let second = txo(0x41, amount: 200, account: baseAccount) + let firstMaterial = fingerprintMaterial(first) + let secondMaterial = fingerprintMaterial(second) + + XCTAssertEqual( + diagnosticFingerprint([firstMaterial, secondMaterial]), + diagnosticFingerprint([secondMaterial, firstMaterial]) + ) + XCTAssertNotEqual(firstMaterial, fingerprintMaterial(txo(0x40, amount: 101))) + XCTAssertNotEqual(firstMaterial, fingerprintMaterial(txo(0x40, height: 201))) + XCTAssertNotEqual(firstMaterial, fingerprintMaterial(txo(0x40, script: Data([0x52])))) + XCTAssertNotEqual(firstMaterial, fingerprintMaterial(txo(0x40, locked: true))) + XCTAssertNotEqual( + firstMaterial, + fingerprintMaterial(txo(0x40, account: account(type: 1))) + ) + } + + private func fingerprintMaterial(_ txo: Txo) -> Data { + diagnosticTxoFingerprint( + outpoint: txo.outpoint, + amount: txo.amount, + height: txo.height, + scriptPubKey: txo.scriptPubKey, + isLocked: txo.isLocked, + account: txo.account + ) + } +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticsTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticsTests.swift new file mode 100644 index 00000000000..86e273b2227 --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/CoreWalletDiagnosticsTests.swift @@ -0,0 +1,554 @@ +import Foundation +import SwiftData +import XCTest +@testable import SwiftDashSDK + +/// Regression coverage for the diagnostic that identifies #4438: a sent +/// transaction consumes a CoinJoin output and pays change back to an address +/// owned by the wallet's BIP44 account, but the owned output is absent from +/// SwiftData. The same test exercises the complete structured-log line so a +/// future field addition cannot accidentally expose wallet material. +@MainActor +final class CoreWalletDiagnosticsTests: XCTestCase { + private static let fixtureHex = + "01000000011111111111111111111111111111111111111111111111111111111111111111" + + "030000006a4730303030303030303030303030303030303030303030303030303030303030" + + "30303030303030303030303030303030303030303030303030303030303030303030303030" + + "303030210324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c" + + "ffffffff02204e0200000000001976a91414db4138d56a2ecfb10881a9be394d9f321985b2" + + "88ac0000000000000000066a04aaaaaaaa00000000" + + private static let fixtureAddress = "yNDj28QBMm5sY6bLjFcNdWRNef24KLQNuQ" + private static let fixtureTxidDisplay = + "bf7479216e5ba76f60bf11654c881824c6f9cdbb64eebe332cf835a3391cb5d5" + + private let walletId = Data(repeating: 0xa1, count: 32) + + /// These tests assert over the complete `run.log`; a backlog buffered by + /// an earlier suite must not be replayed into it. + override func setUp() async throws { + SDKLogger.resetForTesting() + } + + private var fixtureData: Data { + var data = Data() + var index = Self.fixtureHex.startIndex + while index < Self.fixtureHex.endIndex { + let next = Self.fixtureHex.index(index, offsetBy: 2) + data.append(UInt8(Self.fixtureHex[index.. URL { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent( + "CoreWalletDiagnosticsTests-\(UUID().uuidString)", + isDirectory: true + ) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return directory + } + + private func logLines(in session: URL, event: String) throws -> [String] { + SDKLogger.flush() + let log = try String( + contentsOf: session.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + return log.split(separator: "\n").map(String.init).filter { + $0.contains("event=\(event) ") + } + } + + private struct Fixture { + let handler: PlatformWalletPersistenceHandler + let context: ModelContext + let spendingTransaction: PersistentTransaction + let bip44Account: PersistentAccount + let bip44Address: PersistentCoreAddress + let decoded: DecodedTransaction + } + + private func makeMissingOwnedOutputFixture() throws -> Fixture { + let container = try DashModelContainer.createInMemory() + let context = ModelContext(container) + context.autosaveEnabled = false + let handler = PlatformWalletPersistenceHandler( + modelContainer: container, + network: .testnet + ) + + let wallet = PersistentWallet(walletId: walletId, network: .testnet) + context.insert(wallet) + + let bip44 = PersistentAccount( + wallet: wallet, + accountType: 0, + accountIndex: 0, + accountTypeName: "Standard" + ) + bip44.standardTag = 0 + context.insert(bip44) + + let coinJoin = PersistentAccount( + wallet: wallet, + accountType: 1, + accountIndex: 0, + accountTypeName: "CoinJoin" + ) + context.insert(coinJoin) + + let address = PersistentCoreAddress( + address: Self.fixtureAddress, + poolTypeTag: 1, + addressIndex: 4, + derivationPath: "privacy-fixture-path" + ) + address.account = bip44 + context.insert(address) + + // This is the output that the decoded fixture spends (11…11:3). + // Empty consensus bytes keep it out of the transaction decoder while + // preserving the real ownership relation used by the audit. + let funding = PersistentTransaction( + txid: Data(repeating: 0x11, count: 32), + transactionData: Data(), + context: 2, + blockHeight: 100, + netAmount: 151_072 + ) + context.insert(funding) + let coinJoinTxo = PersistentTxo( + transaction: funding, + vout: 3, + amount: 151_072, + address: "coinjoin-input-address", + scriptPubKey: Data([0x51]), + height: 100 + ) + coinJoinTxo.account = coinJoin + coinJoinTxo.walletId = walletId + coinJoinTxo.isConfirmed = true + context.insert(coinJoinTxo) + + let decoded = try TransactionDecoder.decode(fixtureData, network: .testnet) + let spending = PersistentTransaction( + txid: decoded.txid, + transactionData: fixtureData, + context: 2, + blockHeight: 101, + direction: 1, + netAmount: -151_072 + ) + spending.involvedAccounts.append(coinJoin) + coinJoinTxo.spendingTransaction = spending + coinJoinTxo.isSpent = true + context.insert(spending) + + try context.save() + return Fixture( + handler: handler, + context: context, + spendingTransaction: spending, + bip44Account: bip44, + bip44Address: address, + decoded: decoded + ) + } + + func testCoinJoinSpendWithMissingBip44ChangeDetects4438AndLogIsPrivate() async throws { + let fixture = try makeMissingOwnedOutputFixture() + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics(walletId: walletId) + XCTAssertNotNil(databaseSnapshot) + + let summaries = try logLines(in: session, event: "core_owned_output_audit_summary") + let summary = try XCTUnwrap(summaries.last) + XCTAssertTrue(summary.contains("candidate_transaction_count=1"), summary) + XCTAssertTrue(summary.contains("coinjoin_to_bip44_missing_count=1"), summary) + XCTAssertTrue(summary.contains("coinjoin_to_bip44_missing_value_duffs=151072"), summary) + XCTAssertTrue(summary.contains("owned_bip44_output_count=1"), summary) + XCTAssertTrue(summary.contains("persisted_valid_count=0"), summary) + XCTAssertTrue(summary.contains("total_anomaly_count=1"), summary) + // Output 1 of the fixture is `OP_RETURN`: no address to attribute. + XCTAssertTrue(summary.contains("output_address_undecodable_count=1"), summary) + XCTAssertTrue(summary.contains("unattributed_output_count=0"), summary) + XCTAssertTrue(summary.contains("bip44_address_pool_size=1"), summary) + + let anomalies = try logLines(in: session, event: "core_owned_output_anomaly") + let anomaly = try XCTUnwrap(anomalies.last) + XCTAssertTrue(anomaly.contains(#"reason="missing_txo""#), anomaly) + + // Assert privacy over every line generated by the complete snapshot, + // not just over one hand-constructed formatter input. + SDKLogger.flush() + let completeLog = try String( + contentsOf: session.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + XCTAssertFalse(completeLog.contains(Self.fixtureAddress)) + XCTAssertFalse(completeLog.contains(Self.fixtureTxidDisplay)) + XCTAssertFalse(completeLog.contains(Self.fixtureHex)) + XCTAssertFalse(completeLog.contains("privacy-fixture-path")) + let rawTxidHex = fixture.decoded.txid.map { String(format: "%02x", $0) }.joined() + let reversedTxidHex = fixture.decoded.txid.reversed().map { + String(format: "%02x", $0) + }.joined() + let scriptHex = fixture.decoded.outputs[0].scriptPubkey.map { + String(format: "%02x", $0) + }.joined() + let rawOutpointHex = PersistentTxo.makeOutpoint( + txid: fixture.decoded.txid, + vout: 0 + ).map { String(format: "%02x", $0) }.joined() + XCTAssertFalse(completeLog.contains(rawTxidHex)) + XCTAssertFalse(completeLog.contains(reversedTxidHex)) + XCTAssertFalse(completeLog.contains(scriptHex)) + XCTAssertFalse(completeLog.contains(rawOutpointHex)) + XCTAssertFalse(completeLog.contains(walletId.map { String(format: "%02x", $0) }.joined())) + XCTAssertFalse(completeLog.contains(Data(repeating: 0x11, count: 32).map { + String(format: "%02x", $0) + }.joined())) + } + + func testPersistedBip44ChangeClears4438Alarm() async throws { + let fixture = try makeMissingOwnedOutputFixture() + let output = fixture.decoded.outputs[0] + let change = PersistentTxo( + transaction: fixture.spendingTransaction, + vout: 0, + amount: output.valueDuffs, + address: try XCTUnwrap(output.address), + scriptPubKey: output.scriptPubkey, + height: fixture.spendingTransaction.blockHeight + ) + change.account = fixture.bip44Account + change.coreAddress = fixture.bip44Address + change.walletId = walletId + change.isConfirmed = true + fixture.context.insert(change) + try fixture.context.save() + + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics(walletId: walletId) + XCTAssertNotNil(databaseSnapshot) + + let summaries = try logLines(in: session, event: "core_owned_output_audit_summary") + let summary = try XCTUnwrap(summaries.last) + XCTAssertTrue(summary.contains("coinjoin_to_bip44_missing_count=0"), summary) + XCTAssertTrue(summary.contains("owned_bip44_output_count=1"), summary) + XCTAssertTrue(summary.contains("persisted_valid_count=1"), summary) + XCTAssertTrue(summary.contains("total_anomaly_count=0"), summary) + XCTAssertTrue(try logLines(in: session, event: "core_owned_output_anomaly").isEmpty) + } + + /// Above the row ceilings the export must refuse the exact audit outright + /// and say so — never truncate the table and misclassify — while every + /// lightweight snapshot still lands. The fixture has two transactions and + /// one TXO, so a ceiling of one transaction is over the line. + func testExportDeclinesExactAuditAboveRowLimitsButKeepsSnapshots() async throws { + let fixture = try makeMissingOwnedOutputFixture() + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics( + walletId: walletId, + limits: CoreDiagnosticRowLimits( + crossWalletTxoRows: 100, + exactAuditTransactionRows: 1 + ) + ) + XCTAssertNotNil(databaseSnapshot) + + let summaries = try logLines(in: session, event: "core_owned_output_audit_summary") + let summary = try XCTUnwrap(summaries.last) + XCTAssertTrue(summary.contains("audit_incomplete=true"), summary) + XCTAssertTrue(summary.contains(#"reason="tables_too_large_for_exact_audit""#), summary) + XCTAssertTrue(summary.contains("transaction_row_count=2"), summary) + XCTAssertTrue(summary.contains("transaction_row_limit=1"), summary) + XCTAssertTrue(summary.contains("txo_row_count=1"), summary) + // Declining is not the same as finding nothing: no per-output verdict + // may be emitted for an audit that never ran. + XCTAssertFalse(summary.contains("coinjoin_to_bip44_missing_count"), summary) + XCTAssertTrue(try logLines(in: session, event: "core_owned_output_anomaly").isEmpty) + + let walletSnapshot = try XCTUnwrap( + try logLines(in: session, event: "core_db_wallet_snapshot").last + ) + XCTAssertTrue(walletSnapshot.contains(#"txo_scan_scope="cross_wallet""#), walletSnapshot) + XCTAssertTrue(walletSnapshot.contains("transaction_scan_available=false"), walletSnapshot) + XCTAssertTrue(walletSnapshot.contains("txo_count=1"), walletSnapshot) + XCTAssertFalse(try logLines(in: session, event: "core_db_account_snapshot").isEmpty) + XCTAssertFalse(try logLines(in: session, event: "core_db_anomaly_summary").isEmpty) + XCTAssertFalse(try logLines(in: session, event: "asset_lock_db_snapshot").isEmpty) + } + + /// Over the TXO ceiling the scan narrows to this wallet's denormalized id + /// and the snapshot records that scope, so an analyst knows relationship- + /// only rows and cross-wallet duplicates were outside its view. + func testExportNarrowsTxoScanToWalletAboveTxoRowLimit() async throws { + let fixture = try makeMissingOwnedOutputFixture() + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics( + walletId: walletId, + limits: CoreDiagnosticRowLimits( + crossWalletTxoRows: 0, + exactAuditTransactionRows: 100 + ) + ) + XCTAssertNotNil(databaseSnapshot) + + let walletSnapshot = try XCTUnwrap( + try logLines(in: session, event: "core_db_wallet_snapshot").last + ) + XCTAssertTrue(walletSnapshot.contains(#"txo_scan_scope="wallet_id_only""#), walletSnapshot) + XCTAssertTrue(walletSnapshot.contains("txo_count=1"), walletSnapshot) + // A narrowed TXO scan alone is enough to decline the audit, even + // though the transaction table is under its own ceiling. + let summary = try XCTUnwrap( + try logLines(in: session, event: "core_owned_output_audit_summary").last + ) + XCTAssertTrue(summary.contains(#"reason="tables_too_large_for_exact_audit""#), summary) + XCTAssertTrue(summary.contains("txo_row_limit=0"), summary) + } + + /// Without the change address's `PersistentCoreAddress` row the audit can + /// attribute nothing, and must say so through the counter rather than + /// report a clean wallet — this is the "false all-clear" from review. + func testMissingAddressRowIsCountedAsUnattributedNotCleared() async throws { + let fixture = try makeMissingOwnedOutputFixture() + fixture.context.delete(fixture.bip44Address) + try fixture.context.save() + + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics(walletId: walletId) + XCTAssertNotNil(databaseSnapshot) + + let summary = try XCTUnwrap( + try logLines(in: session, event: "core_owned_output_audit_summary").last + ) + XCTAssertTrue(summary.contains("candidate_transaction_count=1"), summary) + XCTAssertTrue(summary.contains("bip44_address_pool_size=0"), summary) + XCTAssertTrue(summary.contains("unattributed_output_count=1"), summary) + XCTAssertTrue(summary.contains("output_address_undecodable_count=1"), summary) + XCTAssertTrue(summary.contains("owned_bip44_output_count=0"), summary) + // Zero here means "of what could be attributed" — and the counters + // above show that was nothing. + XCTAssertTrue(summary.contains("coinjoin_to_bip44_missing_count=0"), summary) + XCTAssertTrue(summary.contains("total_anomaly_count=0"), summary) + } + + /// A change row that names this wallet by its denormalized id but has no + /// account relationship is this wallet's row with a broken link, not + /// another wallet's row — the same rule that admits it must judge it. + func testOwnedRowWithBrokenRelationshipIsRelationshipMissingNotWrongWallet() async throws { + let fixture = try makeMissingOwnedOutputFixture() + let output = fixture.decoded.outputs[0] + let change = PersistentTxo( + transaction: fixture.spendingTransaction, + vout: 0, + amount: output.valueDuffs, + address: try XCTUnwrap(output.address), + scriptPubKey: output.scriptPubkey, + height: fixture.spendingTransaction.blockHeight + ) + change.walletId = walletId + change.isConfirmed = true + fixture.context.insert(change) + try fixture.context.save() + + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + let databaseSnapshot = await fixture.handler.emitCoreWalletDatabaseDiagnostics(walletId: walletId) + XCTAssertNotNil(databaseSnapshot) + + let summary = try XCTUnwrap( + try logLines(in: session, event: "core_owned_output_audit_summary").last + ) + XCTAssertTrue(summary.contains("coinjoin_to_bip44_missing_count=0"), summary) + XCTAssertTrue(summary.contains("total_anomaly_count=1"), summary) + let anomaly = try XCTUnwrap( + try logLines(in: session, event: "core_owned_output_anomaly").last + ) + XCTAssertTrue(anomaly.contains(#"reason="relationship_missing""#), anomaly) + XCTAssertFalse(anomaly.contains("wrong_wallet"), anomaly) + } + + /// `outpoint` is `@Attribute(.unique)`, so a duplicated outpoint only ever + /// exists in a corrupt store and cannot be saved through a context. The + /// resolver is therefore exercised on transient rows. + func testRepresentativeTxoPrefersThisWalletsRowRegardlessOfFetchOrder() throws { + let ours = PersistentWallet(walletId: walletId, network: .testnet) + let theirs = PersistentWallet(walletId: Data(repeating: 0xB2, count: 32), network: .testnet) + let ourAccount = PersistentAccount( + wallet: ours, accountType: 0, accountIndex: 0, accountTypeName: "Standard" + ) + let theirAccount = PersistentAccount( + wallet: theirs, accountType: 0, accountIndex: 0, accountTypeName: "Standard" + ) + let transaction = PersistentTransaction( + txid: Data(repeating: 0x44, count: 32), + transactionData: Data(), + context: 2, + blockHeight: 1, + netAmount: 0 + ) + func row(amount: UInt64, account: PersistentAccount?, walletId: Data) -> PersistentTxo { + let txo = PersistentTxo( + transaction: transaction, + vout: 0, + amount: amount, + address: "duplicate-outpoint", + scriptPubKey: Data([0x51]), + height: 1 + ) + txo.account = account + txo.walletId = walletId + return txo + } + let ourRow = row(amount: 1, account: ourAccount, walletId: walletId) + let theirRow = row(amount: 2, account: theirAccount, walletId: theirs.walletId) + let theirOtherRow = row(amount: 3, account: theirAccount, walletId: theirs.walletId) + + XCTAssertTrue( + PlatformWalletPersistenceHandler.representativeTxo( + rows: [theirRow, ourRow], walletId: walletId + ) === ourRow + ) + XCTAssertTrue( + PlatformWalletPersistenceHandler.representativeTxo( + rows: [ourRow, theirRow], walletId: walletId + ) === ourRow + ) + // Ours by denormalized id alone still wins: the audit judges by the + // same rule that admits, and reports the broken link separately. + let ourBrokenRow = row(amount: 4, account: nil, walletId: walletId) + XCTAssertTrue( + PlatformWalletPersistenceHandler.representativeTxo( + rows: [theirRow, ourBrokenRow], walletId: walletId + ) === ourBrokenRow + ) + // No owned row: still the same answer whichever order the fetch gave. + let forward = PlatformWalletPersistenceHandler.representativeTxo( + rows: [theirRow, theirOtherRow], walletId: walletId + ) + let reversed = PlatformWalletPersistenceHandler.representativeTxo( + rows: [theirOtherRow, theirRow], walletId: walletId + ) + XCTAssertNotNil(forward) + XCTAssertTrue(forward === reversed) + XCTAssertNil( + PlatformWalletPersistenceHandler.representativeTxo(rows: nil, walletId: walletId) + ) + XCTAssertNil( + PlatformWalletPersistenceHandler.representativeTxo(rows: [], walletId: walletId) + ) + } + + func testRestoreOnlyLogsLightweightBufferSnapshotAndNoDeepStartupEvents() throws { + let fixture = try makeMissingOwnedOutputFixture() + fixture.bip44Account.accountExtendedPubKeyBytes = Data(repeating: 0x02, count: 78) + + let validTransaction = PersistentTransaction( + txid: Data(repeating: 0x22, count: 32), + transactionData: Data(), + context: 2, + blockHeight: 102, + netAmount: 100 + ) + fixture.context.insert(validTransaction) + let validTxo = PersistentTxo( + transaction: validTransaction, + vout: 0, + amount: 100, + address: "valid-restore-address", + scriptPubKey: Data([0x51]), + height: 102 + ) + validTxo.account = fixture.bip44Account + validTxo.walletId = walletId + validTxo.isConfirmed = true + fixture.context.insert(validTxo) + + let missingAccountTransaction = PersistentTransaction( + txid: Data(repeating: 0x33, count: 32), + transactionData: Data(), + context: 2, + blockHeight: 103, + netAmount: 200 + ) + fixture.context.insert(missingAccountTransaction) + let missingAccountTxo = PersistentTxo( + transaction: missingAccountTransaction, + vout: 0, + amount: 200, + address: "missing-account-restore-address", + scriptPubKey: Data([0x52]), + height: 103 + ) + missingAccountTxo.walletId = walletId + missingAccountTxo.isConfirmed = true + fixture.context.insert(missingAccountTxo) + try fixture.context.save() + + let session = try temporaryDirectory() + XCTAssertTrue(SDKLogger.installFileSink(at: session, includeDebug: false)) + + let result = fixture.handler.loadWalletList() + XCTAssertFalse(result.errored) + XCTAssertEqual(result.count, 1) + let entries = try XCTUnwrap(result.entries) + defer { fixture.handler.loadWalletListFree(entries: UnsafeRawPointer(entries)) } + + let snapshots = try logLines(in: session, event: "core_restore_buffer_snapshot") + let snapshot = try XCTUnwrap(snapshots.last) + XCTAssertTrue(snapshot.contains("candidate_count=2"), snapshot) + XCTAssertTrue(snapshot.contains("candidate_value_duffs=300"), snapshot) + XCTAssertTrue(snapshot.contains("emitted_count=1"), snapshot) + XCTAssertTrue(snapshot.contains("emitted_value_duffs=100"), snapshot) + XCTAssertTrue(snapshot.contains("skipped_missing_account_count=1"), snapshot) + XCTAssertTrue(snapshot.contains(#"checkpoint="restore_buffer""#), snapshot) + XCTAssertFalse(snapshot.contains("fingerprint"), snapshot) + + SDKLogger.flush() + let completeLog = try String( + contentsOf: session.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + // The complete set of events the pre-export path emits, spelled exactly + // as `PlatformWalletManagerCoreDiagnostics` writes them — a name that + // is never emitted (`core_db_memory_diff`, say, whose real event is + // `core_db_memory_diff_item`) would make its guard vacuous. + let deepDiagnosticEvents = [ + "asset_lock_db_group", + "asset_lock_db_memory_diff_item", + "asset_lock_db_memory_diff_summary", + "asset_lock_db_snapshot", + "asset_lock_memory_group", + "asset_lock_memory_snapshot", + "core_db_account_snapshot", + "core_db_anomaly_summary", + "core_db_memory_diff_item", + "core_db_memory_diff_summary", + "core_db_txo_anomaly", + "core_db_wallet_snapshot", + "core_diagnostics_unavailable", + "core_memory_account_snapshot", + "core_memory_snapshot_unavailable", + "core_owned_output_anomaly", + "core_owned_output_audit_summary", + "shielded_store_snapshot", + ] + for event in deepDiagnosticEvents { + XCTAssertFalse(completeLog.contains("event=\(event) "), event) + } + } +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Dev1StoreUpgradeTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Dev1StoreUpgradeTests.swift new file mode 100644 index 00000000000..d32df34bc7d --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Dev1StoreUpgradeTests.swift @@ -0,0 +1,465 @@ +import CoreData +import Foundation +import SwiftData +import XCTest + +@testable import SwiftDashSDK + +/// Pins that a real v4.2.0-dev.1 store opens through the path the SDK actually +/// ships, and keeps its Core wallet records. +/// +/// The staged `DashMigrationPlan` alone cannot open it: staged migration +/// matches a store by each registered `VersionedSchema`'s checksum, and only +/// `PersistentAssetLock` is frozen so far (`DashSchemaFrozenModels.swift`), so +/// the drifted `PersistentDocumentType` / `PersistentIndex` shapes leave a +/// dev.1 store matching no registered version — Cocoa error 134504. Hosts turn +/// that throw into a launch crash, which is why `DashModelContainer.open` falls +/// back to inferred lightweight migration for exactly that error. These tests +/// drive that production entry point rather than rebuilding a look-alike +/// container, so the fallback — and its limits — cannot regress unnoticed. +@MainActor +final class Dev1StoreUpgradeTests: XCTestCase { + private var directory: URL! + private var fixtureSQLite: Data! + + override func setUp() async throws { + let resourceURL = try XCTUnwrap( + Bundle.module.url( + forResource: "DashModel-v4.2.0-dev.1.sqlite", + withExtension: "zlib", + subdirectory: "Fixtures" + ) + ) + let compressed = try Data(contentsOf: resourceURL) + // This resource is produced with Foundation's `.zlib` compressor. + // A Python zlib-wrapped stream is not accepted by NSData on iOS. + fixtureSQLite = try (compressed as NSData).decompressed(using: .zlib) as Data + XCTAssertEqual(fixtureSQLite.count, 647_168) + + directory = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: true + ) + // Every test starts from an empty logger and installs its own sink + // before touching a store, so the log it reads holds only its own + // lines and nothing buffered by an earlier suite can replay into it. + SDKLogger.resetForTesting() + XCTAssertTrue(SDKLogger.installFileSink(at: directory, includeDebug: false)) + } + + override func tearDown() async throws { + try? FileManager.default.removeItem(at: directory) + } + + /// A fresh copy of the dev.1 fixture under `name`, as a store configuration. + private func dev1Configuration(named name: String) throws -> ModelConfiguration { + let storeURL = directory.appendingPathComponent(name) + try fixtureSQLite.write(to: storeURL, options: .atomic) + return configuration(at: storeURL) + } + + private func configuration(at storeURL: URL) -> ModelConfiguration { + ModelConfiguration( + schema: DashModelContainer.schema, + url: storeURL, + allowsSave: true, + cloudKitDatabase: .none + ) + } + + private func logLines(event: String) throws -> [String] { + SDKLogger.flush() + let log = try String( + contentsOf: directory.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + return log.split(separator: "\n").map(String.init).filter { + $0.contains("event=\(event) ") + } + } + + private func assertDev1Rows(in container: ModelContainer) throws { + let context = ModelContext(container) + let wallets = try context.fetch(FetchDescriptor()) + let accounts = try context.fetch(FetchDescriptor()) + + XCTAssertEqual(wallets.count, 1) + XCTAssertEqual(accounts.count, 1) + XCTAssertEqual(wallets[0].walletId, Data(repeating: 0xA1, count: 32)) + XCTAssertEqual(wallets[0].birthHeight, 2_400_000) + XCTAssertEqual(wallets[0].syncedHeight, 2_500_000) + XCTAssertEqual(accounts[0].accountType, 0) + XCTAssertEqual(accounts[0].accountIndex, 0) + XCTAssertEqual( + accounts[0].accountExtendedPubKeyBytes, + Data(repeating: 0x02, count: 78) + ) + XCTAssertEqual(accounts[0].wallet.walletId, wallets[0].walletId) + } + + func testDev1StoreOpensThroughProductionFactoryAndPreservesCoreRows() throws { + // The staged plan on its own is what crashes hosts today. Assert it on + // its own copy of the fixture — a failed open must not be what the + // production path below is then handed — and assert the precondition + // the fallback keys on: this store matches no registered version. The + // pair keeps naming the cause until the remaining models are frozen, + // at which point the match flips to `true` and the staged attempt + // starts succeeding. + let stagedOnly = try dev1Configuration(named: "StagedOnly.sqlite") + XCTAssertEqual(DashModelContainer.classifyStore(at: stagedOnly.url), .driftedRegisteredVersion) + XCTAssertThrowsError( + try ModelContainer( + for: DashModelContainer.schema, + migrationPlan: DashMigrationPlan.self, + configurations: [stagedOnly] + ) + ) + + let container = try DashModelContainer.open( + try dev1Configuration(named: "DashModel.sqlite") + ) + try assertDev1Rows(in: container) + + let staged = try logLines(event: "core_store_staged_migration_failed") + XCTAssertEqual(staged.count, 1, staged.joined(separator: "\n")) + let result = try XCTUnwrap(try logLines(event: "core_store_open_result").last) + XCTAssertTrue(result.contains(#"migration_path="inferred_fallback""#), result) + XCTAssertTrue(result.contains(#"result="success""#), result) + } + + /// The fallback's "self-heal" claim: once inferred migration has opened a + /// dev.1 store, it carries the current schema's checksum, so the very next + /// open must succeed through the staged plan with no fallback at all. + func testFallbackMigratedStoreReopensThroughStagedPlan() throws { + let storeURL = directory.appendingPathComponent("DashModel.sqlite") + try fixtureSQLite.write(to: storeURL, options: .atomic) + + XCTAssertEqual(DashModelContainer.classifyStore(at: storeURL), .driftedRegisteredVersion) + try autoreleasepool { + let first = try DashModelContainer.open(configuration(at: storeURL)) + try assertDev1Rows(in: first) + } + XCTAssertEqual(try logLines(event: "core_store_staged_migration_failed").count, 1) + // Inferred migration rewrote the store under the current schema, so + // it now matches a registered version and the fallback is never + // needed again. + XCTAssertEqual(DashModelContainer.classifyStore(at: storeURL), .matchesRegisteredVersion) + + let second = try DashModelContainer.open(configuration(at: storeURL)) + try assertDev1Rows(in: second) + + // Still exactly one staged failure — the reopen did not need the + // fallback — and the latest result names the staged path. + XCTAssertEqual(try logLines(event: "core_store_staged_migration_failed").count, 1) + let results = try logLines(event: "core_store_open_result") + XCTAssertEqual(results.count, 2, results.joined(separator: "\n")) + let reopen = try XCTUnwrap(results.last) + XCTAssertTrue(reopen.contains(#"migration_path="staged""#), reopen) + XCTAssertTrue(reopen.contains(#"result="success""#), reopen) + XCTAssertTrue(reopen.contains("store_existed_before_open=true"), reopen) + } + + /// Any failure other than "unknown model version" must surface untouched: + /// once a custom `MigrationStage` exists, a failure inside it reopened + /// without the plan would stamp the current checksum on a store that never + /// ran that stage, so it could never run later. + func testNonMigrationOpenFailureIsRethrownWithoutFallback() throws { + let storeURL = directory.appendingPathComponent("DashModel.sqlite") + try Data(repeating: 0x5A, count: 4096).write(to: storeURL, options: .atomic) + + // Unreadable metadata is not a version question — and not the typed + // "newer build" error either, which would send the user to update. + XCTAssertEqual(DashModelContainer.classifyStore(at: storeURL), .unreadable) + XCTAssertThrowsError(try DashModelContainer.open(configuration(at: storeURL))) { error in + XCTAssertNil(error as? DashModelContainerError, "corrupt store must not read as a newer build") + } + + XCTAssertTrue(try logLines(event: "core_store_staged_migration_failed").isEmpty) + let result = try XCTUnwrap(try logLines(event: "core_store_open_result").last) + XCTAssertTrue(result.contains(#"migration_path="staged""#), result) + XCTAssertTrue(result.contains(#"result="failure""#), result) + XCTAssertTrue(result.contains(#"store_verdict="unreadable""#), result) + // The failed store's path is redacted from the error message. + XCTAssertFalse(result.contains(storeURL.path), result) + } + + /// A store carrying an entity this SDK does not have fails the staged open + /// like a drifted store does, but must NOT be handed to inferred + /// migration: that would open it and drop the entity's table without a + /// word. The pre-fallback crash was the safe outcome, and it stays one. + /// + /// It is refused as `unplaceable`, and the refusal carries no typed error: + /// an unknown entity name does not say which build is older. This SDK + /// renamed `PersistentUtxo` to `PersistentTxo`, so a store predating that + /// rename looks exactly like this one — and telling its owner to update + /// the app or reset the wallet would be wrong and destructive. + func testStoreWithAnUnknownEntityIsRefusedWithoutFallback() throws { + let storeURL = directory.appendingPathComponent("DashModel.sqlite") + try autoreleasepool { + let newer = Schema(DashModelContainer.modelTypes + [FutureOnlyModel.self]) + let configuration = ModelConfiguration( + schema: newer, + url: storeURL, + allowsSave: true, + cloudKitDatabase: .none + ) + let container = try ModelContainer(for: newer, configurations: [configuration]) + let context = ModelContext(container) + context.insert(FutureOnlyModel(marker: 7)) + try context.save() + } + + guard case .unplaceable(let reason) = DashModelContainer.classifyStore(at: storeURL) + else { + return XCTFail("a store with an unknown entity must classify as unplaceable") + } + XCTAssertTrue(reason.contains("FutureOnlyModel"), reason) + + XCTAssertThrowsError(try DashModelContainer.open(configuration(at: storeURL))) { error in + XCTAssertNil( + error as? DashModelContainerError, + "an unknown entity name carries no direction and must not claim one: \(error)" + ) + } + XCTAssertTrue(try logLines(event: "core_store_staged_migration_failed").isEmpty) + let result = try XCTUnwrap(try logLines(event: "core_store_open_result").last) + XCTAssertTrue(result.contains(#"migration_path="staged""#), result) + XCTAssertTrue(result.contains(#"result="failure""#), result) + XCTAssertTrue(result.contains("store_verdict=\"unplaceable:unknown_entities="), result) + // And the store is untouched: still unplaceable, still refused. + guard case .unplaceable = DashModelContainer.classifyStore(at: storeURL) else { + return XCTFail("a refused open must not rewrite the store") + } + } + /// The attribute-only disagreement: a store whose + /// `PersistentWalletManagerMetadata` has one attribute the live model does + /// not, keeping V3's version identifier. Its identifier is registered and + /// every entity name is known, so only the per-entity comparison can tell + /// it from the pinned drift — and must, because inferred migration would + /// drop the attribute's values without a word. + /// + /// It is refused as `unplaceable`, NOT as a newer build: the same shape + /// arises the day an attribute is added to any unfrozen live model, where + /// every existing store is the OLDER one. So SwiftData's own error comes + /// through and the host never offers a reset off the back of it. + func testStoreWithAnAttributeOnlyNewerEntityIsRefusedWithoutFallback() throws { + let storeURL = directory.appendingPathComponent("DashModel.sqlite") + try autoreleasepool { + let newer = Schema(versionedSchema: AttributeOnlyNewerSchema.self) + let configuration = ModelConfiguration( + schema: newer, + url: storeURL, + allowsSave: true, + cloudKitDatabase: .none + ) + let container = try ModelContainer(for: newer, configurations: [configuration]) + let context = ModelContext(container) + context.insert(AttributeOnlyNewerSchema.PersistentWalletManagerMetadata( + networkRaw: 1, + futureAttribute: 42 + )) + try context.save() + } + + // The reason lists every entity that disagrees, and this fixture is a + // real store built from the live models, so any change to any of them + // adds a name — as the swept-transaction work did to + // `PersistentPendingInput`, `PersistentTxo` and `PersistentWallet`. + // Assert the verdict and the entity this test actually creates, not + // the whole list, or this becomes a tripwire for unrelated schema work + // rather than a test of the classification. + guard case .unplaceable(let reason) = + DashModelContainer.classifyStore(at: storeURL) + else { + return XCTFail("an attribute-only disagreement must be unplaceable") + } + XCTAssertTrue(reason.hasPrefix("unexpected_entity_drift="), reason) + XCTAssertTrue(reason.contains("PersistentWalletManagerMetadata"), reason) + XCTAssertThrowsError(try DashModelContainer.open(configuration(at: storeURL))) { error in + XCTAssertNil( + error as? DashModelContainerError, + "a disagreement with no direction must not claim a newer build: \(error)" + ) + } + XCTAssertTrue(try logLines(event: "core_store_staged_migration_failed").isEmpty) + let result = try XCTUnwrap(try logLines(event: "core_store_open_result").last) + XCTAssertTrue(result.contains(#"result="failure""#), result) + XCTAssertTrue( + result.contains("store_verdict=\"unplaceable:unexpected_entity_drift="), + result + ) + XCTAssertTrue(result.contains("PersistentWalletManagerMetadata"), result) + } + + /// `knownDriftedEntityHashes` must be exactly what the fixture shows, no + /// wider and byte for byte: every entity it names disagrees with V1's + /// model for this store, none disagrees that it does not name, and each + /// listed hash is the one the store carries. When a shape gets frozen, + /// this is the test that says to shrink the table. + func testKnownDriftedEntityHashesArePinnedToTheFixture() throws { + let storeURL = try dev1Configuration(named: "Pin.sqlite").url + let metadata = try NSPersistentStoreCoordinator.metadataForPersistentStore( + ofType: NSSQLiteStoreType, at: storeURL, options: nil + ) + let storeHashes = try XCTUnwrap(metadata[NSStoreModelVersionHashesKey] as? [String: Data]) + let identifiers = try XCTUnwrap(metadata[NSStoreModelVersionIdentifiersKey] as? [String]) + XCTAssertEqual(identifiers, [DashSchemaV1.versionIdentifier.description]) + + let v1 = try XCTUnwrap(NSManagedObjectModel.makeManagedObjectModel(for: DashSchemaV1.models)) + let disagreeing = Set(storeHashes.compactMap { name, hash in + v1.entityVersionHashesByName[name] == hash ? nil : name + }) + let known = DashModelContainer.knownDriftedEntityHashes + XCTAssertEqual( + disagreeing, Set(known.keys), + "fixture drifts on \(disagreeing.sorted()); the table must name exactly those" + ) + for (name, hash) in known { + XCTAssertEqual(storeHashes[name], hash, "\(name): the pinned hash must be the store's") + } + } + + /// The decision on plain values: which stores the fallback may answer. + func testStoreSchemaVerdictOnPlainValues() { + let a = Data([1]), b = Data([2]) + let current: Set = ["PersistentWallet", "PersistentDocumentType", "PersistentIndex"] + let v1 = DashModelContainer.RegisteredVersionHashes( + identifier: "1.0.0", + entityHashes: ["PersistentWallet": a, "PersistentDocumentType": a, "PersistentIndex": a] + ) + func verdict( + _ store: [String: Data], + identifiers: [String] = ["1.0.0"], + matches: Bool = false + ) -> DashModelContainer.StoreSchemaVerdict { + DashModelContainer.storeSchemaVerdict( + matchesRegisteredVersion: matches, + storeEntityHashes: store, + storeVersionIdentifiers: identifiers, + registered: [v1], + currentEntities: current + ) + } + + // A compatible store is never inspected further. + XCTAssertEqual(verdict(["PersistentWallet": b], matches: true), .matchesRegisteredVersion) + // Drift confined to the known entities WITH their known shapes may be + // migrated; the same entities with any other shape may not. + let knownDocumentType = DashModelContainer.knownDriftedEntityHashes["PersistentDocumentType"]! + let knownIndex = DashModelContainer.knownDriftedEntityHashes["PersistentIndex"]! + XCTAssertEqual( + verdict(["PersistentWallet": a, "PersistentDocumentType": knownDocumentType, "PersistentIndex": knownIndex]), + .driftedRegisteredVersion + ) + XCTAssertEqual( + verdict(["PersistentWallet": a, "PersistentDocumentType": knownDocumentType, "PersistentIndex": a]), + .driftedRegisteredVersion, + "one drifted entity with its known shape, the other untouched" + ) + XCTAssertEqual( + verdict(["PersistentWallet": a, "PersistentDocumentType": b, "PersistentIndex": knownIndex]), + .unplaceable(reason: "unexpected_entity_drift=PersistentDocumentType"), + "a known entity with an unknown shape is not the pinned drift — and not evidence of direction" + ) + // The attribute-only downgrade: same names, kept identifier, but the + // disagreement is on an entity that is not known to have drifted. + XCTAssertEqual( + verdict(["PersistentWallet": b, "PersistentDocumentType": a, "PersistentIndex": a]), + .unplaceable(reason: "unexpected_entity_drift=PersistentWallet") + ) + // Mixed: known drift plus one unexpected entity still refuses. + XCTAssertEqual( + verdict(["PersistentWallet": b, "PersistentDocumentType": knownDocumentType, "PersistentIndex": a]), + .unplaceable(reason: "unexpected_entity_drift=PersistentWallet") + ) + XCTAssertEqual( + DashModelContainer.storeSchemaVerdict( + matchesRegisteredVersion: false, + storeEntityHashes: ["PersistentWallet": a], + storeVersionIdentifiers: ["1.0.0"], + registered: [], + currentEntities: current + ), + .unplaceable(reason: "no_registered_models"), + "no model built from any schema is a fact about this build, not the store" + ) + XCTAssertEqual( + verdict(["PersistentWallet": a], identifiers: ["9.0.0"]), + .unplaceable(reason: "unregistered_version_identifier=9.0.0"), + "an identifier we do not register may be pre-V1 or de-registered, not only future" + ) + // Unplaceable, NOT a newer build: `open` must rethrow SwiftData's own + // error for these rather than tell the user their wallet came from a + // newer app and offer a reset. + XCTAssertEqual( + verdict(["PersistentWallet": a], identifiers: []), + .unplaceable(reason: "no_version_identifier") + ) + XCTAssertEqual( + verdict([:]), + .unplaceable(reason: "no_entity_hashes"), + "no hashes means nothing was compared; drift may not be claimed" + ) + XCTAssertEqual( + verdict(["PersistentWallet": a]), + .unplaceable(reason: "no_entity_disagreement"), + "every hash the store carries agrees and it still is not compatible — " + + "it differs by something these hashes do not describe, not by the pinned drift" + ) + XCTAssertEqual( + verdict(["PersistentWallet": a, "FutureOnlyModel": a]), + .unplaceable(reason: "unknown_entities=FutureOnlyModel"), + "an entity this schema lacks may be a newer build's addition or an " + + "older build's since-renamed model (PersistentUtxo -> PersistentTxo)" + ) + } +} + +/// An entity no registered SDK schema has — what a store written by a future +/// build looks like to this one. +@Model +final class FutureOnlyModel { + var marker: Int + + init(marker: Int) { + self.marker = marker + } +} + +/// V3 exactly as a newer build would write it: the same version identifier +/// and the same entity set, with one attribute added to a relationship-free +/// entity. Nested so the clone shares the live entity's name (SwiftData +/// derives it from the unqualified type name) without touching the live type. +enum AttributeOnlyNewerSchema: VersionedSchema { + static var versionIdentifier: Schema.Version { DashSchemaV3.versionIdentifier } + + static var models: [any PersistentModel.Type] { + DashModelContainer.modelTypes.filter { + ObjectIdentifier($0) != ObjectIdentifier(SwiftDashSDK.PersistentWalletManagerMetadata.self) + } + [PersistentWalletManagerMetadata.self] + } + + @Model + final class PersistentWalletManagerMetadata { + @Attribute(.unique) var networkRaw: UInt32 + var combinedSyncHeight: UInt32 + var combinedSyncBlockHash: Data? + var walletCount: Int + var createdAt: Date + var lastUpdated: Date + /// The one thing this build has that the SDK's model does not. + var futureAttribute: Int + + init(networkRaw: UInt32, futureAttribute: Int) { + self.networkRaw = networkRaw + self.combinedSyncHeight = 0 + self.combinedSyncBlockHash = nil + self.walletCount = 0 + self.createdAt = Date() + self.lastUpdated = Date() + self.futureAttribute = futureAttribute + } + } +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/DashModel-v4.2.0-dev.1.sqlite.zlib b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/DashModel-v4.2.0-dev.1.sqlite.zlib new file mode 100644 index 00000000000..836d5beeb5a Binary files /dev/null and b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/DashModel-v4.2.0-dev.1.sqlite.zlib differ diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/README.md b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/README.md new file mode 100644 index 00000000000..1838819f6e0 --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/Fixtures/README.md @@ -0,0 +1,17 @@ +# v4.2.0-dev.1 migration fixture + +`DashModel-v4.2.0-dev.1.sqlite.zlib` is a synthetic SwiftData store created +from the exact `v4.2.0-dev.1` model sources. It contains one wallet and one +BIP44 account with non-secret marker bytes; it contains no production wallet +material. + +- uncompressed SQLite size: `647168` bytes +- uncompressed SHA-256: `17c2e93e655b79c43d023f41a4a4360e511d8f97af56aedfce32bd73c0158e58` +- compression: Foundation `NSData.CompressionAlgorithm.zlib` + +The regression test opens a copy through `DashModelContainer.open` — the same +entry point every host uses — and verifies that the Core wallet records survive. +It also asserts that the staged `DashMigrationPlan` alone still rejects this +store, which is the reason that entry point falls back to inferred lightweight +migration; see `DashSchemaFrozenModels.swift` for why the V1/V2 checksums no +longer match a dev.1 store. diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/PlatformWalletShutdownTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/PlatformWalletShutdownTests.swift index 081b070a955..52ff0b9cfad 100644 --- a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/PlatformWalletShutdownTests.swift +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/PlatformWalletShutdownTests.swift @@ -141,6 +141,65 @@ final class PlatformWalletShutdownTests: XCTestCase { XCTAssertEqual(metrics.steps.map(\.name), Self.expectedOrder) } + func testDiagnosticsDoNotBlockSyncAdmissionButShutdownWaitsForThem() async throws { + let recorder = TeardownRecorder() + let manager = PlatformWalletManager.makeForTesting( + handle: 19, + calls: Self.makeCalls(recorder: recorder) + ) + + try manager.admitCoreDiagnosticsNativeOp() + + // Diagnostics only perform read-only FFI work. A synchronous wallet + // operation must therefore pass native-op admission while diagnostics + // are active. The empty seed then fails at its own validation seam, + // proving admission did not reject it as an overlapping native op. + XCTAssertThrowsError( + try manager.createWallet(seed: Data(), network: .testnet) + ) { error in + guard let walletError = error as? PlatformWalletError, + case .invalidParameter = walletError + else { + return XCTFail("expected invalidParameter, got \(error)") + } + } + + let shutdownTask = Task { await manager.shutdown() } + try await Task.sleep(for: .milliseconds(20)) + XCTAssertEqual(manager.handle, 19, "shutdown must not take the handle early") + XCTAssertTrue(recorder.names.isEmpty, "native teardown must wait for diagnostics") + + manager.finishCoreDiagnosticsNativeOp() + let metrics = await shutdownTask.value + + XCTAssertEqual(manager.handle, NULL_HANDLE) + XCTAssertEqual(metrics.steps.map(\.name), Self.expectedOrder) + XCTAssertEqual(recorder.names, Self.expectedOrder) + } + + func testUnbalancedDiagnosticsFinishCannotUnderflowShutdownCounter() async throws { + let recorder = TeardownRecorder() + let manager = PlatformWalletManager.makeForTesting( + handle: 20, + calls: Self.makeCalls(recorder: recorder) + ) + + // A stray release at zero must be ignored. Without the guard this + // makes the next admission look idle and lets shutdown take its handle. + manager.finishCoreDiagnosticsNativeOp() + try manager.admitCoreDiagnosticsNativeOp() + + let shutdownTask = Task { await manager.shutdown() } + try await Task.sleep(for: .milliseconds(20)) + XCTAssertEqual(manager.handle, 20) + XCTAssertTrue(recorder.names.isEmpty) + + manager.finishCoreDiagnosticsNativeOp() + let metrics = await shutdownTask.value + XCTAssertEqual(metrics.steps.map(\.name), Self.expectedOrder) + XCTAssertEqual(recorder.names, Self.expectedOrder) + } + /// A completed real shutdown makes this manager terminal. Reconfiguration /// must fail before another native handle or callback context is installed. func testConfigurationAfterRealShutdownIsRejected() async { @@ -245,4 +304,16 @@ final class PlatformWalletShutdownTests: XCTestCase { ) XCTAssertTrue(metrics.ranOffMainThread) } + + /// The flag `shutdown()` raises is one-way and readable off the main + /// actor: a diagnostic pass polls it before each FFI read. + func testCoreDiagnosticsCancellationIsOneWay() { + let token = CoreDiagnosticsCancellation() + XCTAssertFalse(token.isCancelled) + token.cancel() + XCTAssertTrue(token.isCancelled) + token.cancel() + XCTAssertTrue(token.isCancelled, "a second cancel must not flip it back") + } + } diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/SDKLoggerPreInstallBufferTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/SDKLoggerPreInstallBufferTests.swift new file mode 100644 index 00000000000..b2af9381abc --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/SDKLoggerPreInstallBufferTests.swift @@ -0,0 +1,99 @@ +import Foundation +import XCTest + +@testable import SwiftDashSDK + +/// The file sink is installed by `LoggingPreferences.configure()`, well after +/// the host has already opened its store in `init()`. Everything emitted in +/// between must reach `swift/run.log` in order once the sink exists — and a +/// host that never installs one must not grow the backlog without bound. +/// +/// Tested on a fresh `SDKLoggerState` rather than through `SDKLogger`: the +/// process-wide singleton has no way back to "no sink installed" once any +/// test has installed one. +final class SDKLoggerPreInstallBufferTests: XCTestCase { + private var session: URL! + + override func setUpWithError() throws { + session = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: session, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: session) + } + + private func writtenLines(_ state: SDKLoggerState) throws -> [String] { + state.flush() + let log = try String( + contentsOf: session.appendingPathComponent("swift/run.log"), + encoding: .utf8 + ) + return log.split(separator: "\n").map(String.init) + } + + func testLinesRecordedBeforeInstallReplayInOrderAheadOfLaterLines() throws { + let state = SDKLoggerState() + state.record(severity: .info, line: "first") + state.record(severity: .warning, line: "second") + + let outcome = state.installSink(at: session, includeDebug: false) + XCTAssertTrue(outcome.installed) + XCTAssertEqual(outcome.droppedPendingLineCount, 0) + + state.record(severity: .error, line: "third") + XCTAssertEqual(try writtenLines(state), ["first", "second", "third"]) + } + + func testReplayHonoursTheSinkDebugSettingItDidNotKnowYet() throws { + // Before install nothing knows whether debug lines are wanted, so + // they are buffered regardless and filtered at replay. + let hidden = SDKLoggerState() + hidden.record(severity: .debug, line: "debug") + hidden.record(severity: .info, line: "info") + _ = hidden.installSink(at: session, includeDebug: false) + XCTAssertEqual(try writtenLines(hidden), ["info"]) + + try FileManager.default.removeItem(at: session.appendingPathComponent("swift")) + let shown = SDKLoggerState() + shown.record(severity: .debug, line: "debug") + shown.record(severity: .info, line: "info") + _ = shown.installSink(at: session, includeDebug: true) + XCTAssertEqual(try writtenLines(shown), ["debug", "info"]) + } + + /// Head-not-tail: the store-open line is the first in, and it is the one + /// the buffer exists to carry, so overflow discards the newest arrival. + func testBufferKeepsTheHeadAndDropsTheNewestAboveTheLimit() throws { + let state = SDKLoggerState() + let limit = SDKLoggerState.pendingLineLimit + // limit + 1 lines: exactly one over. + for index in 0...limit { + state.record(severity: .info, line: "line-\(index)") + } + + let outcome = state.installSink(at: session, includeDebug: false) + XCTAssertTrue(outcome.installed) + // This count is what `SDKLogger.installFileSink` turns into the + // `log_pre_install_buffer_overflow` event after the replay. + XCTAssertEqual(outcome.droppedPendingLineCount, 1) + + let lines = try writtenLines(state) + XCTAssertEqual(lines.count, limit) + XCTAssertEqual(lines.first, "line-0", "the first line in survives") + XCTAssertEqual(lines.last, "line-\(limit - 1)", "the newest arrival is the one dropped") + } + + func testBufferIsClearedByInstallSoASecondInstallReplaysNothing() throws { + let state = SDKLoggerState() + state.record(severity: .info, line: "once") + _ = state.installSink(at: session, includeDebug: false) + XCTAssertEqual(try writtenLines(state), ["once"]) + + try FileManager.default.removeItem(at: session.appendingPathComponent("swift")) + let outcome = state.installSink(at: session, includeDebug: false) + XCTAssertEqual(outcome.droppedPendingLineCount, 0) + XCTAssertEqual(try writtenLines(state), []) + } +}