diff --git a/DESIGN.md b/DESIGN.md index 8b8a19dcbd..5cebfed188 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -194,14 +194,20 @@ If the table needs `audit: true`, set it both in the schema (for fresh installs) A table is a set of RocksDB column families (`T/` plus `T/`) and a set of catalog rows in the `__dbis__` store, with no transaction spanning the two. `Table.dropTable()` therefore -persists a `dropping: true` flag on the table's primary catalog entry (`T/`) before any -destructive work, then drops the column families (awaited - a failed drop must surface as the -operation's error, never a swallowed rejection), then removes the catalog rows. If the process -dies or a drop fails partway, the tombstone survives; both the boot-time schema load in -`databases.ts` (`completeInterruptedDrop`) and a same-name `table()` create complete the -interrupted drop instead of resurrecting the table. Without this, surviving catalog rows are -silently re-opened with create-if-missing on the next start, which resurrects "deleted" tables -(with their data, if the column families were never actually removed). +persists a `dropping: true` flag, a unique `dropGeneration`, and the current process incarnation on +the table's primary catalog entry (`T/`) before any destructive work. It then starts the local drain +and strict cross-worker barrier together, so every worker stops admission even if the coordinator's +own drain fails. Each worker drains table reads, writes, scans, and index backfills and closes its +column-family handles. Only after every worker acknowledges does the coordinator set +`dropQuiesced: true`, drop the column families, and remove the catalog rows. + +The tombstone survives a partial failure. An unquiesced tombstone from the current process is not +safe to complete or recreate because another worker may still hold a handle; the table remains +unavailable and Harper must restart. After a clean restart the process incarnation differs, so +boot-time reconciliation can finish the drop. A same-name `table()` create may finish only a +quiesced drop or one left by an older process. These checks prevent surviving catalog rows from +being re-opened with create-if-missing and resurrecting a deleted table or an undiscoverable +"ghost" column family. ## MCP protocol surface (`components/mcp/`) diff --git a/package-lock.json b/package-lock.json index b8986f2b12..394a8a28f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1017,11 +1017,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-darwin-x64": { "version": "2.2.2", @@ -1030,11 +1032,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm": { "version": "2.2.2", @@ -1043,11 +1047,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm64": { "version": "2.2.2", @@ -1056,11 +1062,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-x64": { "version": "2.2.2", @@ -1069,11 +1077,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-win32-x64": { "version": "2.2.2", @@ -1082,11 +1092,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@colors/colors": { "version": "1.5.0", @@ -2560,9 +2572,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2579,9 +2588,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2598,9 +2604,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2617,9 +2620,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3336,11 +3336,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { "version": "3.0.4", @@ -3349,11 +3351,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { "version": "3.0.4", @@ -3362,11 +3366,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { "version": "3.0.4", @@ -3375,11 +3381,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { "version": "3.0.4", @@ -3388,11 +3396,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { "version": "3.0.4", @@ -3401,11 +3411,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@noble/hashes": { "version": "1.8.0", @@ -3574,9 +3586,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3594,9 +3603,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3614,9 +3620,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3634,9 +3637,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3654,9 +3654,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3674,9 +3671,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3694,9 +3688,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3714,9 +3705,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index ac8a68e770..9d40dc7fb6 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -9,7 +9,12 @@ import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { when } from '../utility/when.ts'; import { setTimeout as delay } from 'node:timers/promises'; -import { Transaction as RocksTransaction, type Store as RocksStore, constants } from '@harperfast/rocksdb-js'; +import { + RocksDatabase, + Transaction as RocksTransaction, + type Store as RocksStore, + constants, +} from '@harperfast/rocksdb-js'; const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE; import type { RootDatabaseKind } from './databases.ts'; import type { Entry } from './RecordEncoder.ts'; @@ -21,6 +26,8 @@ const trackedTxns = new Set(); // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the // chain root — so a chain child can never become its own timeout root (issue #2231). const supervisedWriteRoots = new Set(); +const activeWriteTransactions = new Set>(); +const terminalReplayCommitFailure = Symbol('terminalReplayCommitFailure'); const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); export const TRANSACTION_STATE = { @@ -139,6 +146,39 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number | oldestAgeMs: oldestOutstandingCommit ? performance.now() - oldestOutstandingCommit.start : undefined, }; } + +/** + * Snapshot the transactions on this worker that have staged writes against any of the supplied stores. + * A table drop marks its stores as dropping before calling this, so no later addWrite can enter the set; + * awaiting the returned promises therefore establishes a closed drain boundary before handles are closed. + */ +export function getPendingWriteResolutions(stores: Iterable): Promise[] { + const targetStores = new Set(stores); + const resolutions: Promise[] = []; + for (const reference of activeWriteTransactions) { + const transaction = reference.deref(); + if (!transaction) { + activeWriteTransactions.delete(reference); + continue; + } + if (transaction.writes.some((write) => write && targetStores.has(write.store))) { + const resolution = transaction.getPendingWriteResolution(); + if (resolution) resolutions.push(resolution.finally(() => void transaction)); + } + } + return resolutions; +} + +export function getPendingReadResolutions(stores: Iterable): Promise[] { + const targetStores = new Set(stores); + const resolutions: Promise[] = []; + for (const transaction of trackedTxns) { + if (!transaction.usesAnyStore(targetStores)) continue; + const resolution = transaction.getPendingReadResolution(); + if (resolution) resolutions.push(resolution); + } + return resolutions; +} // Once per process: committing under open read iterators forces a write replay, so the warning is // about the caller's pattern, not the individual commit. let replayedWritesWarned = false; @@ -325,6 +365,14 @@ type RocksTransactionWithRetry = RocksTransaction & { isRetry?: boolean }; export class DatabaseTransaction implements Transaction { #context: Context; + #pendingWriteResolution?: Promise; + #resolvePendingWrites?: () => void; + #pendingReadResolution?: Promise; + #resolvePendingReads?: () => void; + // `db` identifies the first table; allocate only when one native transaction spans more tables. + #additionalStores?: Set; + #lastTrackedStore?: any; + #dropDrainReference?: WeakRef; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) declare writesByKey?: Map>; @@ -496,8 +544,10 @@ export class DatabaseTransaction implements Transaction { // loop the handle can still hold write intents, and stalled writers with a clean log is // the worst outcome here. harperLogger.warn?.('Failed to release a transaction’s native handle', error); + } finally { + this.finishPendingReads(); + this.completeDeferredContextRelease(); } - this.completeDeferredContextRelease(); } } @@ -514,6 +564,14 @@ export class DatabaseTransaction implements Transaction { } catch (error) { harperLogger.debug?.('releasing timed-out read transaction', error); } + this.finishPendingReads(); + this.completeDeferredContextRelease(); + } + + /** Finish wrapper bookkeeping after the caller has already settled the native handle. */ + completeReadTxn(): void { + this.detachOwnedTransaction(); + this.finishPendingReads(); this.completeDeferredContextRelease(); } @@ -546,6 +604,20 @@ export class DatabaseTransaction implements Transaction { * (see priorStagedWrite). Called by both engines' addWrite. */ linkWrite(operation: TransactionWrite): void { + if (operation.store?.dropping) { + const databaseName = operation.store.rootStore?.databaseName; + const tableName = String(operation.store.name ?? '').replace(/\/$/, ''); + const error: any = new ServerError( + `Table ${databaseName ? databaseName + '.' : ''}${tableName || 'unknown'} is being dropped`, + 409 + ); + error.code = 'ERR_TABLE_DROPPING'; + throw error; + } + if (!this.#dropDrainReference && operation.store?.rootStore instanceof RocksDatabase) { + const reference = (this.#dropDrainReference = new WeakRef(this)); + activeWriteTransactions.add(reference); + } if (operation.key === undefined) return; let writesForStore = (this.writesByKey ??= new Map()).get(operation.store); if (!writesForStore) this.writesByKey.set(operation.store, (writesForStore = new Map())); @@ -560,10 +632,74 @@ export class DatabaseTransaction implements Transaction { * reused transaction never bases a write on a previous batch's staged state. */ clearWrites(): void { + this.finishPendingWrites(); this.writes = []; this.writesByKey = undefined; } + getPendingWriteResolution(): Promise | undefined { + if (!this.#dropDrainReference) return; + this.#pendingWriteResolution ??= new Promise((resolve) => { + this.#resolvePendingWrites = resolve; + }); + return this.#pendingWriteResolution; + } + + getPendingReadResolution(): Promise | undefined { + if (!this.transaction || !(this.readTxnsUsed > 0)) return; + this.#pendingReadResolution ??= new Promise((resolve) => { + this.#resolvePendingReads = resolve; + }); + return this.#pendingReadResolution; + } + + trackStore(store: any): void { + if (this.db !== store && this.#lastTrackedStore !== store) { + this.#lastTrackedStore = store; + (this.#additionalStores ??= new Set()).add(store); + } + } + + usesAnyStore(stores: Set): boolean { + if (stores.has(this.db)) return true; + for (const store of this.#additionalStores ?? []) { + if (stores.has(store)) return true; + } + return false; + } + + hasWritesForAnyStore(stores: Set): boolean { + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + if (transaction.writes.some((write) => write && stores.has(write.store))) return true; + } + return false; + } + + hasOpenReadsForAnyStore(stores: Set): boolean { + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + if (transaction.transaction && transaction.readTxnsUsed > 0 && transaction.usesAnyStore(stores)) return true; + } + return false; + } + + private finishPendingWrites(): void { + if (this.#dropDrainReference) { + activeWriteTransactions.delete(this.#dropDrainReference); + this.#dropDrainReference = undefined; + } + this.#resolvePendingWrites?.(); + this.#pendingWriteResolution = undefined; + this.#resolvePendingWrites = undefined; + } + + private finishPendingReads(): void { + this.#resolvePendingReads?.(); + this.#pendingReadResolution = undefined; + this.#resolvePendingReads = undefined; + this.#additionalStores = undefined; + this.#lastTrackedStore = undefined; + } + /** * Drop this transaction's back-reference from its context once completed (commit or abort), * so a long-lived context (e.g. an MQTT subscription context held open for the life of a @@ -858,7 +994,11 @@ export class DatabaseTransaction implements Transaction { } // with options.transaction set this is a retry round — the save loop above already // re-staged the writes into it - commitResolution = transaction.commit() as Promise; + try { + commitResolution = transaction.commit() as Promise; + } catch (error) { + commitResolution = this.abortSynchronousReplayCommit(transaction, error, !!options.doneWriting); + } recordCommitLatency(commitResolution, performance.now()); // Write-queue-depth accounting for this replay commit happens uniformly below, via // trackOutstandingCommit(commitResolution) — see that function's comment. Omitting @@ -886,6 +1026,7 @@ export class DatabaseTransaction implements Transaction { } else { // no more reads need to be performed, just commit/abort based if there are any writes this.detachOwnedTransaction(); // any further operations operate immediately + this.finishPendingReads(); if (transaction) { this.writes = this.writes.filter((write) => write); // filter out removed entries if (this.writes.length > 0) { @@ -893,7 +1034,11 @@ export class DatabaseTransaction implements Transaction { // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That // sentinel (a number) is why commitResolution is typed // Promise; it is handled in the resolve callback below. - commitResolution = transaction.commit(); + try { + commitResolution = transaction.commit(); + } catch (error) { + this.abortSynchronousCommit(transaction, error); + } // Record how long this commit stays outstanding (submit → settle) as a distribution // metric. This is the same clock the overload check uses (trackOutstandingCommit // stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the @@ -1021,7 +1166,10 @@ export class DatabaseTransaction implements Transaction { // migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was // retried here, the rejection propagated out of the unawaited onCommit() handler as an // unhandled rejection and the write was silently dropped — records lost mid-copy (#308). - if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') { + if ( + !error[terminalReplayCommitFailure] && + (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') + ) { // if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry // for future transactions this.retries++; @@ -1079,6 +1227,32 @@ export class DatabaseTransaction implements Transaction { } catch (abortError) { harperLogger.debug?.('aborting transaction after failed commit', abortError); } + const nextTransaction = this.next; + this.next = null; + for ( + let linkedTransaction = nextTransaction; + linkedTransaction; + linkedTransaction = linkedTransaction.next + ) { + try { + linkedTransaction.abortLink(); + } catch (abortError) { + harperLogger.debug?.('aborting linked transaction after failed commit', abortError); + } + } + try { + for (const write of this.writes) { + if (write?.savedBlobs) + cleanupUnusedBlobs( + write.savedBlobs, + collectRetainedFileIds(write.store.getEntry(write.key)?.value) + ); + } + } catch (cleanupError) { + harperLogger.debug?.('cleaning up writes after failed commit', cleanupError); + } finally { + this.clearWrites(); + } // A terminal failure is just as final as a success — release the context's // back-reference here too, or transaction.ts's onComplete() (which has no // rejection handler of its own) would leave a long-lived context pinning this @@ -1115,23 +1289,42 @@ export class DatabaseTransaction implements Transaction { return txnResolution; }, (error) => { - this.abort(); + try { + this.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting transaction after failed commit', abortError); + } throw error; } ); } abort(): void { - while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it - // Defensively release any native handle whose reference bookkeeping was already consumed. - if (this.transaction) this.releaseReadTxn(); + let firstError: unknown; + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + try { + transaction.abortLink(); + } catch (error) { + firstError ??= error; + } + } + if (firstError) throw firstError; + } + private abortLink(): void { this.open = TRANSACTION_STATE.CLOSED; - this.drainCompletions(); try { + while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it + // A write-only transaction never took a read reference (getReadTxn was never called), so the loop + // above releases nothing even though save() created a native handle; release it here instead of + // leaking the handle and its snapshot until GC. abortChainAfterRetries() detaches the handle + // before calling abort(), so this is a no-op there rather than a double-abort. + if (this.transaction) this.releaseReadTxn(); + this.drainCompletions(); for (const write of this.writes) { if (write?.savedBlobs) cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value)); } } finally { + // reset the transaction even if blob inspection fails this.clearWrites(); // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free": // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of @@ -1140,16 +1333,67 @@ export class DatabaseTransaction implements Transaction { // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. this.releaseContext(!this.timedOut); - const next = this.next; - this.next = null; - if (next) { - try { - next.abort(); - } catch (error) { - harperLogger.debug?.('cleaning up a chained transaction during abort', error); - } + } + } + private abortSynchronousReplayCommit(transaction: RocksTransaction, error: unknown, final: boolean): Promise { + try { + transaction.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting replay transaction after synchronous commit failure', abortError); + } + if (!this.writesAbandoned) { + this.writesAbandoned = true; + try { + this.transaction?.abandonWrites?.(); + } catch (abandonError) { + harperLogger.debug?.('abandoning retained writes after synchronous replay commit failure', abandonError); + } + } + try { + for (const write of this.writes) { + if (write?.savedBlobs) + cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value)); + } + } catch (cleanupError) { + harperLogger.debug?.('cleaning up blobs after synchronous replay commit failure', cleanupError); + } finally { + this.clearWrites(); + this.releaseContext(final); + } + const nextTransaction = this.next; + this.next = null; + for (let linkedTransaction = nextTransaction; linkedTransaction; linkedTransaction = linkedTransaction.next) { + try { + linkedTransaction.abortLink(); + } catch (abortError) { + harperLogger.debug?.('aborting linked transaction after synchronous replay commit failure', abortError); } } + // Keep the retained read handle alive until its iterators drain, but do not let a retryable native + // code re-enter commit after the staged writes above were discarded. + const terminalError: any = new Error(error instanceof Error ? error.message : String(error), { cause: error }); + if (error instanceof Error) { + const originalStack = error.stack; + Object.setPrototypeOf(terminalError, Object.getPrototypeOf(error)); + Object.defineProperties(terminalError, Object.getOwnPropertyDescriptors(error)); + Object.defineProperty(terminalError, 'cause', { configurable: true, value: error }); + Object.defineProperty(terminalError, 'stack', { configurable: true, value: originalStack, writable: true }); + } + Object.defineProperty(terminalError, terminalReplayCommitFailure, { value: true }); + return Promise.reject(terminalError); + } + private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { + try { + transaction.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting native transaction after synchronous commit failure', abortError); + } + try { + this.abort(); + } catch (abortError) { + harperLogger.debug?.('cleaning up transaction after synchronous commit failure', abortError); + } + throw error; } /** * Give up on a chain of linked transactions after exhausting conflict retries: poison every link @@ -1170,6 +1414,7 @@ export class DatabaseTransaction implements Transaction { // loop cannot spin on a nulled handle. Not to avoid a double abort: rocksdb-js tolerates // abort-after-abort, and it is abort-after-COMMIT that throws. const detached = txn.detachOwnedTransaction(); + txn.finishPendingReads(); const committingTransaction = txn === this ? headTransaction : detached; try { committingTransaction?.abort(); @@ -1189,7 +1434,7 @@ export class DatabaseTransaction implements Transaction { // abort() synchronously walks savedBlobs and can call write.store.getEntry(), which can throw // (closed store, decode error). Catch and continue so one link's wrapper-cleanup failure can't // strand later links' native handles — they were already detached/aborted above regardless. - txn.abort(); + txn.abortLink(); } catch (abortError) { harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError); } @@ -1225,7 +1470,7 @@ export class DatabaseTransaction implements Transaction { } for (let txn: DatabaseTransaction = this; txn; txn = txn.next) { try { - txn.abort(); + txn.abortLink(); } catch (error) { harperLogger.debug?.(`Error aborting timed-out transaction in chain: ${error.message}`); } @@ -1251,6 +1496,9 @@ export class DatabaseTransaction implements Transaction { harperLogger.debug?.('cleaning up after a failed synchronous commit', abortError); } throw error; + } finally { + this.finishPendingReads(); + this.finishPendingWrites(); } this.detachOwnedTransaction(); } diff --git a/resources/Table.ts b/resources/Table.ts index dc4a2836e4..dc768ebe23 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -10,6 +10,7 @@ import { SYSTEM_TABLE_NAMES, SYSTEM_SCHEMA_NAME, MAX_SET_TIMEOUT_MS, + TABLE_DROP_PREPARE_OPERATION, } from '../utility/hdbTerms.ts'; import { type Database } from 'lmdb'; import { Script } from 'node:vm'; @@ -38,6 +39,8 @@ import { priorStagedWrite, isReleasedTransaction, TRANSACTION_STATE, + getPendingReadResolutions, + getPendingWriteResolutions, } from './DatabaseTransaction.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; import { addSubscription } from './transactionBroadcast.ts'; @@ -51,7 +54,7 @@ import { } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js'; -import { databases, table } from './databases.ts'; +import { databases, table, prepareTableDrop } from './databases.ts'; import { searchByIndex, findAttribute, @@ -66,7 +69,7 @@ import { isStaticResourceInstance } from './staticResourceDispatch.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount, getProcessInstanceId } from '../server/threads/manageThreads.js'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; @@ -80,7 +83,7 @@ import { onStorageReclamation, getStorageSpaceStats } from '../server/storageRec import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { throttle } from '../server/throttle.ts'; -import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js'; +import { RocksDatabase, Transaction as RocksTransaction, constants as rocksConstants } from '@harperfast/rocksdb-js'; import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction'; import { contentTypes } from '../server/serverHelpers/contentTypes'; import { type JsonSchemaFragment, projectAttributesToProperties } from './jsonSchemaTypes.ts'; @@ -125,6 +128,7 @@ const RECORD_PRUNING_INTERVAL = 60000; // one minute // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path. const EVICTION_BATCH_SIZE = 100; +const MAX_INFLIGHT_MAINTENANCE_REMOVALS = 50; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; @@ -403,7 +407,102 @@ export function makeTable(options) { // in-flight commit promises here so dropTable() can drain them first, and stop admitting // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); + const pendingTableOperations = new Set<{ + completion: Promise; + label: string; + cancel?: () => void; + }>(); let droppingTable = false; + let coordinatingDrop = false; + let storesClosed = false; + let dropPreparation: Promise | undefined; + let deleteCallbackHandle: { remove: () => void } | undefined; + const tableStores = () => [...Object.values(indices), primaryStore].filter(Boolean); + const tableDroppingError = () => { + const error: any = new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 409); + error.code = 'ERR_TABLE_DROPPING'; + return error; + }; + // Rocks operations that bypass txnForContext must hold this token across every yield so drop can see them. + const beginTableOperation = (label: string, cancel?: () => void) => { + if (!isRocksDB) return () => {}; + if (droppingTable) throw tableDroppingError(); + let resolve: () => void; + const completion = new Promise((scanResolve) => { + resolve = scanResolve; + }); + const operation = { completion, label, cancel }; + pendingTableOperations.add(operation); + let active = true; + return () => { + if (!active) return; + active = false; + pendingTableOperations.delete(operation); + resolve(); + }; + }; + const stopCleanupTimer = () => { + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + cleanupTimerCompletion?.resolve(); + cleanupTimerCompletion = undefined; + } + }; + const stopBackgroundScans = () => { + stopCleanupTimer(); + if (recordExpirationInterval) clearInterval(recordExpirationInterval); + deleteCallbackHandle?.remove(); + }; + const markTableDropping = () => { + droppingTable = true; + stopBackgroundScans(); + for (const operation of [...pendingTableOperations]) { + try { + operation.cancel?.(); + } catch (error) { + logger.warn?.(`Unable to cancel ${operation.label} while dropping ${databaseName}.${tableName}`, error); + } + } + if (isRocksDB) { + for (const store of tableStores()) (store as any).dropping = true; + } + delete databases[databaseName]?.[tableName]; + }; + const drainTableOperations = async () => { + const directOperations = [...pendingTableOperations]; + const pending = new Set>([ + ...pendingSourceCommits, + ...directOperations.map(({ completion }) => completion), + ...getPendingWriteResolutions(tableStores()), + ...getPendingReadResolutions(tableStores()), + ]); + const indexingOperation = (TableResource as any).indexingOperation; + if (indexingOperation) pending.add(indexingOperation); + if (!pending.size) return; + let timer: NodeJS.Timeout; + const timedOut = Symbol('timedOut'); + const result = await Promise.race([ + Promise.allSettled(pending), + new Promise((resolve) => { + timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); + }), + ]); + clearTimeout(timer); + if (result === timedOut) { + const directOperationLabels = directOperations + .filter((operation) => pendingTableOperations.has(operation)) + .map(({ label }) => label); + throw new Error( + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop request is durable and the table is unavailable; restart Harper to complete the drop.` + ); + } + }; + const closeTableStores = () => { + if (storesClosed) return; + for (const store of tableStores()) store.close?.(); + storesClosed = true; + }; let createdTimeProperty: Attribute | undefined, updatedTimeProperty: Attribute | undefined, expiresAtProperty: Attribute | undefined; @@ -413,7 +512,6 @@ export function makeTable(options) { if (attribute.expiresAt) expiresAtProperty = attribute; if (attribute.isPrimaryKey) primaryKeyAttribute = attribute; } - let deleteCallbackHandle: { remove: () => void }; let prefetchIds = []; let prefetchCallbacks = []; let untilNextPrefetch = 1; @@ -421,7 +519,9 @@ export function makeTable(options) { let cleanupInterval = 86400000; let cleanupPriority = 0; let lastCleanupInterval: number; - let cleanupTimer: NodeJS.Timeout; + let cleanupTimer: NodeJS.Timeout | undefined; + let cleanupTimerCompletion: { resolve: () => void } | undefined; + let recordExpirationInterval: NodeJS.Timeout; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table @@ -1126,6 +1226,7 @@ export function makeTable(options) { const asyncIdExpansionThreshold = type === 'Int' ? 0x200 : 0x100000; if (nextId + asyncIdExpansionThreshold >= idIncrementer.maxSafeId) { const updateEnd = (inTxn) => { + if (droppingTable) return; // we update the end of the allocation range after verifying we don't have any conflicting ids in front of us idIncrementer.maxSafeId = nextId + (type === 'Int' ? 0x3ff : 0x3fffff); let idAfter = (type === 'Int' ? Math.pow(2, 31) : Math.pow(2, 49)) - 1; @@ -1150,7 +1251,7 @@ export function makeTable(options) { return; } logger.info?.('New id allocation', nextId, idIncrementer.maxSafeId, version); - primaryStore.put( + const completion = primaryStore.put( Symbol.for('id_allocation'), { start: updatedIdAllocation.start, @@ -1161,6 +1262,7 @@ export function makeTable(options) { Date.now(), version ); + return inTxn ? undefined : completion; } else { // indicate that we have run out of ids in the allocated range, so we need to allocate a new range logger.warn?.( @@ -1174,7 +1276,21 @@ export function makeTable(options) { } }; if (nextId + asyncIdExpansionThreshold === idIncrementer.maxSafeId) { - setImmediate(updateEnd); // if we are getting kind of close to the end, we try to update it asynchronously + const finishIdAllocationUpdate = beginTableOperation('id allocation update'); + setImmediate(() => { + try { + const completion = updateEnd(false); + if (completion?.then) { + completion.then(finishIdAllocationUpdate, (error) => { + finishIdAllocationUpdate(); + if (!droppingTable) logger.warn?.(`Error updating id allocation for ${tableName}`, error); + }); + } else finishIdAllocationUpdate(); + } catch (error) { + finishIdAllocationUpdate(); + if (!droppingTable) logger.warn?.(`Error updating id allocation for ${tableName}`, error); + } + }); } else if (nextId + 100 >= idIncrementer.maxSafeId) { logger.warn?.( `Synchronous id allocation required on table ${tableName}${ @@ -1359,163 +1475,176 @@ export function makeTable(options) { return coerceType(id, primaryKeyAttribute); } + static async _prepareDrop({ closeStores = true } = {}) { + markTableDropping(); + dropPreparation ??= drainTableOperations().catch((error) => { + dropPreparation = undefined; + throw error; + }); + await dropPreparation; + if (closeStores && !coordinatingDrop) closeTableStores(); + } + static async dropTable() { + if (storesClosed) { + throw new ServerError( + `Cannot retry dropping ${databaseName}.${tableName} through closed handles; restart Harper to resume the durable drop`, + 503 + ); + } + const rootStore = primaryStore.rootStore; + const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; + const processInstanceId = sharedRocksStore ? getProcessInstanceId() : undefined; + if (sharedRocksStore && processInstanceId == null) { + throw new ServerError( + `Cannot safely coordinate dropping ${databaseName}.${tableName} from an unregistered worker`, + 503 + ); + } + const activeTransaction = contextStorage.getStore()?.transaction; + const currentTableStores = new Set(tableStores()); + if ( + sharedRocksStore && + activeTransaction instanceof DatabaseTransaction && + (activeTransaction.hasWritesForAnyStore(currentTableStores) || + activeTransaction.hasOpenReadsForAnyStore(currentTableStores)) + ) { + const error: any = new ClientError( + `Cannot drop ${databaseName}.${tableName} from a transaction with active reads or staged writes to that table; complete the transaction first`, + 409 + ); + error.code = 'ERR_TABLE_DROP_IN_TRANSACTION'; + throw error; + } + let dropGeneration: string | undefined; if (databaseName === databasePath) { - // Persist a drop tombstone on the primary catalog entry BEFORE any - // destructive work. If the process dies or a column family drop fails - // partway through, the tombstone survives with the catalog rows, and - // the next startup (or a same-name create) completes the drop via - // completeInterruptedDrop in databases.ts instead of resurrecting - // the table. const primaryCatalogKey = TableResource.tableName + '/'; const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); if (primaryMeta && !primaryMeta.dropping) { primaryMeta.dropping = true; - // Stamps this drop's identity so the interrupted-drop retry budget in - // databases.ts can be scoped to THIS drop rather than the table name: a - // worker that exhausts the budget for a table can observe the catalog - // mid-flight between this drop's completion and a same-name recreate's - // own drop, without ever seeing a non-tombstoned row to reset on. Keying - // the budget by generation instead makes the new drop's tombstone carry - // its own fresh key regardless of what any worker last observed. primaryMeta.dropGeneration = randomUUID(); - // put is rebound to putSync on RocksDB stores; on LMDB it returns - // a promise, so await it to make the tombstone durable before the - // destructive work below + primaryMeta.dropQuiesced = !sharedRocksStore; + // A random process-start identity (not the PID, which containers commonly reuse) lets + // recovery distinguish a live process that may still hold handles from a clean restart. + if (sharedRocksStore) primaryMeta.dropProcessInstance = processInstanceId; const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); if (tombstoneWrite?.then) await tombstoneWrite; } + dropGeneration = primaryMeta?.dropGeneration; } - // A get() against a sourcedFrom table resolves to its caller before the resolved - // record's cache write has committed (see getFromSource) - the write lands "in the - // background" for latency reasons. Flip this BEFORE removing the table from the - // schema below: getFromSource() checks it and skips caching (treats the load as - // noCacheStore) for any call it admits from here on, including one that slipped in - // through a stale reference to this Table between the two steps. - droppingTable = true; - // Remove the table from the in-memory schema immediately so concurrent - // requests get "table does not exist" instead of racing the column - // family drops below. If a drop fails past this point the table stays - // invisible, and the tombstone guarantees the drop completes on the - // next startup (or on a same-name create). - delete databases[databaseName][tableName]; - // The above stops new source-fill writes from starting, but a write from a get() - // that already returned to its caller may still be in flight. Dropping the column - // families out from under that write is a genuine invariant violation, not just a - // benign race: RocksDB rejects the still-open write batch with "Invalid column - // family specified in write batch" (or "Could not access column family N"), which - // can also abort this drop before it removes the tombstoned catalog rows - leaving - // the table stuck "dropping" for completeInterruptedDrop to retry (and fail - // identically) on every subsequent load (harper#1381). Drain any in-flight commits - // before the blob sweep below (so it observes every row a drain-caught write just - // committed) and before touching a single column family. - // - // Bounded, and fails CLOSED: the tracked promise covers the whole source round-trip - // plus the local commit (see getFromSource), so a hung/slow source or a slow commit - // (e.g. a large blob write) could otherwise wedge this drop forever. Rather than - // give up and drop anyway - which would reopen exactly the race this drain exists to - // close, just less often - a timeout FAILS the drop. The tombstone written above is - // already durable, so completeInterruptedDrop picks the drop back up on the next - // load, once the stuck write has had time to finish. - if (pendingSourceCommits.size) { - const pending = [...pendingSourceCommits]; - let timer: NodeJS.Timeout; - const timedOut = Symbol('timedOut'); - const result = await Promise.race([ - Promise.allSettled(pending), - new Promise((resolve) => { - timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); - }), - ]); - clearTimeout(timer); - if (result === timedOut) { - throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.` - ); + coordinatingDrop = true; + let locallyQuiesced = false; + try { + if (sharedRocksStore) { + // Once the tombstone is durable, every worker must stop admission even if this + // coordinator's drain fails. Otherwise peers can acknowledge writes that restart + // recovery would later destroy without ever receiving the barrier. + const [localPreparation, remotePreparation] = await Promise.allSettled([ + prepareTableDrop(rootStore.path, tableName, dropGeneration, TableResource), + signalling.signalTableDropPreparation({ + operation: TABLE_DROP_PREPARE_OPERATION, + schema: databaseName, + table: tableName, + path: rootStore.path, + dropGeneration, + }), + ]); + locallyQuiesced = localPreparation.status === 'fulfilled'; + const failedPreparation = + localPreparation.status === 'rejected' + ? localPreparation.reason + : remotePreparation.status === 'rejected' + ? remotePreparation.reason + : undefined; + if (failedPreparation) { + const quiescenceError: any = new ServerError( + `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable. Restart Harper to complete this durable drop. ${failedPreparation.message ?? String(failedPreparation)}`, + 503 + ); + quiescenceError.code = failedPreparation.code; + quiescenceError.cause = failedPreparation; + throw quiescenceError; + } + } else { + await TableResource._prepareDrop({ closeStores: false }); + locallyQuiesced = true; } - } - for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { - if (entry.metadataFlags & HAS_BLOBS && entry.value) { - deleteBlobsInObject(entry.value); + + for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { + if (entry.metadataFlags & HAS_BLOBS && entry.value) deleteBlobsInObject(entry.value); } - } - if (databaseName === databasePath) { - // part of a database. - // Drop the column families, then remove the catalog metadata - never - // the reverse: a removed-then-failed drop orphans a "ghost" column - // family that poisons same-name recreates, so a genuine drop failure - // must surface and leave the tombstoned catalog rows for the reconcile. - // - // A drop is broadcast to every worker thread, and each holds its own - // handle to the same underlying column family, so a concurrent worker - // (or completeInterruptedDrop) may already have dropped it - surfaced - // as "Column family already dropped!". That is the intended end state, - // not a failure, so tolerate it. The catalog rows are removed only if - // this drop's tombstone is still the live primary row: a concurrent - // same-name create completes the interrupted drop and writes fresh - // catalog rows, and clobbering those would orphan the new table. - const removeTombstonedCatalog = () => { - const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); - if (!currentPrimary?.dropping) return false; - for (const attribute of attributes) { - dbisDb.remove(TableResource.tableName + '/' + attribute.name); - } - dbisDb.remove(TableResource.tableName + '/'); - return true; - }; - const rootStore = primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) { - // Serialize the drops + catalog removal against a concurrent - // same-name create (and completeInterruptedDrop) under the database's - // 'update-attributes' exclusive lock - the same lock the create path - // holds. It is a synchronous spin lock that blocks the event loop, so - // the locked section MUST stay synchronous: drop with dropSync (as - // completeInterruptedDrop does), never an awaited drop(), or a - // concurrent create's spin would deadlock waiting on a drop that the - // blocked event loop can never resolve. - while (!rootStore.tryLock('update-attributes')) {} - let removed = false; - try { + + if (databaseName === databasePath) { + // Keep the tombstone until every column family is gone; reversing this order can orphan + // an undiscoverable "ghost" family when a drop fails partway through. + const removeTombstonedCatalog = () => { + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping || currentPrimary.dropGeneration !== dropGeneration) return false; for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) - try { - index.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); - } + dbisDb.remove(TableResource.tableName + '/' + attribute.name); } + dbisDb.remove(TableResource.tableName + '/'); + return true; + }; + if (rootStore instanceof RocksDatabase) { + // Concurrent creates take this synchronous spin lock too. Nothing in the locked section + // may await, or this worker can deadlock the process while the lock owner needs its event loop. + while (!rootStore.tryLock('update-attributes')) {} + let removed = false; try { - primaryStore.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping || currentPrimary.dropGeneration !== dropGeneration) { + throw new ServerError(`Drop generation changed while preparing ${databaseName}.${tableName}`, 409); + } + currentPrimary.dropQuiesced = true; + (dbisDb as any).putSync(TableResource.tableName + '/', currentPrimary); + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) + try { + index.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + } + try { + primaryStore.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + closeTableStores(); + removed = removeTombstonedCatalog(); + } finally { + rootStore.unlock('update-attributes'); } - removed = removeTombstonedCatalog(); - } finally { - rootStore.unlock('update-attributes'); + if (removed) await dbisDb.committed; + } else { + const drops = []; + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + } + drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); + await Promise.all(drops); + if (removeTombstonedCatalog()) await dbisDb.committed; } - if (removed) await dbisDb.committed; } else { - // LMDB: no shared column-family double-drop, and its engine lock is - // transactional rather than this spin lock, so keep the awaited drop - // plus the same tombstone-guarded catalog removal. - const drops = []; - for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + await primaryStore.close(); + fs.unlinkSync(primaryStore.path); + } + await signalling.signalSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) + ); + } finally { + coordinatingDrop = false; + if (locallyQuiesced && rootStore instanceof RocksDatabase && !storesClosed) { + try { + closeTableStores(); + } catch (error) { + logger.warn?.(`Failed to close table handles for ${databaseName}.${tableName}`, error); } - drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); - await Promise.all(drops); - if (removeTombstonedCatalog()) await dbisDb.committed; } - } else { - // legacy table per database - await primaryStore.close(); - fs.unlinkSync(primaryStore.path); } - signalling.signalSchemaChange( - new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) - ); } // #section: read-path /** @@ -2081,18 +2210,28 @@ export function makeTable(options) { }); } // RocksDB: eviction writes went directly into the raw transaction via options; commit it directly, - // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses - // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it - // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any).commit().catch((error) => { - // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the - // batched-eviction path does on its own commit failures. committed===true skips the finally abort. - try { - (transaction as any).abort(); - } catch {} + // as DatabaseTransaction.commit() would abort it (no tracked writes). A coordinated-retry conflict + // resolves with RETRY_NOW_VALUE; abandon that eviction and release its native transaction. + const handleCommitFailure = (error) => { + lmdbTransaction.releaseReadTxn(); if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); else logger.warn?.('Error evicting record', id, error); - }); + }; + let commitCompletion: Promise; + try { + commitCompletion = Promise.resolve((transaction as any).commit()); + } catch (error) { + handleCommitFailure(error); + return Promise.resolve(); + } + return commitCompletion.then((result) => { + if (result === rocksConstants.RETRY_NOW_VALUE) { + lmdbTransaction.releaseReadTxn(); + logger.trace?.('Abandoned eviction of busy record', id); + return; + } + lmdbTransaction.completeReadTxn(); + }, handleCommitFailure); } finally { if (!committed) { // Skip path or thrown error: abort instead of committing so we don't apply @@ -2100,7 +2239,7 @@ export function makeTable(options) { if (primaryStore.ifVersion) { (lmdbTransaction as any).abort?.(); } else { - (transaction as any)?.abort?.(); + lmdbTransaction.releaseReadTxn(); } } } @@ -3963,6 +4102,7 @@ export function makeTable(options) { // #section: pub-sub async subscribe(request: SubscriptionRequest): Promise> { + if (isRocksDB && droppingTable) throw tableDroppingError(); if (!request) request = {} as any; const loadAsInstance = (this.constructor as any).loadAsInstance; if (loadAsInstance === false && (request as any).checkPermission) { @@ -4029,6 +4169,7 @@ export function makeTable(options) { return evaluateFilter(rowFilter, event.value, 'rowFilter'); } : null; + if (isRocksDB && droppingTable) throw tableDroppingError(); const subscription = addSubscription( TableResource, thisId, @@ -4093,6 +4234,13 @@ export function makeTable(options) { // in subscription.queue. Without this, the IIFE can fill the queue past // EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists. if (request.listener) subscription!.on('data', request.listener); + let finishInitialScan: () => void; + try { + finishInitialScan = beginTableOperation('subscription replay', () => subscription.close()); + } catch (error) { + subscription.close(); + throw error; + } const result = (async () => { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { @@ -4338,6 +4486,7 @@ export function makeTable(options) { pendingRealTimeQueue = null; } })(); + result.then(finishInitialScan, finishInitialScan); result.catch(failSubscription); function failSubscription(error: any) { if (subscription.closed) return; @@ -4396,7 +4545,9 @@ export function makeTable(options) { // aftercommit path holds an inter-thread lock that must not span event-loop turns. async function runReloadResnapshot() { reloadResnapshotRunning = true; + let finishReloadScan: (() => void) | undefined; try { + finishReloadScan = beginTableOperation('subscription reload scan', () => subscription.close()); await rest(); // defer off the broadcast listener's stack before scanning while (reloadResnapshotPending) { reloadResnapshotPending = false; @@ -4415,6 +4566,7 @@ export function makeTable(options) { } catch (error) { harperLogger.error?.('Error in reload re-snapshot:', error); } finally { + finishReloadScan?.(); reloadResnapshotRunning = false; // A marker that landed after the last pending-check but before we cleared the flag would // otherwise be dropped — re-arm if so (unless the subscription has since closed). @@ -4422,7 +4574,7 @@ export function makeTable(options) { } } function scheduleReloadResnapshot() { - if (!subscription.subscriptions) return; + if (droppingTable || !subscription.subscriptions) return; reloadResnapshotPending = true; if (!reloadResnapshotRunning) runReloadResnapshot(); } @@ -4805,91 +4957,98 @@ export function makeTable(options) { return getStorageSpaceStats(primaryStore.path); } static async getRecordCount(options?: any) { - // iterate through the metadata entries to exclude their count and exclude the deletion counts - const exactCount = options?.exactCount; - const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms - const start = performance.now(); - // `entryCount` (the exact key count) is only needed once the scan blows the time budget -- - // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full - // key-only scan, so we defer it: tables that finish within budget (the common case) and - // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed. - let entryCount = 0; - let halfway = 0; - let counted = false; - let completeForExact = false; - let recordCount = 0; - let entriesScanned = 0; - let limit: number; - for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) { - if (value != null) recordCount++; - entriesScanned++; - await rest(); - if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { - if (!counted) { - counted = true; - entryCount = isRocksDB - ? primaryStore.getKeysCount({ start: undefined }) - : primaryStore.getStats().entryCount; - halfway = Math.floor(entryCount / 2); - } - if (entriesScanned < halfway) { - // it is taking too long, so we will just take this sample and a sample from the end to estimate - limit = entriesScanned; - break; - } - // Past the halfway point already: finishing the scan for an exact count is cheaper - // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration. - completeForExact = true; - } - } - if (limit) { - // in this case we are going to make an estimate of the table count using the first thousand - // entries and last thousand entries - const firstRecordCount = recordCount; - recordCount = 0; - // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but - // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate` - // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source - // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB - // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint. - let reverseScanned = 0; - for (const { value } of primaryStore.getRange({ - start: '\uffff', - reverse: true, - lazy: true, - limit, - snapshot: false, - })) { + const finishRecordCountScan = beginTableOperation('record count scan'); + try { + // iterate through the metadata entries to exclude their count and exclude the deletion counts + const exactCount = options?.exactCount; + const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms + const start = performance.now(); + // `entryCount` (the exact key count) is only needed once the scan blows the time budget -- + // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full + // key-only scan, so we defer it: tables that finish within budget (the common case) and + // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed. + let entryCount = 0; + let halfway = 0; + let counted = false; + let completeForExact = false; + let recordCount = 0; + let entriesScanned = 0; + let limit: number; + for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) { if (value != null) recordCount++; - reverseScanned++; + entriesScanned++; await rest(); - if (reverseScanned >= limit) break; + if (isRocksDB && droppingTable) throw tableDroppingError(); + if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { + if (!counted) { + counted = true; + entryCount = isRocksDB + ? primaryStore.getKeysCount({ start: undefined }) + : primaryStore.getStats().entryCount; + halfway = Math.floor(entryCount / 2); + } + if (entriesScanned < halfway) { + // it is taking too long, so we will just take this sample and a sample from the end to estimate + limit = entriesScanned; + break; + } + // Past the halfway point already: finishing the scan for an exact count is cheaper + // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration. + completeForExact = true; + } + } + if (limit) { + // in this case we are going to make an estimate of the table count using the first thousand + // entries and last thousand entries + const firstRecordCount = recordCount; + recordCount = 0; + // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but + // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate` + // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source + // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB + // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint. + let reverseScanned = 0; + for (const { value } of primaryStore.getRange({ + start: '\uffff', + reverse: true, + lazy: true, + limit, + snapshot: false, + })) { + if (value != null) recordCount++; + reverseScanned++; + await rest(); + if (isRocksDB && droppingTable) throw tableDroppingError(); + if (reverseScanned >= limit) break; + } + // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` + // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting + // those un-scanned slots would inflate the denominator and underestimate the rate. + const sampleSize = limit + reverseScanned; + const recordRate = (recordCount + firstRecordCount) / sampleSize; + const variance = + Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples + (recordRate * (1 - recordRate)) / sampleSize; + const sd = Math.max(Math.sqrt(variance) * entryCount, 1); + const estimatedRecordCount = Math.round(recordRate * entryCount); + // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using + // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement). + const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount); + const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount); + let significantUnit = Math.pow(10, Math.round(Math.log10(sd))); + if (significantUnit > estimatedRecordCount) significantUnit = significantUnit / 10; + recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit; + return { + recordCount, + estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)], + }; } - // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` - // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting - // those un-scanned slots would inflate the denominator and underestimate the rate. - const sampleSize = limit + reverseScanned; - const recordRate = (recordCount + firstRecordCount) / sampleSize; - const variance = - Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples - (recordRate * (1 - recordRate)) / sampleSize; - const sd = Math.max(Math.sqrt(variance) * entryCount, 1); - const estimatedRecordCount = Math.round(recordRate * entryCount); - // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using - // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement). - const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount); - const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount); - let significantUnit = Math.pow(10, Math.round(Math.log10(sd))); - if (significantUnit > estimatedRecordCount) significantUnit = significantUnit / 10; - recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit; return { recordCount, - estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)], }; + } finally { + finishRecordCountScan(); } - return { - recordCount, - }; } /** * When attributes have been changed, we update the accessors that are assigned to this table @@ -5160,98 +5319,165 @@ export function makeTable(options) { this.userSetEmbedders.add(attribute_name); } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise { - let completion: Promise; + const finishHistoryScan = beginTableOperation('history deletion scan'); + const inFlightRemovals = new Set>(); + let removalFailure: unknown; + let removalFailed = false; + const trackRemoval = async (completion: Promise | void) => { + if (!completion || typeof completion.then !== 'function') return; + let tracked: Promise; + tracked = Promise.resolve(completion) + .then(undefined, (error) => { + if (!removalFailed) removalFailure = error; + removalFailed = true; + }) + .finally(() => inFlightRemovals.delete(tracked)); + inFlightRemovals.add(tracked); + if (inFlightRemovals.size >= MAX_INFLIGHT_MAINTENANCE_REMOVALS) await Promise.race(inFlightRemovals); + if (removalFailed) throw removalFailure; + }; let entriesDeleted = 0; - for (const auditRecord of auditStore.getRange({ - start: 0, - end: endTime, - })) { - await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId) continue; - completion = removeAuditEntry(auditStore, auditRecord); - entriesDeleted++; - } - if (cleanupDeletedRecords) { - // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't - // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records - for (const entry of primaryStore.getRange({ start: 0, versions: true })) { - const { value, localTime } = entry; + let operationFailure: unknown; + let operationFailed = false; + try { + for (const auditRecord of auditStore.getRange({ + start: 0, + end: endTime, + })) { await rest(); // yield to other async operations - if (value === null && localTime < endTime) { - completion = removeEntry(primaryStore, entry); + if (isRocksDB && droppingTable) throw tableDroppingError(); + if (auditRecord.tableId !== tableId) continue; + await trackRemoval(removeAuditEntry(auditStore, auditRecord)); + entriesDeleted++; + } + if (cleanupDeletedRecords) { + // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't + // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records + for (const entry of primaryStore.getRange({ start: 0, versions: true })) { + const { value, localTime } = entry; + await rest(); // yield to other async operations + if (isRocksDB && droppingTable) throw tableDroppingError(); + if (value === null && localTime < endTime) { + await trackRemoval(removeEntry(primaryStore, entry)); + } } } + } catch (error) { + operationFailure = error; + operationFailed = true; + } finally { + try { + await Promise.all(inFlightRemovals); + } finally { + finishHistoryScan(); + } } - await completion; + if (operationFailed) throw operationFailure; + if (removalFailed) throw removalFailure; return entriesDeleted; } static async *getHistory(startTime = 0, endTime = Infinity) { - for (const auditRecord of auditStore.getRange({ - start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols - end: endTime, - })) { - await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId) continue; - yield { - id: auditRecord.recordId, - localTime: auditRecord.version, - version: auditRecord.version, - type: auditRecord.type, - value: auditRecord.getValue(primaryStore, true, auditRecord.version), - user: auditRecord.user, - operation: auditRecord.originatingOperation, - }; + let iterator: Iterator | undefined; + let finishHistoryScan: () => void; + finishHistoryScan = beginTableOperation('history iterator', () => { + try { + iterator?.return?.(); + } finally { + finishHistoryScan(); + } + }); + try { + iterator = auditStore + .getRange({ + start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols + end: endTime, + }) + [Symbol.iterator](); + for (let next = iterator.next(); !next.done; next = iterator.next()) { + const auditRecord = next.value; + await rest(); // yield to other async operations + if (isRocksDB && droppingTable) throw tableDroppingError(); + if (auditRecord.tableId !== tableId) continue; + yield { + id: auditRecord.recordId, + localTime: auditRecord.version, + version: auditRecord.version, + type: auditRecord.type, + value: auditRecord.getValue(primaryStore, true, auditRecord.version), + user: auditRecord.user, + operation: auditRecord.originatingOperation, + }; + if (isRocksDB && droppingTable) throw tableDroppingError(); + } + } finally { + try { + iterator?.return?.(); + } finally { + finishHistoryScan(); + } } } static async getHistoryOfRecord(id) { - const history = []; - if (id == undefined) throw new Error('An id is required'); - const entry = primaryStore.getEntry(id); - if (!entry) return history; - let nextVersion = entry.localTime; - if (!nextVersion) throw new Error('The entry does not have a local audit time'); - const count = 0; - const auditWindow = 100; - do { - await rest(); // yield to other async operations - let insertionPoint = history.length; - let highestPreviousVersion = 0; - const start = nextVersion - auditWindow; - for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { - if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) { - history.splice(insertionPoint, 0, { - id: auditRecord.recordId, - localTime: auditRecord.version, - version: auditRecord.version, - type: auditRecord.type, - // reconstruct each entry's record image as of its own version, not the audit - // window boundary (nextVersion), matching getHistory (issue #1330) - value: auditRecord.getValue(primaryStore, true, auditRecord.version), - user: auditRecord.user, - operation: auditRecord.originatingOperation, - }); - if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) { - highestPreviousVersion = auditRecord.previousVersion; + const finishHistoryScan = beginTableOperation('record history scan'); + try { + const history = []; + if (id == undefined) throw new Error('An id is required'); + const entry = primaryStore.getEntry(id); + if (!entry) return history; + let nextVersion = entry.localTime; + if (!nextVersion) throw new Error('The entry does not have a local audit time'); + const count = 0; + const auditWindow = 100; + do { + await rest(); // yield to other async operations + if (isRocksDB && droppingTable) throw tableDroppingError(); + let insertionPoint = history.length; + let highestPreviousVersion = 0; + const start = nextVersion - auditWindow; + for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { + if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) { + history.splice(insertionPoint, 0, { + id: auditRecord.recordId, + localTime: auditRecord.version, + version: auditRecord.version, + type: auditRecord.type, + // reconstruct each entry's record image as of its own version, not the audit + // window boundary (nextVersion), matching getHistory (issue #1330) + value: auditRecord.getValue(primaryStore, true, auditRecord.version), + user: auditRecord.user, + operation: auditRecord.originatingOperation, + }); + if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) { + highestPreviousVersion = auditRecord.previousVersion; + } } } - } - nextVersion = highestPreviousVersion; - } while (count < 1000 && nextVersion); - return history.reverse(); + nextVersion = highestPreviousVersion; + } while (count < 1000 && nextVersion); + return history.reverse(); + } finally { + finishHistoryScan(); + } } - static clear() { - // clear the primary store and every secondary index dbi (same pattern used by - // runIndexing when rebuilding from scratch), so clear() doesn't leave stale - // index entries pointing at records that no longer exist. - const promises = [primaryStore.clear()]; - for (const key in indices) { - const index = indices[key]; - promises.push(index.clearAsync ? index.clearAsync() : index.clear()); + static async clear() { + const finishClear = beginTableOperation('table clear'); + try { + // clear the primary store and every secondary index dbi (same pattern used by + // runIndexing when rebuilding from scratch), so clear() doesn't leave stale + // index entries pointing at records that no longer exist. + const promises = [primaryStore.clear()]; + for (const key in indices) { + const index = indices[key]; + promises.push(index.clearAsync ? index.clearAsync() : index.clear()); + } + return await Promise.all(promises); + } finally { + finishClear(); } - return Promise.all(promises); } static cleanup() { deleteCallbackHandle?.remove(); + stopBackgroundScans(); } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -5597,6 +5823,7 @@ export function makeTable(options) { } } function txnForContext(context: Context) { + if (isRocksDB && droppingTable) throw tableDroppingError(); let transaction = context?.transaction; if (isReleasedTransaction(transaction)) transaction = undefined; if (transaction) { @@ -5608,7 +5835,11 @@ export function makeTable(options) { } do { // See if this is a transaction for our database and if so, use it - if (transaction.db?.path === primaryStore.path) return transaction; + if (transaction.db?.path === primaryStore.path) { + // Every tracked iterator must join here so the drop drain can attribute every borrowed table store. + transaction.trackStore(primaryStore); + return transaction; + } // try the next one: const nextTxn = transaction.next; if (!nextTxn) { @@ -6219,6 +6450,9 @@ export function makeTable(options) { } return { + cancel(): void { + pending = []; + }, add(type: 'evict' | 'tombstone', key: any, version: number): Promise | void { pending.push({ type, key, version }); if (pending.length >= EVICTION_BATCH_SIZE) { @@ -6249,9 +6483,11 @@ export function makeTable(options) { lastCleanupInterval = cleanupInterval; if (getWorkerIndex() === getWorkerCount() - 1) { // run on the last thread so we aren't overloading lower-numbered threads - if (cleanupTimer) clearTimeout(cleanupTimer); + stopCleanupTimer(); if (!cleanupInterval) return; - return new Promise((resolve) => { + return new Promise((resolve) => { + const thisCleanupCompletion = { resolve }; + cleanupTimerCompletion = thisCleanupCompletion; const startOfYear = new Date(); startOfYear.setMonth(0); startOfYear.setDate(1); @@ -6305,13 +6541,17 @@ export function makeTable(options) { return false; } + let finishCleanupScan: (() => void) | undefined; + let batcher: ReturnType | undefined; + let count = 0; + let scanCompleted = false; try { - let count = 0; + finishCleanupScan = beginTableOperation('cleanup scan'); let removeDeletedRecords = !audit || isRocksDB; // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize // the per-record commit cost; LMDB keeps the per-record path (eventTurnBatching already // coalesces async writes per event turn). - const batcher = isRocksDB ? createEvictionBatcher() : undefined; + batcher = isRocksDB ? createEvictionBatcher() : undefined; // iterate through all entries to find expired records and deleted records for (const entry of primaryStore.getRange({ start: false, @@ -6349,12 +6589,25 @@ export function makeTable(options) { } } await rest(); + if (droppingTable) { + batcher?.cancel(); + return; + } } - if (batcher) await batcher.drain(); - logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); + scanCompleted = true; } catch (error) { - logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + if (!droppingTable) logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + } finally { + try { + if (droppingTable) batcher?.cancel(); + if (batcher) await batcher.drain(); + await Promise.all(outstandingCleanupOperations.filter(Boolean)); + } finally { + finishCleanupScan?.(); + } } + if (scanCompleted) logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); + if (cleanupTimerCompletion === thisCleanupCompletion) cleanupTimerCompletion = undefined; resolve(undefined); cleanupPriority = 0; // reset the priority })), @@ -6367,19 +6620,34 @@ export function makeTable(options) { } function addDeleteRemoval() { deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id: Id, version: number) => { - primaryStore.remove(id, version); + const finishRemoval = beginTableOperation('audit delete removal'); + try { + const removal = primaryStore.remove(id, version); + if (removal?.then) { + removal.then(finishRemoval, (error) => { + finishRemoval(); + logger.warn?.(`Audit delete removal error for ${tableName}:`, error); + }); + } else finishRemoval(); + return removal; + } catch (error) { + finishRemoval(); + throw error; + } }); } function runRecordExpirationEviction() { // Periodically evict expired records, searching for records who expiresAt timestamp is before now if (getWorkerIndex() === 0) { // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting - setInterval(async () => { + recordExpirationInterval = setInterval(async () => { // go through each database and table and then search for expired entries // find any entries that are set to expire before now if (runningRecordExpiration) return; runningRecordExpiration = true; + let finishExpirationScan: (() => void) | undefined; try { + finishExpirationScan = beginTableOperation('expiration scan'); const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); @@ -6400,10 +6668,12 @@ export function makeTable(options) { } } await rest(); + if (droppingTable) return; } } catch (error) { - logger.error?.('Error in evicting old records', error); + if (!droppingTable) logger.error?.('Error in evicting old records', error); } finally { + finishExpirationScan?.(); runningRecordExpiration = false; } }, RECORD_PRUNING_INTERVAL).unref(); diff --git a/resources/auditStore.ts b/resources/auditStore.ts index b57b88665e..9ecd13d13c 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -153,7 +153,8 @@ export function openAuditStore(rootStore) { auditStore.deleteCallbacks = deleteCallbacks; return { remove() { - delete deleteCallbacks[tableId]; + if (deleteCallbacks[tableId] === callback) delete deleteCallbacks[tableId]; + if (auditStore.tableStores[tableId] === table) delete auditStore.tableStores[tableId]; }, }; }; diff --git a/resources/databases.ts b/resources/databases.ts index 55cd4fba2b..9da007e884 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -23,7 +23,7 @@ import { ClientError } from '../utility/errors/hdbError.ts'; import { _assignPackageExport } from '../globals.js'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { markItcReadyForStorage, SchemaEventMsg } from '../server/threads/itc.js'; import { workerData } from 'worker_threads'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent } = harperLogger; @@ -284,6 +284,7 @@ export function toRocksCompression(compression: unknown): unknown { } function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { + markItcReadyForStorage(); options.disableWAL ??= true; const legacyOptions = options as { compression?: unknown }; // A configured codec applies to every column family, overriding whatever per-table metadata @@ -379,7 +380,21 @@ const MAX_INTERRUPTED_DROP_ATTEMPTS = 3; // resolved, so a resolution can only ever identify the outer path+table key, // never the specific spent generation to target. const interruptedDropAttempts = new Map>(); +const incompleteTableDropPreparations = new Map>(); const interruptedDropTableKey = (storePath: string, tableName: string) => `${storePath}\0${tableName}`; +const tableDropPreparationKey = (storePath: string, tableName: string, generation?: string) => + `${storePath}\0${tableName}\0${generation ?? 'legacy'}`; + +function retainTableForDropPreparation(storePath: string, tableName: string, generation: string | undefined, Table) { + const preparationKey = tableDropPreparationKey(storePath, tableName, generation); + let matchingTables = incompleteTableDropPreparations.get(preparationKey); + if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); + matchingTables.add(Table); + // Marking and cancellation happen synchronously before _prepareDrop's first await. Keep the class + // in the preparation set after the drain so the later strict barrier can close its handles. + return Table._prepareDrop({ closeStores: false }); +} + function getInterruptedDropAttempts(storePath: string, tableName: string, generation?: string): number { return interruptedDropAttempts.get(interruptedDropTableKey(storePath, tableName))?.get(generation ?? 'legacy') ?? 0; } @@ -772,11 +787,30 @@ function initStores( clearInterruptedDropEntries(path, tableName); continue; } + // A tombstone always removes the worker-local class, even when this worker cannot safely + // perform the physical cleanup. Keeping it in definedTables would leave a stale class and + // its dropped handles reachable after this reconcile pass. + definedTables?.delete(tableName); + if (!canCompleteInterruptedDrop(tableDef.primary)) { + // The tombstone can become visible here before this worker receives the coordinator's + // strict barrier. Prepare the still-live class now; otherwise the registry cleanup below + // makes it undiscoverable to prepareTableDrop while its timers and handles remain active. + const liveTable = tables[tableName]; + if (liveTable) + retainTableForDropPreparation(rootStore.path, tableName, tableDef.primary.dropGeneration, liveTable).catch( + (error) => logger.warn(`Failed to quiesce ${databaseName}.${tableName} during schema reconciliation`, error) + ); + logger.debug( + `Deferring interrupted drop of table ${databaseName}.${tableName} until worker quiescence or a clean process start` + ); + tablesToLoad.delete(tableName); + continue; + } const generation = tableDef.primary?.dropGeneration; const failedAttempts = getInterruptedDropAttempts(path, tableName, generation); if (failedAttempts < MAX_INTERRUPTED_DROP_ATTEMPTS) { try { - completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName); + completeInterruptedDropWithLock(rootStore, attributesDbi, databaseName, tableName, generation); // Sweep every generation this worker has ever tracked for this table, not // just the one just resolved: if a prior generation was exhausted here, // then resolved+recreated+re-dropped by another worker as this generation @@ -784,7 +818,6 @@ function initStores( // place that sweeps), the prior generation's entry would otherwise never // be cleared. clearInterruptedDropEntries(path, tableName); - definedTables?.delete(tableName); } catch (error) { const attempt = failedAttempts + 1; setInterruptedDropAttempts(path, tableName, generation, attempt); @@ -1668,7 +1701,13 @@ export function table(tableDefinition: TableDefinition): Tabl // create below starts from a clean slate; treating the tombstoned // entry as an existing table would recurse forever on the stale // catalog row. - completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName); + if (!canCompleteInterruptedDrop(existingTableMeta)) { + throw new ClientError( + `Table '${databaseName}.${tableName}' has an interrupted drop that was not quiesced across workers; restart Harper before recreating it`, + 409 + ); + } + completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, existingTableMeta.dropGeneration); // This resolves the drop without ever going through the schema-load // reconcile below, which is the only other place that returns a spent // budget. Without clearing it here too, a table that gets dropped again @@ -2072,16 +2111,38 @@ async function runIndexing(Table, attributes, indicesToRemove) { await signalling.signalSchemaChange( new SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName) ); + let hadIndexingErrors = false; + const pendingOperations = new Set>(); + const trackOperation = (operation: T | Promise): T | Promise => { + if (!(operation as Promise)?.then) return operation; + const pendingOperation = Promise.resolve(operation); + pendingOperations.add(pendingOperation); + pendingOperation.then( + () => pendingOperations.delete(pendingOperation), + (error) => { + pendingOperations.delete(pendingOperation); + hadIndexingErrors = true; + logger.error(error); + } + ); + return operation; + }; + const drainSubmittedOperations = async () => { + while (pendingOperations.size > 0) await Promise.allSettled([...pendingOperations]); + }; let lastResolution; for (const index of indicesToRemove) { - lastResolution = index.drop(); + lastResolution = trackOperation(index.drop()); } let interrupted; - let hadIndexingErrors = false; const attributeErrorReported = {}; let indexed = 0; const attributesLength = attributes.length; await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn + if (Table.primaryStore.dropping) { + await drainSubmittedOperations(); + return; + } if (attributesLength > 0) { let start: any; for (const attribute of attributes) { @@ -2105,6 +2166,10 @@ async function runIndexing(Table, attributes, indicesToRemove) { versions: true, snapshot: false, // don't hold a read transaction this whole time })) { + if (Table.primaryStore.dropping) { + await drainSubmittedOperations(); + return; + } if (!record) continue; // deletion entry // TODO: Do we ever need to interrupt due to a schema change that was not a restart? //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up @@ -2132,7 +2197,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { const values = getIndexedValues(value, index.indexNulls); if (values) { for (let i = 0, l = values.length; i < l; i++) { - lastResolution = index.put(values[i], key); + lastResolution = trackOperation(index.put(values[i], key)); } } } catch (error) { @@ -2152,11 +2217,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { when( lastResolution, () => outstanding--, - (error) => { - outstanding--; - hadIndexingErrors = true; - logger.error(error); - } + () => outstanding-- ); if (workerData && workerData.restartNumber !== manageThreads.restartNumber) { interrupted = true; @@ -2165,9 +2226,12 @@ async function runIndexing(Table, attributes, indicesToRemove) { // occasionally update our progress so if we crash, we can resume for (const attribute of attributes) { attribute.lastIndexedKey = key; - Table.dbisDB.put(attribute.key, attribute); + trackOperation(Table.dbisDB.put(attribute.key, attribute)); + } + if (interrupted) { + await drainSubmittedOperations(); + return; } - if (interrupted) return; } if (outstanding > MAX_OUTSTANDING_INDEXING) await lastResolution; else if (outstanding > MIN_OUTSTANDING_INDEXING) @@ -2175,23 +2239,9 @@ async function runIndexing(Table, attributes, indicesToRemove) { else if (didSynchronousIndexing) await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed } } - // Await the last pending put. If it rejects, that is also an indexing error. - // Note: the when() calls above already attach rejection handlers to each record's - // last-put promise; this try-catch specifically handles the case where lastResolution - // itself rejects (i.e. the very last put in the loop failed) which would otherwise - // throw past the hadIndexingErrors check to the outer catch. The broader issue of - // unhandled rejections from non-last puts in multi-value attributes is pre-existing - // and out of scope for this fix. - try { - await lastResolution; - } catch (error) { - hadIndexingErrors = true; - logger.error(error); - } - // Yield one more event turn so any queued when() error callbacks (which fire as - // microtasks when their tracked promise settles) have a chance to set hadIndexingErrors - // before we decide whether to mark indexing as complete. - await new Promise((resolve) => setImmediate(resolve)); + // A backfill is quiesced only after every write it submitted has settled. + await drainSubmittedOperations(); + if (Table.primaryStore.dropping) return; if (hadIndexingErrors) { // Some records failed to index. Persist the failure marker in the descriptor so // the next call to table() (including after a restart with a fresh PID) re-triggers @@ -2289,8 +2339,35 @@ async function runIndexing(Table, attributes, indicesToRemove) { * actionable, and logging here on every attempt would flood at the same * volume this function's callers are bounding. */ -function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, tableName: string) { +function completeInterruptedDropWithLock( + rootStore, + attributesDbi, + databaseName: string, + tableName: string, + dropGeneration?: string +) { + if (!(rootStore instanceof RocksDatabase)) { + return completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, dropGeneration); + } + while (!rootStore.tryLock('update-attributes')) {} + try { + return completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, dropGeneration); + } finally { + rootStore.unlock('update-attributes'); + } +} + +function completeInterruptedDrop( + rootStore, + attributesDbi, + databaseName: string, + tableName: string, + dropGeneration?: string +) { logger.debug(`Completing interrupted drop of table ${databaseName}.${tableName}`); + const primaryCatalogKey = tableName + '/'; + const primaryMeta = (attributesDbi as any).getSync(primaryCatalogKey); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) return false; if (rootStore instanceof RocksDatabase) { for (const columnName of (rootStore as any).columns) { if (columnName.startsWith(tableName + '/')) { @@ -2331,7 +2408,6 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, // recognizes the table as mid-drop - instead of the tombstone vanishing // first and stranding orphaned attribute rows that the next load would // misread as a live (non-dropping) table. - const primaryCatalogKey = tableName + '/'; let removePrimaryLast = false; for (const key of attributesDbi.getKeys({ start: tableName + '/', end: tableName + '0' })) { if (key === primaryCatalogKey) { @@ -2344,6 +2420,59 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, (attributesDbi as any).removeSync(key); } if (removePrimaryLast) (attributesDbi as any).removeSync(primaryCatalogKey); + return true; +} + +function canCompleteInterruptedDrop(primaryMeta): boolean { + // A completed barrier is safe immediately. An incomplete barrier is safe only after a clean + // process start, when none of the handles from the recorded incarnation can still exist. + // Tombstones written before the incarnation field existed necessarily came from an older process. + const processInstanceId = manageThreads.getProcessInstanceId(); + return ( + primaryMeta?.dropQuiesced === true || + (processInstanceId != null && primaryMeta?.dropProcessInstance !== processInstanceId) + ); +} + +export async function prepareTableDrop( + storePath: string, + tableName: string, + dropGeneration: string | undefined, + preserveTable?: any +): Promise { + const preparationKey = tableDropPreparationKey(storePath, tableName, dropGeneration); + let matchingTables = incompleteTableDropPreparations.get(preparationKey); + if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); + if (preserveTable) matchingTables.add(preserveTable); + let generationMismatch: ClientError | undefined; + for (const databaseName of Object.getOwnPropertyNames(databases)) { + const databaseTables = databases[databaseName]; + const Table = databaseTables?.[tableName]; + if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; + const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { + generationMismatch ??= new ClientError( + `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, + 409 + ); + continue; + } + matchingTables.add(Table); + } + // A failed hidden class is no longer discoverable through databases; retain it for a later barrier. + const preparations = await Promise.allSettled( + [...matchingTables].map(async (Table) => { + await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); + matchingTables.delete(Table); + }) + ); + const failedPreparation = preparations.find( + (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' + ); + if (failedPreparation) throw failedPreparation.reason; + if (!matchingTables.size && incompleteTableDropPreparations.get(preparationKey) === matchingTables) + incompleteTableDropPreparations.delete(preparationKey); + if (generationMismatch) throw generationMismatch; } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/resources/transaction.ts b/resources/transaction.ts index b2dd2abcdf..ca8b710817 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -23,26 +23,25 @@ export function transaction( callback?: (transaction: Transaction) => T ): T { let context: Context; - let asyncStorageContext; if (typeof ctx === 'function') { // optional first argument, handle case of no request callback = ctx; - asyncStorageContext = contextStorage.getStore(); - context = asyncStorageContext ?? {}; + context = contextStorage.getStore() ?? {}; } else { // The released placeholder is an absent argument, not a context: normalized before the fallback // chain below so it resolves to the ambient store exactly as the `null` it replaced did, rather // than to a bare `{}` that would drop the caller's user, session and timestamp. const contextArg = isReleasedTransaction(ctx) ? undefined : ctx; // request argument included, but null or undefined, so maybe create a new one - context = contextArg ?? (asyncStorageContext = contextStorage.getStore()) ?? {}; + context = contextArg ?? contextStorage.getStore() ?? {}; } if (typeof callback !== 'function') { throw new TypeError('Callback function must be provided to transaction'); } if (context?.transaction?.open === TRANSACTION_STATE.OPEN && typeof callback === 'function') { - return callback(context.transaction); // nothing to be done, already in open transaction + const invokeCallback = () => callback(context.transaction); + return contextStorage.getStore() === context ? invokeCallback() : contextStorage.run(context, invokeCallback); } const transaction = new DatabaseTransaction(); @@ -53,10 +52,8 @@ export function transaction( transaction.setContext(context); let result; try { - result = - (context as any).isExplicit || asyncStorageContext - ? callback(transaction) - : contextStorage.run(context, () => callback(transaction)); + const invokeCallback = () => callback(transaction); + result = contextStorage.getStore() === context ? invokeCallback() : contextStorage.run(context, invokeCallback); if ((result as any)?.then) { return (result as any).then(onComplete, onError); } @@ -66,18 +63,28 @@ export function transaction( return onComplete(result); // when the transaction function completes, run this to commit the transaction function onComplete(result) { - const committed = transaction.commit({ doneWriting: true }); - if ((committed as any).then) { - return (committed as any).then(() => { + try { + const committed = transaction.commit({ doneWriting: true }); + if ((committed as any).then) { + return (committed as any).then(() => result); + } else { return result; - }); - } else { - return result; + } + } catch (error) { + return onCommitError(error); } } + function onCommitError(error) { + try { + transaction.abort(); + } catch {} + throw error; + } // if the transaction function throws an error, we abort function onError(error) { - transaction.abort(); + try { + transaction.abort(); + } catch {} throw error; } } diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index 9b48ef3266..c54c52d4c6 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -12,7 +12,7 @@ const harperBridge = require('../../dataLayer/harperBridge/harperBridge.ts'); const process = require('process'); const { isMainThread, workerData } = require('worker_threads'); -const { resetDatabases, closeDatabase } = require('../../resources/databases.ts'); +const { resetDatabases, closeDatabase, prepareTableDrop } = require('../../resources/databases.ts'); /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -46,6 +46,10 @@ async function schemaHandler(event) { } hdbLogger.trace(`ITC schemaHandler received schema event:`, event); + if (event.message.operation === hdbTerms.TABLE_DROP_PREPARE_OPERATION) { + await prepareTableDrop(event.message.path, event.message.table, event.message.dropGeneration); + return; + } // restore_backup: this thread must release its store handles so the restore can purge and // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while // the restoring marker is present, and reloads it on the completion signal (marker gone). diff --git a/server/threads/itc.js b/server/threads/itc.js index 992237f4e4..642786be2a 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,39 +3,92 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { threadId } = require('worker_threads'); -const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); +const { isMainThread, parentPort, threadId, workerData } = require('worker_threads'); +const harperLogger = require('../../utility/logging/harper_logger.ts'); +const { + onMessageFromWorkers, + broadcastWithAcknowledgement, + broadcastWithStrictAcknowledgement, + sendToThreadWithStrictAcknowledgement, +} = require('./manageThreads.js'); module.exports = { sendItcEvent, + sendItcEventStrict, + markItcReadyForStorage, validateEvent, SchemaEventMsg, UserEventMsg, }; let serverItcHandlers; +const storageReadySignal = !isMainThread && workerData?.itcReadyBuffer && new Int32Array(workerData.itcReadyBuffer); +let storageReady = false; +const STRICT_COORDINATOR_ACK_TIMEOUT_MS = 60000; onMessageFromWorkers(async (event, sender) => { - serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - validateEvent(event); - if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); + const requestId = event?.requestId; + let handlerError; + try { + serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); + if (serverItcHandlers[event.type]) { + const validationError = validateEvent(event); + if (validationError) throw new Error(validationError); + await serverItcHandlers[event.type](event); + } + if (event.relayStrictToWorkers && isMainThread) { + const relayedEvent = { ...event, relayStrictToWorkers: false, requestId: undefined }; + await broadcastWithStrictAcknowledgement(relayedEvent, undefined, event.message.originator); + } + } catch (error) { + handlerError = error; + } + if (handlerError) harperLogger.error('ITC event handler failed', handlerError); + if (requestId && sender) { + try { + sender.postMessage({ + type: 'ack', + id: requestId, + ...(handlerError && { + error: { + message: handlerError.message ?? String(handlerError), + code: handlerError.code, + }, + }), + }); + } catch (error) { + harperLogger.error('Unable to acknowledge ITC event', error); + } } - if (event.requestId && sender) - sender.postMessage({ - type: 'ack', - id: event.requestId, - }); }); +function markItcReadyForStorage() { + if (storageReady || !storageReadySignal) return; + storageReady = true; + Atomics.store(storageReadySignal, 0, 1); + parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); +} /** * Emits an ITC event to the ITC server. * @param event */ function sendItcEvent(event) { + stampOriginator(event); + return broadcastWithAcknowledgement(event); +} + +function sendItcEventStrict(event) { + stampOriginator(event); + if (isMainThread) return broadcastWithStrictAcknowledgement(event); + event.relayStrictToWorkers = true; + // The main thread first prepares itself and then runs its own 30-second worker broadcast. + // The worker-to-main deadline must cover both phases rather than racing the nested deadline. + return sendToThreadWithStrictAcknowledgement(0, event, STRICT_COORDINATOR_ACK_TIMEOUT_MS); +} + +function stampOriginator(event) { // Always stamp originator so handlers can send direct responses back. // The main thread's threadId is 0 (worker_threads convention); parentPort.threadId // is set to 0 in workers, so sendToThread(0, ...) routes back to main. if (event.message) event.message.originator = threadId; - return broadcastWithAcknowledgement(event); } /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 87d829c9c3..159040a1e8 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -17,7 +17,7 @@ const { setHeapSnapshotNearHeapLimit } = typeof globalThis.Bun !== 'undefined' ? const hdbTerms = require('../../utility/hdbTerms.ts'); const envMgr = require('../../utility/environment/environmentManager.ts'); const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { randomBytes } = require('crypto'); +const { randomBytes, randomUUID } = require('crypto'); const { _assignPackageExport } = require('../../globals.js'); const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); const { resolvePreloadModules } = require('./resolvePreload.ts'); @@ -47,6 +47,11 @@ const isBun = typeof globalThis.Bun !== 'undefined'; const MB = 1024 * 1024; const workers = []; // these are our child workers that we are managing const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) +const PROCESS_INSTANCE_ENV = 'HARPER_INTERNAL_PROCESS_INSTANCE_ID'; +const processInstanceId = isMainThread + ? randomUUID() + : workerData?.processInstanceId || process.env[PROCESS_INSTANCE_ENV]; +if (isMainThread) process.env[PROCESS_INSTANCE_ENV] = processInstanceId; const MAX_UNEXPECTED_RESTARTS = 50; // Threads get 10s to die before they're forced. In dev (`harper dev`) we widen this: a reload's old // worker may be disposing a native runtime (e.g. @harperfast/vite's rolldown dev server) and forcing it @@ -142,8 +147,11 @@ module.exports = { onMessageByType, broadcast, broadcastWithAcknowledgement, + broadcastWithStrictAcknowledgement, + sendToThreadWithStrictAcknowledgement, getWorkerIndex, getWorkerCount, + getProcessInstanceId, getTicketKeys, setMainIsWorker, setTerminateTimeout, @@ -189,6 +197,9 @@ function setTerminateTimeout(newTimeout) { function getWorkerIndex() { return workerData ? workerData.workerIndex : isMainWorker ? 0 : undefined; } +function getProcessInstanceId() { + return processInstanceId; +} function getWorkerCount() { return workerData ? workerData.workerCount : isMainWorker ? 1 : undefined; } @@ -205,10 +216,13 @@ let workerCount = 1; // should be assigned when workers are created const RESERVED_WORKER_DATA_KEYS = [ 'addPorts', 'addThreadIds', + 'addItcReadyBuffers', + 'itcReadyBuffer', 'workerIndex', 'workerCount', 'name', 'restartNumber', + 'processInstanceId', 'ticketKeys', 'noServerStart', '__proto__', // never a legitimate payload name; spread would define it as an own property @@ -341,6 +355,7 @@ function startWorker(path, options = {}) { channelsToConnect.push(channel); portsToSend.push(channel.port2); } + const itcReadyBuffer = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); if (!extname(path)) path += '.js'; @@ -383,10 +398,13 @@ function startWorker(path, options = {}) { ...collectProvidedWorkerData(options), addPorts: portsToSend, addThreadIds: channelsToConnect.map((channel) => channel.existingPort.threadId), + addItcReadyBuffers: channelsToConnect.map((channel) => channel.existingPort.itcReadySignal?.buffer), + itcReadyBuffer, workerIndex: options.workerIndex, workerCount: (workerCount = options.threadCount), name: options.name, restartNumber: module.exports.restartNumber, + processInstanceId, ticketKeys: getTicketKeys(), }, transferList: portsToSend, @@ -402,11 +420,12 @@ function startWorker(path, options = {}) { port: port1, threadId: worker.threadId, isJobWorker, + itcReadyBuffer, }, [port1] ); } - addPort(worker, true, isJobWorker); + addPort(worker, true, isJobWorker, itcReadyBuffer); worker.unexpectedRestarts = options.unexpectedRestarts || 0; worker.startCopy = () => { // in a shutdown sequence we use overlapping restarts, starting the new thread while waiting for the old thread @@ -695,71 +714,137 @@ let nextId = 1; // worker (its port close fires the same ack handlers), so on timeout we proceed best-effort. const DEFAULT_ACK_TIMEOUT_MS = 30000; function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return new Promise((resolve) => { + return broadcastAwaitingAcknowledgements(message, timeout, false, false); +} + +// Destructive work uses the strict variant: every connected thread, including jobs, must finish +// its handler successfully or exit completely. A timeout, ambiguous MessagePort disconnect, +// handler error, or post failure rejects so the caller can leave its durable recovery marker in +// place without touching storage. +function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS, excludedThreadId) { + const ports = + excludedThreadId == null ? connectedPorts : connectedPorts.filter((port) => port.threadId !== excludedThreadId); + return broadcastAwaitingAcknowledgements(message, timeout, true, true, ports, true); +} + +function sendToThreadWithStrictAcknowledgement(threadId, message, timeout = DEFAULT_ACK_TIMEOUT_MS) { + const port = connectedPorts.find((port) => port.threadId === threadId); + if (!port) { + const error = new Error(`Worker thread ${threadId} is not connected`); + error.code = 'ERR_ITC_THREAD_NOT_CONNECTED'; + return Promise.reject(error); + } + return broadcastAwaitingAcknowledgements(message, timeout, true, true, [port]); +} + +function broadcastAwaitingAcknowledgements( + message, + timeout, + strict, + includeJobWorkers, + ports = connectedPorts, + skipUnready = false +) { + return new Promise((resolve, reject) => { let waitingCount = 0; let timer; + let setupComplete = false; + let finished = false; + const failures = []; // Tracks the handlers still awaiting an ack for THIS broadcast. Doubles as an // idempotency guard: a port's handler runs at most once whether it's driven by an ack, // the close listener, or the timeout below. const pending = new Set(); const finish = () => { + if (finished || !setupComplete || waitingCount !== 0) return; + finished = true; if (timer) { clearTimeout(timer); timer = undefined; } - resolve(); + if (strict && failures.length) { + const error = new Error( + `ITC broadcast (type ${message.type}) failed on ${failures.length} worker thread(s): ${failures + .map(({ threadId, message }) => `${threadId}: ${message}`) + .join('; ')}` + ); + error.code = 'ERR_ITC_ACKNOWLEDGEMENT'; + error.failures = failures; + reject(error); + } else resolve(); }; - for (let port of connectedPorts) { - // Job workers run a single isolated task and exit; they don't participate in - // schema-change gossip. Including them causes a deadlock: the broadcast waits for - // the job worker's ACK while the job worker's event loop is busy waiting for the - // same broadcast to complete (re-entrant schema change triggered by the job op). - if (port.isJobWorker) continue; - try { - let requestId = nextId++; - const ackHandler = () => { - if (!pending.delete(ackHandler)) return; // already settled for this port - awaitingResponses.delete(requestId); - if (--waitingCount === 0) { - finish(); + for (let port of ports) { + // Storage opening publishes readiness atomically before creating the first RocksDB handle. + const itcReady = port.itcReady || (port.itcReadySignal && Atomics.load(port.itcReadySignal, 0) === 1); + if (skipUnready && !itcReady) continue; + // Ordinary post-change gossip excludes transient job workers. Strict pre-change barriers + // include them because a job can hold the same native handles; the main-thread relay keeps + // a job-originated async schema operation re-entrant while it awaits its own ACK. + if (port.isJobWorker && !includeJobWorkers) continue; + const targetThreadId = port.threadId; + let referenced = false; + let requestId = nextId++; + const ackHandler = (acknowledgement, threadExited = false) => { + if (!pending.delete(ackHandler)) return; // already settled for this port + awaitingResponses.delete(requestId); + if (strict) { + const ackError = acknowledgement?.error; + if (ackError) { + failures.push({ + threadId: targetThreadId, + message: ackError.message ?? String(ackError), + }); + } else if (!acknowledgement && !threadExited) { + failures.push({ threadId: targetThreadId, message: 'worker disconnected before acknowledging' }); } - if (port !== parentPort && --port.refCount === 0) { - port.unref(); - } - }; - ackHandler.port = port; - pending.add(ackHandler); + } + waitingCount--; + if (referenced && port !== parentPort && --port.refCount === 0) port.unref(); + finish(); + }; + ackHandler.port = port; + ackHandler.threadId = targetThreadId; + pending.add(ackHandler); + waitingCount++; + awaitingResponses.set((message.requestId = requestId), ackHandler); + try { port.ref(); port.refCount = (port.refCount || 0) + 1; - awaitingResponses.set((message.requestId = requestId), ackHandler); + referenced = true; if (!port.hasAckCloseListener) { // just set a single close listener that can clean up all the ack handlers for a port that is closed port.hasAckCloseListener = true; - port.on(port.close ? 'close' : 'exit', () => { + const disconnectEvent = port.close ? 'close' : 'exit'; + port.on(disconnectEvent, () => { for (let [, ackHandler] of awaitingResponses) { if (ackHandler.port === port) { - ackHandler(); + // A Worker exit means the thread can no longer use its stale handles. A sibling + // MessagePort can close while its owning thread remains alive, so that stays a NACK. + ackHandler(undefined, disconnectEvent === 'exit'); } } }); } port.postMessage(message); - waitingCount++; } catch (error) { harperLogger.error(`Unable to send message to worker`, error); + ackHandler({ error: { message: error.message ?? String(error) } }); } } - if (waitingCount === 0) return resolve(); + setupComplete = true; + if (waitingCount === 0) return finish(); if (timeout > 0) { timer = setTimeout(() => { timer = undefined; const stuck = []; for (let ackHandler of [...pending]) { - stuck.push(ackHandler.port?.threadId); - ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves + stuck.push(ackHandler.threadId); + ackHandler({ error: { message: `no acknowledgement within ${timeout}ms` } }); } harperLogger.warn( - `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` + strict + ? `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; refusing destructive work` + : `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` ); }, timeout); timer.unref?.(); @@ -842,7 +927,7 @@ if (parentPort && workerData?.addPorts) { for (let i = 0, l = workerData.addPorts.length; i < l; i++) { let port = workerData.addPorts[i]; port.threadId = workerData.addThreadIds[i]; - addPort(port); + addPort(port, false, false, workerData.addItcReadyBuffers?.[i]); } setInterval(() => { // post our memory usage as a resource report, reporting our memory usage @@ -1140,8 +1225,9 @@ function removePort(port, deadThreadId) { } } -function addPort(port, keepRef, isJobWorker) { +function addPort(port, keepRef, isJobWorker, itcReadyBuffer) { if (isJobWorker) port.isJobWorker = true; + if (itcReadyBuffer) port.itcReadySignal = new Int32Array(itcReadyBuffer); connectedPorts.push(port); // Capture threadId now — Bun resets port.threadId to -1 by the time 'exit' fires. const portThreadId = port.threadId; @@ -1151,13 +1237,15 @@ function addPort(port, keepRef, isJobWorker) { addProcessGroup(portThreadId, message.processGroupId); } else if (message.type === UNREGISTER_PROCESS_GROUP) { removeProcessGroup(portThreadId, message.processGroupId); + } else if (message.type === hdbTerms.ITC_EVENT_TYPES.ITC_READY) { + port.itcReady = true; } else if (message.type === ADDED_PORT) { message.port.threadId = message.threadId; - addPort(message.port, false, message.isJobWorker); + addPort(message.port, false, message.isJobWorker, message.itcReadyBuffer); } else if (message.type === ACKNOWLEDGEMENT) { let completion = awaitingResponses.get(message.id); if (completion) { - completion(); + completion(message); } } else if (message.type === REMOVE_PORT) { const idx = connectedPorts.findIndex((p) => p.threadId === message.threadId); diff --git a/unitTests/resources/Resource-get-context.test.js b/unitTests/resources/Resource-get-context.test.js index c89ad1ab56..953e60f9ca 100644 --- a/unitTests/resources/Resource-get-context.test.js +++ b/unitTests/resources/Resource-get-context.test.js @@ -268,7 +268,7 @@ describe('dropTable waits for in-flight source-populated cache writes (harper#13 assert.strictEqual(dropResolved, true, 'dropTable() should resolve once the pending write has landed'); }); - it('does not start a new cache write once dropTable() has begun (late admission during the drain)', async function () { + it('does not admit a new cache read once dropTable() has begun', async function () { setupTestDBPath(); setMainIsWorker(true); @@ -306,23 +306,13 @@ describe('dropTable waits for in-flight source-populated cache writes (harper#13 const dropPromise = TestTable.dropTable(); - // A get() admitted while the drop is draining must still return fresh source data... - const lateResult = await TestTable.get('late', {}); - assert.ok(lateSourceCalled, 'the source should still be consulted for a late-admitted read'); - assert.strictEqual(lateResult.name, 'value'); - // ...but must not have started a new cache write into a table that's being dropped. The - // get() call itself resolves before its own cache write would land (that's the whole - // bug this file covers), so a correctly-blocked write and one that merely hasn't landed - // YET look identical immediately after the await above. Give a generous, bounded window - // for an (incorrectly) unblocked local write to land - the drop itself is still parked on - // the gated first write, so this checks storage well before any column family is - // touched, not a race against the drop. - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.strictEqual( - TestTable.primaryStore.getSync('late'), - undefined, - 'a get() admitted after dropTable() started must not cache its result' + assert.throws( + () => TestTable.get('late', {}), + (error) => error?.code === 'ERR_TABLE_DROPPING', + 'a read arriving after the drain boundary must fail before opening a transaction' ); + assert.strictEqual(lateSourceCalled, false, 'the source must not be consulted after drop preparation begins'); + assert.strictEqual(TestTable.primaryStore.getSync('late'), undefined, 'a rejected read must not cache a result'); releaseFirst(); await firstGetPromise; diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js new file mode 100644 index 0000000000..294f040e87 --- /dev/null +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -0,0 +1,203 @@ +'use strict'; + +require('../testUtils'); +const { parentPort } = require('node:worker_threads'); +const { setupTestDBPath } = require('../testUtils'); +const { table, databases, closeLoadedDatabases } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { + onMessageByType, + getProcessInstanceId, + sendToThreadWithStrictAcknowledgement, +} = require('#js/server/threads/manageThreads'); + +const MESSAGE_TYPE = 'drop-table-quiescence-test'; +const CONTROL_TYPE = 'drop-table-quiescence-control'; +let TestTable; +let aliasPreparations = 0; +let aliasClosedStores = false; +let releaseEmbed; +let releaseRead; +let releaseTransaction; + +function report(event, details = {}) { + parentPort.postMessage({ type: MESSAGE_TYPE, event, ...details }); +} + +function runWorkerFixture() { + onMessageByType(CONTROL_TYPE, () => {}); + setupTestDBPath(); + + process.on('unhandledRejection', (error) => { + report('unhandled-rejection', { error: error?.stack ?? String(error) }); + }); + + parentPort + .on('message', async (message) => { + if (message.type !== CONTROL_TYPE) return; + try { + switch (message.command) { + case 'initialize': { + aliasPreparations = 0; + aliasClosedStores = false; + const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; + if (message.withEmbed) { + attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); + } + TestTable = table({ + table: message.table, + database: message.database ?? 'test', + attributes, + }); + if (message.withAlias) { + const aliasName = `${message.database ?? 'test'}_alias`; + databases[aliasName] = { + [message.table]: { + primaryStore: TestTable.primaryStore, + dbisDB: TestTable.dbisDB, + async _prepareDrop({ closeStores }) { + aliasPreparations++; + aliasClosedStores ||= closeStores; + if (closeStores) TestTable.primaryStore.close(); + }, + }, + }; + } + if (message.withEmbed) { + const embedGate = new Promise((resolve) => { + releaseEmbed = resolve; + }); + TestTable.setEmbedAttribute('vector', async () => { + report('embed-entered'); + await embedGate; + return [1, 2, 3]; + }); + TestTable.sourcedFrom({ + get: async (id) => ({ id, name: 'gated' }), + available: () => true, + }); + } + + if (typeof TestTable._prepareDrop === 'function') { + const prepareDrop = TestTable._prepareDrop; + TestTable._prepareDrop = async function (options) { + report('prepare-entered'); + await prepareDrop.call(this, options); + let handlesClosed = false; + try { + TestTable.primaryStore.getSync('__drop-close-probe__'); + } catch { + handlesClosed = true; + } + report('prepare-finished', { handlesClosed }); + }; + } + report('ready', { processInstanceId: getProcessInstanceId() }); + break; + } + case 'begin-source-read': + TestTable.get(message.id, {}).then( + () => report('source-read-resolved'), + (error) => report('source-read-rejected', { error: error?.stack ?? String(error) }) + ); + break; + case 'release-embed': + releaseEmbed(); + break; + case 'begin-transaction': { + const context = {}; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + transaction(context, async () => { + await TestTable.put({ id: message.id, name: 'pending' }, context); + report('transaction-staged'); + await transactionGate; + }).then( + () => report('transaction-resolved'), + (error) => report('transaction-rejected', { error: error?.stack ?? String(error) }) + ); + break; + } + case 'release-transaction': + releaseTransaction(); + break; + case 'begin-read': { + const context = {}; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + transaction(context, async (dbTransaction) => { + TestTable._readTxnForContext(context); + const readTransaction = dbTransaction.useReadTxn(); + const iterator = TestTable.primaryStore + .getRange({ start: false, transaction: readTransaction }) + [Symbol.iterator](); + iterator.next(); + report('read-open'); + await readGate; + try { + iterator.next(); + } finally { + iterator.return?.(); + dbTransaction.doneReadTxn(); + } + }).then( + () => report('read-resolved'), + (error) => report('read-rejected', { error: error?.stack ?? String(error) }) + ); + break; + } + case 'release-read': + releaseRead(); + break; + case 'reject-prepare': + TestTable._prepareDrop = async () => { + report('prepare-entered'); + throw new Error('injected worker quiescence failure'); + }; + report('reject-prepare-armed'); + break; + case 'send-foreign-strict': + await sendToThreadWithStrictAcknowledgement(0, { type: 'resource_report', heapUsed: 1 }, 1000); + report('foreign-strict-acknowledged'); + break; + case 'drop-table': { + const originalDropSync = TestTable.primaryStore.dropSync; + if (message.interruptAfterColumnFamilyDrop) { + TestTable.primaryStore.dropSync = function (...args) { + originalDropSync.apply(this, args); + throw new Error('injected interruption after column-family drop'); + }; + } + try { + await TestTable.dropTable(); + report('drop-result', { outcome: 'resolved', aliasPreparations, aliasClosedStores }); + } catch (error) { + report('drop-result', { + outcome: 'rejected', + error: error?.stack ?? String(error), + aliasPreparations, + aliasClosedStores, + }); + } finally { + TestTable.primaryStore.dropSync = originalDropSync; + } + break; + } + case 'shutdown': + closeLoadedDatabases(); + report('shutdown-complete'); + parentPort.unref(); + break; + } + } catch (error) { + report('command-error', { command: message.command, error: error?.stack ?? String(error) }); + } + }) + .ref(); + + report('booted'); +} + +if (parentPort) runWorkerFixture(); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js new file mode 100644 index 0000000000..f8b5c1493b --- /dev/null +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -0,0 +1,1463 @@ +'use strict'; + +require('../testUtils'); +const assert = require('node:assert'); +const { mkdirSync } = require('node:fs'); +const path = require('node:path'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); +const { + table, + database, + databases, + getDatabases, + prepareTableDrop, + resetDatabases, +} = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { DatabaseTransaction } = require('#src/resources/DatabaseTransaction'); +const env = require('#src/utility/environment/environmentManager'); +const terms = require('#src/utility/hdbTerms'); +const { ITC_EVENT_TYPES, TABLE_DROP_PREPARE_OPERATION, THREAD_TYPES } = terms; +const { + broadcastWithStrictAcknowledgement, + startWorker, + onMessageByType, + setMainIsWorker, + getProcessInstanceId, +} = require('#js/server/threads/manageThreads'); + +const WORKER_FIXTURE = path.join(__dirname, 'dropTableQuiescence-worker.js'); +const UNREADY_WORKER_FIXTURE = path.join(__dirname, 'dropTableUnready-worker.js'); +const MESSAGE_TYPE = 'drop-table-quiescence-test'; +const CONTROL_TYPE = 'drop-table-quiescence-control'; +let testPath; + +function defineTable(name, withEmbed = false, databaseName = 'test') { + const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; + if (withEmbed) attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); + return table({ table: name, database: databaseName, attributes }); +} + +function startDropWorker(workerIndex, threadCount, name = 'drop-table-quiescence-test') { + const queued = new Map(); + const waiting = new Map(); + const errors = []; + let fatalError; + const nextEvent = (event) => { + if (fatalError) return Promise.reject(fatalError); + const prior = queued.get(event); + if (prior?.length) return Promise.resolve(prior.shift()); + return new Promise((resolve, reject) => { + let eventWaiters = waiting.get(event); + if (!eventWaiters) waiting.set(event, (eventWaiters = [])); + eventWaiters.push({ resolve, reject }); + }); + }; + const fail = (error) => { + fatalError = error instanceof Error ? error : new Error(String(error)); + for (const eventWaiters of waiting.values()) { + for (const waiter of eventWaiters.splice(0)) waiter.reject(fatalError); + } + }; + const receive = (message) => { + if (message.type !== MESSAGE_TYPE) return; + if ( + message.event === 'command-error' || + message.event === 'unhandled-rejection' || + message.event === 'read-rejected' || + message.event === 'transaction-rejected' + ) { + errors.push(message); + fail(new Error(message.error)); + return; + } + const eventWaiters = waiting.get(message.event); + if (eventWaiters?.length) eventWaiters.shift().resolve(message); + else { + let eventQueue = queued.get(message.event); + if (!eventQueue) queued.set(message.event, (eventQueue = [])); + eventQueue.push(message); + } + }; + const booted = nextEvent('booted'); + const worker = startWorker(WORKER_FIXTURE, { + name, + workerIndex, + threadCount, + autoRestart: false, + onStarted(spawnedWorker) { + spawnedWorker.on('message', receive); + spawnedWorker.once('error', fail); + }, + }); + const send = (command, details = {}) => worker.postMessage({ type: CONTROL_TYPE, command, ...details }); + return { + worker, + booted, + errors, + nextEvent, + send, + async shutdown() { + const shutdown = nextEvent('shutdown-complete').catch(() => undefined); + const exited = new Promise((resolve) => worker.once('exit', resolve)); + send('shutdown'); + await Promise.race([shutdown, exited]); + worker.wasShutdown = true; + await worker.terminate(); + }, + }; +} + +async function shutdownWorkers(...workers) { + await Promise.all(workers.filter(Boolean).map((worker) => worker.shutdown().catch(() => undefined))); +} + +describe('dropTable worker quiescence', function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + + before(() => { + testPath = setupTestDBPath(); + setMainIsWorker(true); + onMessageByType(MESSAGE_TYPE, () => {}); + }); + + after(() => { + setMainIsWorker(false); + }); + + it('drains an ordinary staged transaction before touching the column families', async function () { + const Table = defineTable(`DropStagedTxn_${process.pid}_${Date.now()}`); + const context = {}; + let staged; + const stagedPromise = new Promise((resolve) => { + staged = resolve; + }); + let releaseTransaction; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + const transactionPromise = transaction(context, async () => { + await Table.put({ id: 'held', name: 'pending' }, context); + staged(); + await transactionGate; + }); + await stagedPromise; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + let earlyError; + try { + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must suspend on the staged transaction before dropping its primary column family' + ); + } catch (error) { + earlyError = error; + } finally { + if (destructivePhaseStarted) transaction.abort(context); + releaseTransaction(); + } + const [transactionResult, dropResult] = await Promise.allSettled([transactionPromise, dropPromise]); + if (earlyError) throw earlyError; + if (transactionResult.status === 'rejected') throw transactionResult.reason; + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(destructivePhaseStarted, true, 'dropTable() should continue after the transaction settles'); + }); + + it('rejects a drop from its own staged-write transaction before tombstoning', async function () { + const tableName = `DropOwnTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await assert.rejects( + () => + transaction(async () => { + await Table.put({ id: 'staged', name: 'pending' }); + await Table.dropTable(); + }), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + + it('aborts staged writes on linked transactions before dropping the table', async function () { + const Table = defineTable(`DropAbortedLinkedTxn_${process.pid}_${Date.now()}`); + const expectedError = new Error('abort linked transaction'); + let linkedTransaction; + + await assert.rejects( + () => + transaction(async (transactionHead) => { + linkedTransaction = transactionHead.next = new DatabaseTransaction(); + linkedTransaction.db = Table.primaryStore; + await Table.put({ id: 'aborted', name: 'not committed' }, { transaction: linkedTransaction }); + throw expectedError; + }), + expectedError + ); + assert.strictEqual(linkedTransaction.writes.length, 0, 'abort must clear every linked write set'); + assert.strictEqual(await Table.get('aborted'), null); + await Table.dropTable(); + }); + + it('aborts linked writes when the head commit rejects terminally', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const HeadTable = defineTable(`DropTerminalHead_${process.pid}_${Date.now()}`); + const LinkedTable = defineTable(`DropTerminalLinked_${process.pid}_${Date.now()}`); + const headTransaction = new DatabaseTransaction(); + headTransaction.db = HeadTable.primaryStore; + const linkedTransaction = (headTransaction.next = new DatabaseTransaction()); + linkedTransaction.db = LinkedTable.primaryStore; + await HeadTable.put({ id: 'head', name: 'pending' }, { transaction: headTransaction }); + await LinkedTable.put({ id: 'linked', name: 'pending' }, { transaction: linkedTransaction }); + const headNativeTransaction = headTransaction.transaction; + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function (...args) { + if (this === headNativeTransaction) + return Promise.reject(Object.assign(new Error('forced terminal commit failure'), { code: 'ERR_CORRUPTION' })); + return originalCommit.apply(this, args); + }; + try { + await assert.rejects(() => headTransaction.commit({ doneWriting: true }), /forced terminal commit failure/); + } finally { + Transaction.prototype.commit = originalCommit; + } + assert.strictEqual(headTransaction.writes.length, 0, 'the failed head must clear its own write set'); + assert.strictEqual(headTransaction.next, null, 'the failed head must detach its linked transaction chain'); + assert.strictEqual(linkedTransaction.writes.length, 0, 'the failed head must abort every linked write set'); + assert.strictEqual(linkedTransaction.transaction, null, 'the failed head must release every linked native handle'); + await Promise.all([HeadTable.dropTable(), LinkedTable.dropTable()]); + }); + + it('rejects a drop from its own read transaction before tombstoning', async function () { + const tableName = `DropOwnReadTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await transaction(async (dbTransaction) => { + Table._readTxnForContext({ transaction: dbTransaction }); + await assert.rejects( + () => Table.dropTable(), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + }); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + + it('rejects a drop from a nested explicit transaction before tombstoning', async function () { + const tableName = `DropOwnNestedTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await transaction(async () => { + const nestedContext = { isExplicit: true }; + await assert.rejects( + () => + transaction(nestedContext, async () => { + await Table.put({ id: 'nested-staged', name: 'pending' }, nestedContext); + await Table.dropTable(); + }), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + }); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + + it('preserves shared stores while preparing a database alias', async function () { + const tableName = `DropAliasedTable_${process.pid}_${Date.now()}`; + const databaseName = `DropAliasDb_${process.pid}_${Date.now()}`; + const aliasName = `${databaseName}_alias`; + const storePath = path.join(testPath, 'drop-alias-database'); + const dropGeneration = 'shared-store-test'; + const sharedPrimaryStore = { rootStore: { path: storePath } }; + const dbisDB = { + getSync() { + return { dropping: true, dropGeneration }; + }, + }; + let coordinatorCloseStores; + let aliasCloseStores; + const Table = { + primaryStore: sharedPrimaryStore, + dbisDB, + async _prepareDrop({ closeStores }) { + coordinatorCloseStores = closeStores; + }, + }; + const AliasTable = { + primaryStore: sharedPrimaryStore, + dbisDB, + async _prepareDrop({ closeStores }) { + aliasCloseStores = closeStores; + }, + }; + databases[databaseName] = { [tableName]: Table }; + databases[aliasName] = { [tableName]: AliasTable }; + try { + await prepareTableDrop(storePath, tableName, dropGeneration, Table); + assert.strictEqual(coordinatorCloseStores, false); + assert.strictEqual(aliasCloseStores, false, "an alias must not close the coordinator's shared stores"); + } finally { + delete databases[databaseName]; + delete databases[aliasName]; + } + }); + + it('retains a failed hidden preparation for a safe retry', async function () { + const tableName = `DropFailedPreparation_${process.pid}_${Date.now()}`; + const databaseName = `DropFailedPreparationDb_${process.pid}_${Date.now()}`; + const storePath = path.join(testPath, 'drop-failed-preparation-database'); + const dropGeneration = 'failed-preparation-test'; + let attempts = 0; + const Table = { + primaryStore: { rootStore: { path: storePath } }, + dbisDB: { + getSync() { + return { dropping: true, dropGeneration }; + }, + }, + async _prepareDrop() { + if (++attempts === 1) throw new Error('forced preparation failure'); + }, + }; + databases[databaseName] = { [tableName]: Table }; + try { + await assert.rejects(prepareTableDrop(storePath, tableName, dropGeneration), /forced preparation failure/); + } finally { + delete databases[databaseName]; + } + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(attempts, 2, 'the failed hidden table must be retried by the next preparation barrier'); + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(attempts, 2, 'a successful retry must release the retained table'); + }); + + it('releases a generation-mismatched preparation before a later retry', async function () { + const tableName = `DropMismatchedPreparation_${process.pid}_${Date.now()}`; + const databaseName = `DropMismatchedPreparationDb_${process.pid}_${Date.now()}`; + const storePath = path.join(testPath, 'drop-mismatched-preparation-database'); + const dropGeneration = 'expected-generation'; + let liveTableAttempts = 0; + let retainedTableAttempts = 0; + const Table = { + primaryStore: { rootStore: { path: storePath } }, + dbisDB: { + getSync() { + return { dropping: true, dropGeneration: 'different-generation' }; + }, + }, + async _prepareDrop() { + liveTableAttempts++; + }, + }; + const RetainedTable = { + primaryStore: { rootStore: { path: storePath } }, + async _prepareDrop() { + retainedTableAttempts++; + }, + }; + databases[databaseName] = { [tableName]: Table }; + try { + await assert.rejects( + prepareTableDrop(storePath, tableName, dropGeneration, RetainedTable), + /generation does not match/ + ); + } finally { + delete databases[databaseName]; + } + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(liveTableAttempts, 0, 'the mismatched live table must not be prepared'); + assert.strictEqual(retainedTableAttempts, 1, 'a retained class must be closed before the mismatch NACKs'); + }); + + it('does not wait for a read iterator on another table in the same database', async function () { + const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); + const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); + const context = {}; + let readOpened; + const readOpenedPromise = new Promise((resolve) => { + readOpened = resolve; + }); + let releaseRead; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + let readFinished = false; + const readPromise = transaction(context, async (dbTransaction) => { + otherTable._readTxnForContext(context); + const readTransaction = dbTransaction.useReadTxn(); + const iterator = otherTable.primaryStore + .getRange({ start: false, transaction: readTransaction }) + [Symbol.iterator](); + iterator.next(); + readOpened(); + await readGate; + try { + iterator.next(); + } finally { + iterator.return?.(); + dbTransaction.doneReadTxn(); + readFinished = true; + } + }); + await readOpenedPromise; + + const originalDropSync = droppedTable.primaryStore.dropSync; + let destructivePhaseStarted = false; + droppedTable.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = droppedTable.dropTable(); + let earlyError; + try { + await waitFor(() => destructivePhaseStarted, { + message: 'an unrelated table reader must not widen the drop drain to the whole database', + }); + assert.strictEqual(readFinished, false, 'the unrelated iterator must still be open when the drop proceeds'); + } catch (error) { + earlyError = error; + } finally { + releaseRead(); + } + const [readResult, dropResult] = await Promise.allSettled([readPromise, dropPromise]); + if (earlyError) throw earlyError; + if (readResult.status === 'rejected') throw readResult.reason; + if (dropResult.status === 'rejected') throw dropResult.reason; + }); + + it('cancels and drains a transaction-less range scan before closing the table stores', async function () { + const Table = defineTable(`DropDirectScan_${process.pid}_${Date.now()}`); + await Table.put({ id: 'scan', name: 'held' }); + + const originalSetImmediate = global.setImmediate; + const originalGetRange = Table.primaryStore.getRange; + let scanYieldPending = false; + Table.primaryStore.getRange = function (...args) { + const iterable = originalGetRange.apply(this, args); + return { + *[Symbol.iterator]() { + for (const entry of iterable) { + scanYieldPending = true; + yield entry; + } + }, + }; + }; + let releaseScan; + global.setImmediate = (callback, ...args) => { + if (scanYieldPending && !releaseScan) { + scanYieldPending = false; + releaseScan = () => originalSetImmediate(callback, ...args); + return; + } + return originalSetImmediate(callback, ...args); + }; + let scanPromise; + try { + scanPromise = Table.getRecordCount({ exactCount: true }); + await waitFor(() => releaseScan, { message: 'getRecordCount() did not enter its yielded range scan' }); + } finally { + global.setImmediate = originalSetImmediate; + Table.primaryStore.getRange = originalGetRange; + } + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(originalSetImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the direct range scan'); + } catch (error) { + earlyError = error; + } finally { + releaseScan(); + } + const [scanResult, dropResult] = await Promise.allSettled([scanPromise, dropPromise]); + if (earlyError) throw earlyError; + assert.strictEqual(scanResult.status, 'rejected'); + assert.strictEqual(scanResult.reason.code, 'ERR_TABLE_DROPPING'); + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(destructivePhaseStarted, true); + }); + + it('drains an in-flight cleanup batch before dropping the table stores', async function () { + this.timeout(15000); + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropCleanupBatch_${process.pid}_${Date.now()}`); + Table.setTTLExpiration({ expiration: 3600, eviction: 3600, scanInterval: 3600 }); + for (let id = 0; id < 100; id++) { + await Table.put(id, { name: `expired-${id}` }, { expiresAt: 1 }); + } + + const originalCommit = Transaction.prototype.commit; + const originalGetEntry = Table.primaryStore.getEntry; + const cleanupTransactions = new WeakSet(); + Table.primaryStore.getEntry = function (...args) { + const [, options] = args; + if (options?.transaction) cleanupTransactions.add(options.transaction); + return originalGetEntry.apply(this, args); + }; + let cleanupCommitEntered; + const cleanupCommitStarted = new Promise((resolve) => (cleanupCommitEntered = resolve)); + let releaseCleanupCommit; + const cleanupCommitGate = new Promise((resolve) => (releaseCleanupCommit = resolve)); + let blockedCommit = false; + Transaction.prototype.commit = async function (...args) { + if (!blockedCommit && cleanupTransactions.has(this)) { + blockedCommit = true; + cleanupCommitEntered(); + await cleanupCommitGate; + } + return originalCommit.apply(this, args); + }; + + try { + Table.setTTLExpiration({ expiration: 0.001, eviction: 0.001, scanInterval: 0.001 }); + await cleanupCommitStarted; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the cleanup batch commit'); + } catch (error) { + earlyError = error; + } finally { + releaseCleanupCommit(); + } + await dropPromise; + if (earlyError) throw earlyError; + assert.strictEqual(destructivePhaseStarted, true); + } finally { + releaseCleanupCommit(); + Transaction.prototype.commit = originalCommit; + Table.primaryStore.getEntry = originalGetEntry; + } + }); + + it('drains an active index backfill before dropping the table stores', async function () { + const tableName = `DropIndexBackfill_${process.pid}_${Date.now()}`; + let Table = table({ + table: tableName, + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + for (let id = 0; id < 20; id++) await Table.put({ id, name: `name-${id}` }); + Table = table({ + table: tableName, + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name', indexed: true }, + ], + }); + assert.ok(Table.indexingOperation, 'adding an index to existing records must start a backfill'); + const index = Table.indices.name; + const originalPut = index.put; + let indexingWriteStarted; + const indexingWriteStartedPromise = new Promise((resolve) => (indexingWriteStarted = resolve)); + let releaseFirstIndexingWrite; + let indexingWrites = 0; + const firstIndexingWriteGate = new Promise((resolve) => (releaseFirstIndexingWrite = resolve)); + index.put = async function (...args) { + indexingWrites++; + indexingWriteStarted(); + if (indexingWrites === 1) await firstIndexingWriteGate; + return originalPut.apply(this, args); + }; + try { + await indexingWriteStartedPromise; + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for every submitted index write'); + releaseFirstIndexingWrite(); + await Promise.all([Table.indexingOperation, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + assert.ok(indexingWrites < 20, 'drop preparation must cancel the remaining index backfill'); + } finally { + releaseFirstIndexingWrite(); + index.put = originalPut; + } + }); + + it('drains an audit delete removal before dropping the primary store', async function () { + const storagePath = path.join(testPath, 'audit-delete-removal-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + let releaseRemoval; + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const databaseName = `DropAuditDeleteDb_${process.pid}_${Date.now()}`; + const Table = table({ + table: `DropAuditDeleteRemoval_${process.pid}_${Date.now()}`, + database: databaseName, + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'deleted', name: 'removed' }); + await Table.delete('deleted'); + + const originalRemove = Table.primaryStore.remove; + let removalStarted; + const removalStartedPromise = new Promise((resolve) => { + removalStarted = resolve; + }); + const removalGate = new Promise((resolve) => { + releaseRemoval = resolve; + }); + Table.primaryStore.remove = async function (...args) { + removalStarted(); + await removalGate; + return originalRemove.apply(this, args); + }; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const removeDeletedRecord = Table.auditStore.deleteCallbacks[Table.tableId]; + assert.strictEqual(typeof removeDeletedRecord, 'function'); + assert.strictEqual( + Table.auditStore.tableStores[Table.tableId], + Table.primaryStore, + 'the callback must belong to the table under test' + ); + const deleteRemovalPromise = removeDeletedRecord('deleted', Table.primaryStore.getEntry('deleted').version); + await removalStartedPromise; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must wait for the direct primary-store removal launched by audit pruning' + ); + releaseRemoval(); + await Promise.all([deleteRemovalPromise, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + assert.strictEqual( + Table.auditStore.deleteCallbacks[Table.tableId], + undefined, + 'a dropped table must unregister its audit delete callback' + ); + assert.strictEqual( + Table.auditStore.tableStores[Table.tableId], + undefined, + 'audit retention must not retain the dropped table store' + ); + Table.primaryStore.remove = originalRemove; + } finally { + releaseRemoval?.(); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + + it('drains an asynchronous id-allocation update before dropping the primary store', async function () { + const Table = table({ + table: `DropIdAllocation_${process.pid}_${Date.now()}`, + database: 'test', + attributes: [{ name: 'id', type: 'Int', isPrimaryKey: true }], + }); + Table.getNewId(); + const originalPut = Table.primaryStore.put; + let allocationWriteStarted; + const allocationWriteStartedPromise = new Promise((resolve) => (allocationWriteStarted = resolve)); + let releaseAllocationWrite; + const allocationWriteGate = new Promise((resolve) => (releaseAllocationWrite = resolve)); + Table.primaryStore.put = async function (key, ...args) { + if (key === Symbol.for('id_allocation')) { + allocationWriteStarted(); + await allocationWriteGate; + } + return originalPut.call(this, key, ...args); + }; + try { + for (let count = 1; count < 512; count++) Table.getNewId(); + await allocationWriteStartedPromise; + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the id-allocation write'); + releaseAllocationWrite(); + await dropPromise; + assert.strictEqual(destructivePhaseStarted, true); + } finally { + releaseAllocationWrite(); + Table.primaryStore.put = originalPut; + } + }); + + it('aborts write tracking and the native snapshot when commit throws synchronously', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterCommitThrow_${process.pid}_${Date.now()}`); + assert.strictEqual(Table.primaryStore.getOldestSnapshotTimestamp(), 0); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function () { + throw new Error('forced synchronous commit failure'); + }; + try { + await assert.rejects( + transaction(async () => { + await Table.put('held', { name: 'pending' }); + }), + /forced synchronous commit failure/ + ); + } finally { + Transaction.prototype.commit = originalCommit; + } + assert.strictEqual( + Table.primaryStore.getOldestSnapshotTimestamp(), + 0, + 'a synchronous commit failure must not leave the detached native snapshot open' + ); + await Table.dropTable(); + }); + + it('clears write tracking when abort blob inspection throws', async function () { + const Table = defineTable(`DropAfterAbortCleanupThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.addWrite({ store: Table.primaryStore, key: 'held', savedBlobs: [], deferSave: true }); + const originalGetEntry = Table.primaryStore.getEntry; + Table.primaryStore.getEntry = () => { + throw new Error('forced blob inspection failure'); + }; + try { + assert.throws(() => txn.abort(), /forced blob inspection failure/); + } finally { + Table.primaryStore.getEntry = originalGetEntry; + } + await Table.dropTable(); + }); + + it('clears write tracking when native read abort throws', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterNativeAbortThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.db = Table.primaryStore; + const nativeTransaction = txn.getReadTxn(); + txn.addWrite({ store: Table.primaryStore, key: 'held', deferSave: true }); + const originalAbort = Transaction.prototype.abort; + Transaction.prototype.abort = function (...args) { + if (this === nativeTransaction) throw new Error('forced native abort failure'); + return originalAbort.apply(this, args); + }; + try { + assert.throws(() => txn.abort(), /forced native abort failure/); + assert.strictEqual(txn.writes.length, 0, 'native abort failure must still clear the tracked writes'); + } finally { + Transaction.prototype.abort = originalAbort; + txn.releaseReadTxn(); + } + await Table.dropTable(); + }); + + it('releases the read drain when doneReadTxn native abort throws', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterDoneReadAbortThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.db = Table.primaryStore; + const nativeTransaction = txn.getReadTxn(); + const readResolution = txn.getPendingReadResolution(); + const originalAbort = Transaction.prototype.abort; + Transaction.prototype.abort = function (...args) { + if (this === nativeTransaction) throw new Error('forced doneReadTxn abort failure'); + return originalAbort.apply(this, args); + }; + try { + assert.throws(() => txn.doneReadTxn(), /forced doneReadTxn abort failure/); + await readResolution; + assert.strictEqual(txn.transaction, null, 'a native abort failure must not retain the wrapper handle'); + } finally { + Transaction.prototype.abort = originalAbort; + nativeTransaction.abort(); + } + await Table.dropTable(); + }); + + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { + const storagePath = path.join(testPath, 'delete-history-removal-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + let releaseRemoval; + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const Table = table({ + table: `DropDeleteHistoryRemoval_${process.pid}_${Date.now()}`, + database: `DropDeleteHistoryDb_${process.pid}_${Date.now()}`, + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + for (let id = 0; id < 51; id++) { + await Table.put({ id, name: 'removed' }); + await Table.delete(id); + } + delete Table.auditStore.deleteCallbacks[Table.tableId]; + + const originalRemove = Table.primaryStore.remove; + let removalsStarted = 0; + const removalGate = new Promise((resolve) => { + releaseRemoval = resolve; + }); + Table.primaryStore.remove = async function (...args) { + removalsStarted++; + await removalGate; + return originalRemove.apply(this, args); + }; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const deleteHistoryPromise = Table.deleteHistory(Date.now() + 1000, true); + await waitFor(() => removalsStarted === 50, { + message: 'deleteHistory() did not fill its bounded removal window', + }); + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(removalsStarted, 50, 'deleteHistory() must bound its pending removal promises'); + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must wait for direct primary-store removals started by deleteHistory()' + ); + releaseRemoval(); + const [deleteHistoryResult, dropResult] = await Promise.allSettled([deleteHistoryPromise, dropPromise]); + assert.strictEqual(deleteHistoryResult.status, 'rejected'); + assert.strictEqual(deleteHistoryResult.reason.code, 'ERR_TABLE_DROPPING'); + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(removalsStarted, 50); + assert.strictEqual(destructivePhaseStarted, true); + Table.primaryStore.remove = originalRemove; + } finally { + releaseRemoval?.(); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + + it('releases an eviction read transaction before dropping the table', async function () { + const Table = defineTable(`DropAfterEvict_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); + }); + + it('releases an eviction read transaction when native commit throws synchronously', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterEvictThrow_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function () { + const error = new Error('Resource busy'); + error.code = 'ERR_BUSY'; + throw error; + }; + try { + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); + } finally { + Transaction.prototype.commit = originalCommit; + } + }); + + it('releases an eviction read transaction on a coordinated-retry conflict', async function () { + const { Transaction, constants } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterEvictRetry_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = () => Promise.resolve(constants.RETRY_NOW_VALUE); + try { + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); + } finally { + Transaction.prototype.commit = originalCommit; + } + }); + + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { + const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); + for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); + const subscription = await Table.subscribe({ isCollection: true }); + await waitFor(() => subscription.currentDrainResolver, { + message: 'subscription replay did not pause on client backpressure', + }); + + await Table.dropTable(); + assert.strictEqual(subscription.closed, true); + }); + + it('closes an abandoned history iterator before dropping its stores', async function () { + const tableName = `DropHistoryIterator_${process.pid}_${Date.now()}`; + const Table = table({ + table: tableName, + database: 'test', + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'history', name: 'held' }); + const history = Table.getHistory()[Symbol.asyncIterator](); + const first = await history.next(); + assert.strictEqual(first.done, false); + + await Table.dropTable(); + await assert.rejects( + () => history.next(), + (error) => error.code === 'ERR_TABLE_DROPPING' + ); + }); + + it('releases a history operation token when iterator cancellation throws', async function () { + const Table = table({ + table: `DropThrowingHistoryIterator_${process.pid}_${Date.now()}`, + database: 'test', + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'history', name: 'held' }); + + const originalGetRange = Table.auditStore.getRange; + let injectedError = false; + Table.auditStore.getRange = function (...args) { + const range = originalGetRange.apply(this, args); + return { + [Symbol.iterator]() { + const iterator = range[Symbol.iterator](); + const originalReturn = iterator.return?.bind(iterator); + iterator.return = (...returnArgs) => { + const result = originalReturn?.(...returnArgs); + if (!injectedError) { + injectedError = true; + throw new Error('forced history iterator cancellation failure'); + } + return result; + }; + return iterator; + }, + }; + }; + + const history = Table.getHistory()[Symbol.asyncIterator](); + try { + assert.strictEqual((await history.next()).done, false); + await Table.dropTable(); + assert.strictEqual(injectedError, true, 'the test must exercise the throwing cancellation path'); + } finally { + Table.auditStore.getRange = originalGetRange; + await history.return?.(); + } + }); + + it('omits a worker until its ITC listener is ready', async function () { + const worker = startWorker(UNREADY_WORKER_FIXTURE, { + name: THREAD_TYPES.JOB, + workerIndex: 1, + threadCount: 2, + autoRestart: false, + }); + try { + await new Promise((resolve, reject) => { + worker.once('online', resolve); + worker.once('error', reject); + }); + assert.notStrictEqual(worker.itcReady, true); + await broadcastWithStrictAcknowledgement({ type: ITC_EVENT_TYPES.SCHEMA, message: { originator: 0 } }, 50); + } finally { + worker.wasShutdown = true; + await worker.terminate(); + } + }); + + it('publishes drop-barrier readiness when opening the first RocksDB store', async function () { + const worker = startWorker(UNREADY_WORKER_FIXTURE, { + name: THREAD_TYPES.JOB, + workerIndex: 1, + threadCount: 2, + autoRestart: false, + }); + try { + await new Promise((resolve, reject) => { + worker.once('online', resolve); + worker.once('error', reject); + }); + assert.strictEqual(Atomics.load(worker.itcReadySignal, 0), 0); + const storageOpened = new Promise((resolve) => + worker.on('message', (message) => message.type === 'storage-opened' && resolve()) + ); + worker.postMessage({ type: 'open-storage', table: `DropStorageReady_${process.pid}_${Date.now()}` }); + await storageOpened; + assert.strictEqual(Atomics.load(worker.itcReadySignal, 0), 1); + const storageClosed = new Promise((resolve) => + worker.on('message', (message) => message.type === 'storage-closed' && resolve()) + ); + worker.postMessage({ type: 'close-storage' }); + await storageClosed; + } finally { + worker.wasShutdown = true; + await worker.terminate(); + } + }); + + it('NACKs a malformed strict schema event', async function () { + const worker = startDropWorker(1, 2); + try { + await worker.booted; + assert.strictEqual(worker.worker.itcReady, true); + worker.worker.itcReady = false; + assert.strictEqual(Atomics.load(worker.worker.itcReadySignal, 0), 1); + await assert.rejects( + () => + broadcastWithStrictAcknowledgement( + { + type: ITC_EVENT_TYPES.SCHEMA, + message: { operation: TABLE_DROP_PREPARE_OPERATION }, + }, + 1000 + ), + /originator/i + ); + } finally { + await worker.shutdown(); + } + }); + + it('does not validate unrelated traffic handled by the shared worker listener', async function () { + const worker = startDropWorker(1, 2); + try { + await worker.booted; + worker.send('send-foreign-strict'); + await worker.nextEvent('foreign-strict-acknowledged'); + } finally { + await worker.shutdown(); + } + }); + + it('quiesces a live class before deferring its unquiesced tombstone', async function () { + const databaseName = `DropReconcileDb_${process.pid}_${Date.now()}`; + const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; + const storagePath = path.join(testPath, 'reconcile-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const Table = defineTable(tableName, false, databaseName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: databaseName, table: null }).dbisDb; + const meta = dbisDb.getSync(`${tableName}/`); + meta.dropping = true; + meta.dropGeneration = 'unquiesced-test'; + meta.dropQuiesced = false; + meta.dropProcessInstance = getProcessInstanceId(); + dbisDb.putSync(`${tableName}/`, meta); + + resetDatabases(); + assert.strictEqual( + getDatabases()[databaseName]?.[tableName], + undefined, + 'an unquiesced table must stay unloaded' + ); + assert.strictEqual( + dbisDb.getSync(`${tableName}/`)?.dropping, + true, + 'recovery must preserve the tombstone while stale handles can still exist in this process' + ); + + assert.strictEqual(Table.primaryStore.dropping, true, 'reconcile must mark the removed class as dropping'); + await prepareTableDrop(rootStore.path, tableName, meta.dropGeneration); + await waitFor( + () => { + try { + Table.primaryStore.getSync('__reconcile-close-probe__'); + return false; + } catch { + return true; + } + }, + { message: 'the strict barrier must close a reconciled class retained for preparation' } + ); + + const priorProcessMeta = dbisDb.getSync(`${tableName}/`); + priorProcessMeta.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, priorProcessMeta); + delete databases[databaseName]?.[tableName]; + resetDatabases(); + assert.strictEqual(getDatabases()[databaseName]?.[tableName], undefined); + assert.strictEqual( + database({ database: databaseName, table: null }).dbisDb.getSync(`${tableName}/`), + undefined, + 'a tombstone from a prior process should complete on restart' + ); + } finally { + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + + it('retries a timed-out preparation on a class already removed from the live schema', async function () { + this.timeout(25000); + const tableName = `DropPreparationRetry_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const rootStore = Table.primaryStore.rootStore; + const context = {}; + let staged; + const stagedPromise = new Promise((resolve) => { + staged = resolve; + }); + let releaseTransaction; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + const transactionPromise = transaction(context, async () => { + await Table.put({ id: 'held-for-retry', name: 'pending' }, context); + staged(); + await transactionGate; + }); + await stagedPromise; + + await assert.rejects(() => Table.dropTable(), /timed out after 10000ms/); + assert.strictEqual(databases.test?.[tableName], undefined); + assert.ok(rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); + assert.throws( + () => Table._readTxnForContext({}), + (error) => error?.code === 'ERR_TABLE_DROPPING', + 'a stale table-class reference must not admit a new read after preparation starts' + ); + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const retryPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'the retry must start a fresh drain'); + } catch (error) { + earlyError = error; + } finally { + releaseTransaction(); + } + const [transactionResult, retryResult] = await Promise.allSettled([transactionPromise, retryPromise]); + if (earlyError) throw earlyError; + if (transactionResult.status === 'rejected') throw transactionResult.reason; + if (retryResult.status === 'rejected') throw retryResult.reason; + assert.ok(!rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); + }); + + it('fails closed when a worker cannot quiesce', async function () { + this.timeout(30000); + const tableName = `DropQuiescenceFailure_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: 'test', table: null }).dbisDb; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName }); + const ready = await remote.nextEvent('ready'); + assert.strictEqual(ready.processInstanceId, getProcessInstanceId()); + remote.send('reject-prepare'); + await remote.nextEvent('reject-prepare-armed'); + + const dropResult = await Table.dropTable().then( + () => ({ outcome: 'resolved' }), + (error) => ({ outcome: 'rejected', error }) + ); + assert.strictEqual(dropResult.outcome, 'rejected', 'a worker NACK must reject the live drop'); + assert.match(dropResult.error.message, /injected worker quiescence failure/); + assert.ok( + rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'no table column family may be dropped after a worker NACK' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true); + assert.strictEqual(tombstone?.dropQuiesced, false); + assert.deepStrictEqual(remote.errors, [], 'the expected NACK must not become an unhandled rejection'); + } finally { + await shutdownWorkers(remote); + const tombstone = dbisDb.getSync(`${tableName}/`); + if (tombstone?.dropping) { + tombstone.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, tombstone); + resetDatabases(); + getDatabases(); + } + } + }); + + it('quiesces peer workers when the coordinator drain fails', async function () { + this.timeout(30000); + const tableName = `DropCoordinatorFailure_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + const originalPrepareDrop = Table._prepareDrop; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName }); + await remote.nextEvent('ready'); + Table._prepareDrop = async function (options) { + await originalPrepareDrop.call(this, options); + throw new Error('injected coordinator drain failure'); + }; + + const remotePrepared = remote.nextEvent('prepare-finished'); + await assert.rejects(() => Table.dropTable(), /injected coordinator drain failure/); + assert.strictEqual( + (await remotePrepared).handlesClosed, + true, + 'a durable tombstone must stop peer admission even when the coordinator fails' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true); + assert.strictEqual(tombstone?.dropQuiesced, false); + } finally { + Table._prepareDrop = originalPrepareDrop; + if (dbisDb.getSync(`${tableName}/`)?.dropping) await Table.dropTable(); + await shutdownWorkers(remote); + } + }); + + it('continues after a worker fully exits during preparation', async function () { + this.timeout(30000); + // Force-terminating a worker intentionally bypasses closeLoadedDatabases(), so rocksdb-js keeps + // that worker's process-global handle and snapshot watermark. Isolate the test's sacrificial + // database so the leaked snapshot cannot pin later blob-reclamation tests on the shared test DB. + const databaseName = `DropWorkerExitDb_${process.pid}_${Date.now()}`; + const tableName = `DropWorkerExit_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName, false, databaseName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: databaseName, table: null }).dbisDb; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName, database: databaseName }); + await remote.nextEvent('ready'); + remote.send('begin-transaction', { id: 'exiting-worker-staged' }); + await remote.nextEvent('transaction-staged'); + + const dropPromise = Table.dropTable(); + await remote.nextEvent('prepare-entered'); + remote.worker.wasShutdown = true; + await remote.worker.terminate(); + remote = undefined; + await dropPromise; + + assert.ok( + !rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'a fully exited worker can no longer issue writes through its stale handles' + ); + } finally { + await shutdownWorkers(remote); + const tombstone = dbisDb.getSync(`${tableName}/`); + if (tombstone?.dropping) { + tombstone.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, tombstone); + resetDatabases(); + getDatabases(); + } + } + }); + + it('drains a remote staged transaction before a worker-originated drop', async function () { + this.timeout(30000); + const tableName = `DropRemoteTransaction_${process.pid}_${Date.now()}`; + defineTable(tableName); + let origin; + let remote; + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName, withAlias: true }); + await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + + remote.send('begin-transaction', { id: 'remote-staged' }); + await remote.nextEvent('transaction-staged'); + const dropResultPromise = origin.nextEvent('drop-result'); + let dropSettled = false; + dropResultPromise.then(() => { + dropSettled = true; + }); + origin.send('drop-table'); + await remote.nextEvent('prepare-entered'); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(dropSettled, false, 'the drop must wait for the remote transaction'); + + remote.send('release-transaction'); + await remote.nextEvent('transaction-resolved'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true); + const dropResult = await dropResultPromise; + assert.strictEqual(dropResult.outcome, 'resolved'); + assert.strictEqual(dropResult.aliasPreparations, 1, 'the originating worker must prepare its alias only once'); + assert.strictEqual(dropResult.aliasClosedStores, false, 'the relayed barrier must not close coordinator handles'); + assert.deepStrictEqual([...origin.errors, ...remote.errors], []); + } finally { + await shutdownWorkers(origin, remote); + } + }); + + it('drains a remote read iterator before closing its handles', async function () { + this.timeout(30000); + const tableName = `DropRemoteRead_${process.pid}_${Date.now()}`; + defineTable(tableName); + let origin; + let remote; + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName }); + await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + + remote.send('begin-read'); + await remote.nextEvent('read-open'); + const dropResultPromise = origin.nextEvent('drop-result'); + let dropSettled = false; + dropResultPromise.then(() => { + dropSettled = true; + }); + origin.send('drop-table'); + await remote.nextEvent('prepare-entered'); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(dropSettled, false, 'the drop must wait for the remote iterator'); + + remote.send('release-read'); + await remote.nextEvent('read-resolved'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true); + const dropResult = await dropResultPromise; + assert.strictEqual(dropResult.outcome, 'resolved'); + assert.deepStrictEqual([...origin.errors, ...remote.errors], []); + } finally { + await shutdownWorkers(origin, remote); + } + }); + + it('quiesces a remote source-cache write and recovers after the column family is already gone', async function () { + this.timeout(30000); + const tableName = `DropWorkerRace_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName, true); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + const rootStore = Table.primaryStore.rootStore; + let origin; + let remote; + + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName, withEmbed: true }); + origin.send('initialize', { table: tableName, withEmbed: true }); + const readyWorkers = await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + assert.deepStrictEqual( + readyWorkers.map((message) => message.processInstanceId), + [getProcessInstanceId(), getProcessInstanceId()], + 'every worker must share the main thread process-incarnation marker' + ); + remote.send('begin-source-read', { id: 'remote-source' }); + await remote.nextEvent('embed-entered'); + + const prepareEntered = remote.nextEvent('prepare-entered').then(() => 'prepare'); + const dropResultPromise = origin.nextEvent('drop-result'); + origin.send('drop-table', { interruptAfterColumnFamilyDrop: true }); + const firstOutcome = await Promise.race([prepareEntered, dropResultPromise.then(() => 'drop-settled')]); + assert.strictEqual( + firstOutcome, + 'prepare', + 'the main-thread coordinator must put the remote worker in the pre-drop drain before the worker-originated drop settles' + ); + + remote.send('release-embed'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true, 'the remote worker must close its table handle before ACK'); + await remote.nextEvent('source-read-resolved'); + + const dropResult = await dropResultPromise; + assert.strictEqual( + dropResult.outcome, + 'rejected', + 'the injected interruption must fail the live drop after removing the primary column family' + ); + assert.match(dropResult.error, /injected interruption/); + + assert.ok( + !rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'the table column families must already be absent before recovery begins' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true, 'the failed catalog cleanup must retain the tombstone'); + assert.strictEqual(tombstone?.dropQuiesced, true, 'the tombstone must record completed worker quiescence'); + + resetDatabases(); + const reloaded = getDatabases(); + assert.strictEqual(reloaded.test?.[tableName], undefined, 'recovery must not resurrect the table'); + assert.strictEqual( + database({ database: 'test', table: null }).dbisDb.getSync(`${tableName}/`), + undefined, + 'recovery must remove the tombstone after confirming the column families are absent' + ); + assert.deepStrictEqual( + [...origin.errors, ...remote.errors], + [], + 'no worker rejection should escape the quiescence or recovery path' + ); + } finally { + await shutdownWorkers(origin, remote); + } + }); +}); diff --git a/unitTests/resources/dropTableUnready-worker.js b/unitTests/resources/dropTableUnready-worker.js new file mode 100644 index 0000000000..6d46c7917c --- /dev/null +++ b/unitTests/resources/dropTableUnready-worker.js @@ -0,0 +1,22 @@ +'use strict'; + +const { parentPort } = require('node:worker_threads'); + +// Deliberately loads no storage or ITC module until the test requests it. +parentPort?.on('message', (message) => { + if (message.type === 'open-storage') { + require('../testUtils'); + const { setupTestDBPath } = require('../testUtils'); + setupTestDBPath(); + const { table } = require('#src/resources/databases'); + table({ + database: 'test', + table: message.table, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + parentPort.postMessage({ type: 'storage-opened' }); + } else if (message.type === 'close-storage') { + require('#src/resources/databases').closeLoadedDatabases(); + parentPort.postMessage({ type: 'storage-closed' }); + } +}); diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index e75b8ba6cc..4b9053f8a5 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -160,4 +160,48 @@ describe('commit with open read iterators commits writes immediately on a replay await delay(100); assert.deepEqual(unhandled, [], 'a replay-commit failure must reject the awaited chain, never float unhandled'); }); + + it('a synchronous replay commit failure leaves the retained iterator usable', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const originalCommit = Transaction.prototype.commit; + const targetDb = LingerTable.primaryStore.store.db; + const context = {}; + let failedTransaction; + let synchronousFailure; + let iterator; + let rejection; + try { + await transaction(context, async () => { + const results = await LingerTable.search({ conditions: [] }, context); + iterator = results[Symbol.asyncIterator](); + await iterator.next(); + await LingerTable.put({ id: 'linger-sync-fail', v: 42 }, context); + failedTransaction = context.transaction; + Transaction.prototype.commit = function (...args) { + if (this.store?.db !== targetDb) return originalCommit.apply(this, args); + synchronousFailure = Object.assign(new Error('forced synchronous replay failure'), { code: 'ERR_BUSY' }); + throw synchronousFailure; + }; + }); + } catch (error) { + rejection = error; + } finally { + Transaction.prototype.commit = originalCommit; + } + assert.match(rejection?.message, /forced synchronous replay failure/); + assert.strictEqual(rejection.code, 'ERR_BUSY', 'the terminal failure must retain its public error code'); + assert.strictEqual(rejection.cause?.code, 'ERR_BUSY', 'the terminal failure must retain the native cause'); + assert.strictEqual( + rejection.stack, + synchronousFailure.stack, + 'the terminal failure must retain its original stack' + ); + assert.strictEqual(context.transaction, failedTransaction, 'context release must wait for the retained iterator'); + assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); + assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); + while (!(await iterator.next()).done); + assert.strictEqual(failedTransaction.transaction, null, 'draining the iterator must release its read handle'); + assert.strictEqual(context.transaction, null, 'draining the iterator must release the context back-reference'); + assert.equal(await LingerTable.get('linger-sync-fail'), null, 'the failed replay must not commit its record'); + }); }); diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index 38dd5024ca..613d611be1 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -189,6 +189,26 @@ describe('Transactions', () => { await committed; assert.deepEqual(order, ['completion', 'commit'], 'commit resolved only after the callback completion'); }); + it('preserves the application error when abort cleanup also fails', function () { + const applicationError = new Error('application failure'); + assert.throws( + () => + transaction({}, (txn) => { + txn.addWrite({ + store: { + getEntry() { + throw new Error('abort cleanup failure'); + }, + }, + key: 'pending', + savedBlobs: [], + deferSave: true, + }); + throw applicationError; + }), + (error) => error === applicationError + ); + }); it('Can run txn with three tables and two databases', async function () { const context = {}; let start = Date.now(); diff --git a/unitTests/server/threads/workerDataProviders.test.js b/unitTests/server/threads/workerDataProviders.test.js index 9423784fd8..fa3e1a8b59 100644 --- a/unitTests/server/threads/workerDataProviders.test.js +++ b/unitTests/server/threads/workerDataProviders.test.js @@ -13,6 +13,7 @@ describe('registerWorkerDataProvider', () => { it('rejects reserved workerData keys, duplicate names, and non-function providers', () => { assert.throws(() => registerWorkerDataProvider('ticketKeys', () => 1), /already in use/); assert.throws(() => registerWorkerDataProvider('addPorts', () => 1), /already in use/); + assert.throws(() => registerWorkerDataProvider('processInstanceId', () => 'other'), /already in use/); // consumed by threadServer.js, not spread by startWorker — must be reserved all the same assert.throws(() => registerWorkerDataProvider('noServerStart', () => true), /already in use/); assert.throws(() => registerWorkerDataProvider('__proto__', () => ({})), /already in use/); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 7828ffde61..3ff00e9849 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -923,6 +923,7 @@ export const JWT_ENUM = { /** ITC Channel Event types */ export const ITC_EVENT_TYPES = { SHUTDOWN: 'shutdown', + ITC_READY: 'itc_ready', CHILD_STARTED: 'child_started', CHILD_STOPPED: 'child_stopped', SCHEMA: 'schema', @@ -951,6 +952,8 @@ export const ITC_EVENT_TYPES = { OPERATION_EXECUTE_RESPONSE: 'operation_execute_response', } as const; +export const TABLE_DROP_PREPARE_OPERATION = 'prepare_drop_table'; + /** Supported thread types */ export const THREAD_TYPES = { HTTP: 'http', diff --git a/utility/signalling.ts b/utility/signalling.ts index aae59d81cd..a748af7630 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -4,7 +4,7 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; let serverItcHandlers; -import { sendItcEvent } from '../server/threads/itc.js'; +import { sendItcEvent, sendItcEventStrict } from '../server/threads/itc.js'; // Await BOTH the local handler and the cross-worker broadcast. The local handler is what // rebuilds THIS thread's cache; firing it un-awaited let the originating worker return success @@ -23,6 +23,15 @@ export async function signalSchemaChange(message: any) { } } +/** + * Quiesce a table on every connected thread before its RocksDB handles are dropped. Unlike normal + * post-change gossip, this rejects unless every handler succeeds; the caller must fail closed. + */ +export function signalTableDropPreparation(message: any) { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, message); + return sendItcEventStrict(event); +} + /** * Notify local listeners that JS resources have just been registered (resources.js loaded). This is * deliberately local-only — no ITC broadcast — because every worker registers its own JS resources,