From 5a42b3bf319e854074afceb7664a7d28f0a8ed8e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 12:14:01 -0600 Subject: [PATCH 01/40] fix: quiesce workers before dropping RocksDB tables Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 57 +++- resources/Table.ts | 267 ++++++++-------- resources/databases.ts | 85 ++++- server/itc/serverHandlers.js | 6 +- server/threads/itc.js | 64 +++- server/threads/manageThreads.js | 110 +++++-- .../resources/dropTableQuiescence-worker.js | 123 ++++++++ .../resources/dropTableQuiescence.test.js | 297 ++++++++++++++++++ .../threads/workerDataProviders.test.js | 1 + utility/hdbTerms.ts | 4 + utility/signalling.ts | 11 +- 11 files changed, 844 insertions(+), 181 deletions(-) create mode 100644 unitTests/resources/dropTableQuiescence-worker.js create mode 100644 unitTests/resources/dropTableQuiescence.test.js diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index ac8a68e770..09477d3acd 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -9,7 +9,12 @@ import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import { convertToMS } from '../utility/common_utils.ts'; import { when } from '../utility/when.ts'; import { setTimeout as delay } from 'node:timers/promises'; -import { Transaction as RocksTransaction, type Store as RocksStore, constants } from '@harperfast/rocksdb-js'; +import { + RocksDatabase, + Transaction as RocksTransaction, + type Store as RocksStore, + constants, +} from '@harperfast/rocksdb-js'; const RETRY_NOW_VALUE = constants.RETRY_NOW_VALUE; import type { RootDatabaseKind } from './databases.ts'; import type { Entry } from './RecordEncoder.ts'; @@ -21,6 +26,7 @@ const trackedTxns = new Set(); // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the // chain root — so a chain child can never become its own timeout root (issue #2231). const supervisedWriteRoots = new Set(); +const activeWriteTransactions = new Set(); const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); export const TRANSACTION_STATE = { @@ -139,6 +145,23 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number | oldestAgeMs: oldestOutstandingCommit ? performance.now() - oldestOutstandingCommit.start : undefined, }; } + +/** + * Snapshot the transactions on this worker that have staged writes against any of the supplied stores. + * A table drop marks its stores as dropping before calling this, so no later addWrite can enter the set; + * awaiting the returned promises therefore establishes a closed drain boundary before handles are closed. + */ +export function getPendingWriteResolutions(stores: Iterable): Promise[] { + const targetStores = new Set(stores); + const resolutions: Promise[] = []; + for (const transaction of activeWriteTransactions) { + if (transaction.writes.some((write) => write && targetStores.has(write.store))) { + const resolution = transaction.getPendingWriteResolution(); + if (resolution) resolutions.push(resolution); + } + } + return resolutions; +} // Once per process: committing under open read iterators forces a write replay, so the warning is // about the caller's pattern, not the individual commit. let replayedWritesWarned = false; @@ -325,6 +348,8 @@ type RocksTransactionWithRetry = RocksTransaction & { isRetry?: boolean }; export class DatabaseTransaction implements Transaction { #context: Context; + #pendingWriteResolution?: Promise; + #resolvePendingWrites?: () => void; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) declare writesByKey?: Map>; @@ -546,6 +571,17 @@ export class DatabaseTransaction implements Transaction { * (see priorStagedWrite). Called by both engines' addWrite. */ linkWrite(operation: TransactionWrite): void { + if (operation.store?.dropping) { + const databaseName = operation.store.rootStore?.databaseName; + const tableName = String(operation.store.name ?? '').replace(/\/$/, ''); + const error: any = new ServerError( + `Table ${databaseName ? databaseName + '.' : ''}${tableName || 'unknown'} is being dropped`, + 409 + ); + error.code = 'ERR_TABLE_DROPPING'; + throw error; + } + if (operation.store?.rootStore instanceof RocksDatabase) activeWriteTransactions.add(this); if (operation.key === undefined) return; let writesForStore = (this.writesByKey ??= new Map()).get(operation.store); if (!writesForStore) this.writesByKey.set(operation.store, (writesForStore = new Map())); @@ -560,10 +596,26 @@ export class DatabaseTransaction implements Transaction { * reused transaction never bases a write on a previous batch's staged state. */ clearWrites(): void { + this.finishPendingWrites(); this.writes = []; this.writesByKey = undefined; } + getPendingWriteResolution(): Promise | undefined { + if (!activeWriteTransactions.has(this)) return; + this.#pendingWriteResolution ??= new Promise((resolve) => { + this.#resolvePendingWrites = resolve; + }); + return this.#pendingWriteResolution; + } + + private finishPendingWrites(): void { + activeWriteTransactions.delete(this); + this.#resolvePendingWrites?.(); + this.#pendingWriteResolution = undefined; + this.#resolvePendingWrites = undefined; + } + /** * Drop this transaction's back-reference from its context once completed (commit or abort), * so a long-lived context (e.g. an MQTT subscription context held open for the life of a @@ -1079,6 +1131,7 @@ export class DatabaseTransaction implements Transaction { } catch (abortError) { harperLogger.debug?.('aborting transaction after failed commit', abortError); } + this.finishPendingWrites(); // A terminal failure is just as final as a success — release the context's // back-reference here too, or transaction.ts's onComplete() (which has no // rejection handler of its own) would leave a long-lived context pinning this @@ -1251,6 +1304,8 @@ export class DatabaseTransaction implements Transaction { harperLogger.debug?.('cleaning up after a failed synchronous commit', abortError); } throw error; + } finally { + this.finishPendingWrites(); } this.detachOwnedTransaction(); } diff --git a/resources/Table.ts b/resources/Table.ts index dc4a2836e4..56d30cdd2f 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -10,6 +10,7 @@ import { SYSTEM_TABLE_NAMES, SYSTEM_SCHEMA_NAME, MAX_SET_TIMEOUT_MS, + TABLE_DROP_PREPARE_OPERATION, } from '../utility/hdbTerms.ts'; import { type Database } from 'lmdb'; import { Script } from 'node:vm'; @@ -38,6 +39,7 @@ import { priorStagedWrite, isReleasedTransaction, TRANSACTION_STATE, + getPendingWriteResolutions, } from './DatabaseTransaction.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; import { addSubscription } from './transactionBroadcast.ts'; @@ -51,7 +53,7 @@ import { } from '../utility/errors/hdbError.ts'; import * as signalling from '../utility/signalling.ts'; import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js'; -import { databases, table } from './databases.ts'; +import { databases, table, prepareTableDrop } from './databases.ts'; import { searchByIndex, findAttribute, @@ -66,7 +68,7 @@ import { isStaticResourceInstance } from './staticResourceDispatch.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; -import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js'; +import { getWorkerIndex, getWorkerCount, getProcessInstanceId } from '../server/threads/manageThreads.js'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; import { buildEmbedBefore, createDefaultEmbedder, type EmbedAttribute, type Embedder } from './models/embedHook.ts'; import { autoCast, autoCastBooleanStrict } from '../utility/common_utils.ts'; @@ -404,6 +406,38 @@ export function makeTable(options) { // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); let droppingTable = false; + let coordinatingDrop = false; + let storesClosed = false; + let dropPreparation: Promise | undefined; + const tableStores = () => [...Object.values(indices), primaryStore].filter(Boolean); + const markTableDropping = () => { + droppingTable = true; + for (const store of tableStores()) (store as any).dropping = true; + delete databases[databaseName]?.[tableName]; + }; + const drainTableWrites = async () => { + const pending = new Set>([...pendingSourceCommits, ...getPendingWriteResolutions(tableStores())]); + if (!pending.size) return; + let timer: NodeJS.Timeout; + const timedOut = Symbol('timedOut'); + const result = await Promise.race([ + Promise.allSettled(pending), + new Promise((resolve) => { + timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); + }), + ]); + clearTimeout(timer); + if (result === timedOut) { + throw new Error( + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight write(s) on ${tableName} to settle; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` + ); + } + }; + const closeTableStores = () => { + if (storesClosed) return; + for (const store of tableStores()) store.close?.(); + storesClosed = true; + }; let createdTimeProperty: Attribute | undefined, updatedTimeProperty: Attribute | undefined, expiresAtProperty: Attribute | undefined; @@ -1359,163 +1393,122 @@ export function makeTable(options) { return coerceType(id, primaryKeyAttribute); } + static async _prepareDrop({ closeStores = true } = {}) { + markTableDropping(); + dropPreparation ??= drainTableWrites(); + await dropPreparation; + if (closeStores && !coordinatingDrop) closeTableStores(); + } + static async dropTable() { + const rootStore = primaryStore.rootStore; + const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; + let dropGeneration: string | undefined; if (databaseName === databasePath) { - // Persist a drop tombstone on the primary catalog entry BEFORE any - // destructive work. If the process dies or a column family drop fails - // partway through, the tombstone survives with the catalog rows, and - // the next startup (or a same-name create) completes the drop via - // completeInterruptedDrop in databases.ts instead of resurrecting - // the table. const primaryCatalogKey = TableResource.tableName + '/'; const primaryMeta = (dbisDb as any).getSync(primaryCatalogKey); if (primaryMeta && !primaryMeta.dropping) { primaryMeta.dropping = true; - // Stamps this drop's identity so the interrupted-drop retry budget in - // databases.ts can be scoped to THIS drop rather than the table name: a - // worker that exhausts the budget for a table can observe the catalog - // mid-flight between this drop's completion and a same-name recreate's - // own drop, without ever seeing a non-tombstoned row to reset on. Keying - // the budget by generation instead makes the new drop's tombstone carry - // its own fresh key regardless of what any worker last observed. primaryMeta.dropGeneration = randomUUID(); - // put is rebound to putSync on RocksDB stores; on LMDB it returns - // a promise, so await it to make the tombstone durable before the - // destructive work below + primaryMeta.dropQuiesced = !sharedRocksStore; + // A random process-start identity (not the PID, which containers commonly reuse) lets + // recovery distinguish a live process that may still hold handles from a clean restart. + if (sharedRocksStore) primaryMeta.dropProcessInstance = getProcessInstanceId(); const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); if (tombstoneWrite?.then) await tombstoneWrite; } + dropGeneration = primaryMeta?.dropGeneration; } - // A get() against a sourcedFrom table resolves to its caller before the resolved - // record's cache write has committed (see getFromSource) - the write lands "in the - // background" for latency reasons. Flip this BEFORE removing the table from the - // schema below: getFromSource() checks it and skips caching (treats the load as - // noCacheStore) for any call it admits from here on, including one that slipped in - // through a stale reference to this Table between the two steps. - droppingTable = true; - // Remove the table from the in-memory schema immediately so concurrent - // requests get "table does not exist" instead of racing the column - // family drops below. If a drop fails past this point the table stays - // invisible, and the tombstone guarantees the drop completes on the - // next startup (or on a same-name create). - delete databases[databaseName][tableName]; - // The above stops new source-fill writes from starting, but a write from a get() - // that already returned to its caller may still be in flight. Dropping the column - // families out from under that write is a genuine invariant violation, not just a - // benign race: RocksDB rejects the still-open write batch with "Invalid column - // family specified in write batch" (or "Could not access column family N"), which - // can also abort this drop before it removes the tombstoned catalog rows - leaving - // the table stuck "dropping" for completeInterruptedDrop to retry (and fail - // identically) on every subsequent load (harper#1381). Drain any in-flight commits - // before the blob sweep below (so it observes every row a drain-caught write just - // committed) and before touching a single column family. - // - // Bounded, and fails CLOSED: the tracked promise covers the whole source round-trip - // plus the local commit (see getFromSource), so a hung/slow source or a slow commit - // (e.g. a large blob write) could otherwise wedge this drop forever. Rather than - // give up and drop anyway - which would reopen exactly the race this drain exists to - // close, just less often - a timeout FAILS the drop. The tombstone written above is - // already durable, so completeInterruptedDrop picks the drop back up on the next - // load, once the stuck write has had time to finish. - if (pendingSourceCommits.size) { - const pending = [...pendingSourceCommits]; - let timer: NodeJS.Timeout; - const timedOut = Symbol('timedOut'); - const result = await Promise.race([ - Promise.allSettled(pending), - new Promise((resolve) => { - timer = setTimeout(() => resolve(timedOut), LOCK_TIMEOUT); - }), - ]); - clearTimeout(timer); - if (result === timedOut) { - throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.length} in-flight source-populated cache write(s) on ${tableName} to settle; refusing to drop the column families out from under a write that may still be staged. The drop tombstone is durable, so this will be retried on the next load.` - ); + coordinatingDrop = true; + let locallyQuiesced = false; + try { + if (sharedRocksStore) { + await prepareTableDrop(rootStore.path, tableName, dropGeneration, TableResource); + locallyQuiesced = true; + await signalling.signalTableDropPreparation({ + originator: process.pid, + operation: TABLE_DROP_PREPARE_OPERATION, + schema: databaseName, + table: tableName, + path: rootStore.path, + dropGeneration, + }); + } else { + await TableResource._prepareDrop({ closeStores: false }); + locallyQuiesced = true; } - } - for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { - if (entry.metadataFlags & HAS_BLOBS && entry.value) { - deleteBlobsInObject(entry.value); + + for (const entry of primaryStore.getRange({ versions: true, snapshot: false, lazy: true })) { + if (entry.metadataFlags & HAS_BLOBS && entry.value) deleteBlobsInObject(entry.value); } - } - if (databaseName === databasePath) { - // part of a database. - // Drop the column families, then remove the catalog metadata - never - // the reverse: a removed-then-failed drop orphans a "ghost" column - // family that poisons same-name recreates, so a genuine drop failure - // must surface and leave the tombstoned catalog rows for the reconcile. - // - // A drop is broadcast to every worker thread, and each holds its own - // handle to the same underlying column family, so a concurrent worker - // (or completeInterruptedDrop) may already have dropped it - surfaced - // as "Column family already dropped!". That is the intended end state, - // not a failure, so tolerate it. The catalog rows are removed only if - // this drop's tombstone is still the live primary row: a concurrent - // same-name create completes the interrupted drop and writes fresh - // catalog rows, and clobbering those would orphan the new table. - const removeTombstonedCatalog = () => { - const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); - if (!currentPrimary?.dropping) return false; - for (const attribute of attributes) { - dbisDb.remove(TableResource.tableName + '/' + attribute.name); - } - dbisDb.remove(TableResource.tableName + '/'); - return true; - }; - const rootStore = primaryStore.rootStore; - if (rootStore instanceof RocksDatabase) { - // Serialize the drops + catalog removal against a concurrent - // same-name create (and completeInterruptedDrop) under the database's - // 'update-attributes' exclusive lock - the same lock the create path - // holds. It is a synchronous spin lock that blocks the event loop, so - // the locked section MUST stay synchronous: drop with dropSync (as - // completeInterruptedDrop does), never an awaited drop(), or a - // concurrent create's spin would deadlock waiting on a drop that the - // blocked event loop can never resolve. - while (!rootStore.tryLock('update-attributes')) {} - let removed = false; - try { + + if (databaseName === databasePath) { + const removeTombstonedCatalog = () => { + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping || currentPrimary.dropGeneration !== dropGeneration) return false; for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) - try { - index.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); - } + dbisDb.remove(TableResource.tableName + '/' + attribute.name); } + dbisDb.remove(TableResource.tableName + '/'); + return true; + }; + if (rootStore instanceof RocksDatabase) { + while (!rootStore.tryLock('update-attributes')) {} + let removed = false; try { - primaryStore.dropSync(); - } catch (error) { - ignoreAlreadyDropped(error); + const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); + if (!currentPrimary?.dropping || currentPrimary.dropGeneration !== dropGeneration) { + throw new ServerError(`Drop generation changed while preparing ${databaseName}.${tableName}`, 409); + } + currentPrimary.dropQuiesced = true; + (dbisDb as any).putSync(TableResource.tableName + '/', currentPrimary); + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) + try { + index.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + } + try { + primaryStore.dropSync(); + } catch (error) { + ignoreAlreadyDropped(error); + } + closeTableStores(); + removed = removeTombstonedCatalog(); + } finally { + rootStore.unlock('update-attributes'); } - removed = removeTombstonedCatalog(); - } finally { - rootStore.unlock('update-attributes'); + if (removed) await dbisDb.committed; + } else { + const drops = []; + for (const attribute of attributes) { + const index = indices[attribute.name]; + if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + } + drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); + await Promise.all(drops); + if (removeTombstonedCatalog()) await dbisDb.committed; } - if (removed) await dbisDb.committed; } else { - // LMDB: no shared column-family double-drop, and its engine lock is - // transactional rather than this spin lock, so keep the awaited drop - // plus the same tombstone-guarded catalog removal. - const drops = []; - for (const attribute of attributes) { - const index = indices[attribute.name]; - if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + await primaryStore.close(); + fs.unlinkSync(primaryStore.path); + } + await signalling.signalSchemaChange( + new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) + ); + } finally { + coordinatingDrop = false; + if (locallyQuiesced && rootStore instanceof RocksDatabase && !storesClosed) { + try { + closeTableStores(); + } catch (error) { + logger.warn(`Failed to close table handles for ${databaseName}.${tableName}`, error); } - drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); - await Promise.all(drops); - if (removeTombstonedCatalog()) await dbisDb.committed; } - } else { - // legacy table per database - await primaryStore.close(); - fs.unlinkSync(primaryStore.path); } - signalling.signalSchemaChange( - new SchemaEventMsg(process.pid, OPERATIONS_ENUM.DROP_TABLE, databaseName, tableName) - ); } // #section: read-path /** diff --git a/resources/databases.ts b/resources/databases.ts index 55cd4fba2b..8ad8591ee2 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -772,11 +772,22 @@ function initStores( clearInterruptedDropEntries(path, tableName); continue; } + // A tombstone always removes the worker-local class, even when this worker cannot safely + // perform the physical cleanup. Keeping it in definedTables would leave a stale class and + // its dropped handles reachable after this reconcile pass. + definedTables?.delete(tableName); + if (!canCompleteInterruptedDrop(tableDef.primary)) { + logger.debug( + `Deferring interrupted drop of table ${databaseName}.${tableName} until worker quiescence or a clean process start` + ); + tablesToLoad.delete(tableName); + continue; + } const generation = tableDef.primary?.dropGeneration; const failedAttempts = getInterruptedDropAttempts(path, tableName, generation); if (failedAttempts < MAX_INTERRUPTED_DROP_ATTEMPTS) { try { - completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName); + completeInterruptedDropWithLock(rootStore, attributesDbi, databaseName, tableName, generation); // Sweep every generation this worker has ever tracked for this table, not // just the one just resolved: if a prior generation was exhausted here, // then resolved+recreated+re-dropped by another worker as this generation @@ -784,7 +795,6 @@ function initStores( // place that sweeps), the prior generation's entry would otherwise never // be cleared. clearInterruptedDropEntries(path, tableName); - definedTables?.delete(tableName); } catch (error) { const attempt = failedAttempts + 1; setInterruptedDropAttempts(path, tableName, generation, attempt); @@ -1668,7 +1678,13 @@ export function table(tableDefinition: TableDefinition): Tabl // create below starts from a clean slate; treating the tombstoned // entry as an existing table would recurse forever on the stale // catalog row. - completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName); + if (!canCompleteInterruptedDrop(existingTableMeta)) { + throw new ClientError( + `Table '${databaseName}.${tableName}' has an interrupted drop that was not quiesced across workers; restart Harper before recreating it`, + 409 + ); + } + completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, existingTableMeta.dropGeneration); // This resolves the drop without ever going through the schema-load // reconcile below, which is the only other place that returns a spent // budget. Without clearing it here too, a table that gets dropped again @@ -2289,8 +2305,35 @@ async function runIndexing(Table, attributes, indicesToRemove) { * actionable, and logging here on every attempt would flood at the same * volume this function's callers are bounding. */ -function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, tableName: string) { +function completeInterruptedDropWithLock( + rootStore, + attributesDbi, + databaseName: string, + tableName: string, + dropGeneration?: string +) { + if (!(rootStore instanceof RocksDatabase)) { + return completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, dropGeneration); + } + while (!rootStore.tryLock('update-attributes')) {} + try { + return completeInterruptedDrop(rootStore, attributesDbi, databaseName, tableName, dropGeneration); + } finally { + rootStore.unlock('update-attributes'); + } +} + +function completeInterruptedDrop( + rootStore, + attributesDbi, + databaseName: string, + tableName: string, + dropGeneration?: string +) { logger.debug(`Completing interrupted drop of table ${databaseName}.${tableName}`); + const primaryCatalogKey = tableName + '/'; + const primaryMeta = (attributesDbi as any).getSync(primaryCatalogKey); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) return false; if (rootStore instanceof RocksDatabase) { for (const columnName of (rootStore as any).columns) { if (columnName.startsWith(tableName + '/')) { @@ -2331,7 +2374,6 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, // recognizes the table as mid-drop - instead of the tombstone vanishing // first and stranding orphaned attribute rows that the next load would // misread as a live (non-dropping) table. - const primaryCatalogKey = tableName + '/'; let removePrimaryLast = false; for (const key of attributesDbi.getKeys({ start: tableName + '/', end: tableName + '0' })) { if (key === primaryCatalogKey) { @@ -2344,6 +2386,39 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, (attributesDbi as any).removeSync(key); } if (removePrimaryLast) (attributesDbi as any).removeSync(primaryCatalogKey); + return true; +} + +function canCompleteInterruptedDrop(primaryMeta): boolean { + // A completed barrier is safe immediately. An incomplete barrier is safe only after a clean + // process start, when none of the handles from the recorded incarnation can still exist. + // Tombstones written before the incarnation field existed necessarily came from an older process. + return ( + primaryMeta?.dropQuiesced === true || primaryMeta?.dropProcessInstance !== manageThreads.getProcessInstanceId() + ); +} + +/** + * Mark and drain every worker-local class backed by this physical table. The drop origin keeps + * its own handles so it can perform the destructive phase; remote workers close theirs after the + * drain so no stale handle can later contaminate RocksDB's shared write path. + */ +export async function prepareTableDrop( + storePath: string, + tableName: string, + dropGeneration: string | undefined, + preserveTable?: any +): Promise { + const matchingTables = new Set(); + for (const databaseName of Object.getOwnPropertyNames(databases)) { + const databaseTables = databases[databaseName]; + const Table = databaseTables?.[tableName]; + if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; + const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) continue; + matchingTables.add(Table); + } + await Promise.all([...matchingTables].map((Table) => Table._prepareDrop({ closeStores: Table !== preserveTable }))); } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/server/itc/serverHandlers.js b/server/itc/serverHandlers.js index 9b48ef3266..c54c52d4c6 100644 --- a/server/itc/serverHandlers.js +++ b/server/itc/serverHandlers.js @@ -12,7 +12,7 @@ const harperBridge = require('../../dataLayer/harperBridge/harperBridge.ts'); const process = require('process'); const { isMainThread, workerData } = require('worker_threads'); -const { resetDatabases, closeDatabase } = require('../../resources/databases.ts'); +const { resetDatabases, closeDatabase, prepareTableDrop } = require('../../resources/databases.ts'); /** * This object/functions are passed to the ITC client instance and dynamically added as event handlers. @@ -46,6 +46,10 @@ async function schemaHandler(event) { } hdbLogger.trace(`ITC schemaHandler received schema event:`, event); + if (event.message.operation === hdbTerms.TABLE_DROP_PREPARE_OPERATION) { + await prepareTableDrop(event.message.path, event.message.table, event.message.dropGeneration); + return; + } // restore_backup: this thread must release its store handles so the restore can purge and // rewrite the database directory. The rescan below (resetDatabases) skips reloading it while // the restoring marker is present, and reloads it on the completion signal (marker gone). diff --git a/server/threads/itc.js b/server/threads/itc.js index 992237f4e4..5ad4bb0b77 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,27 +3,56 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { threadId } = require('worker_threads'); -const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js'); +const { isMainThread, threadId } = require('worker_threads'); +const harperLogger = require('../../utility/logging/harper_logger.ts'); +const { + onMessageFromWorkers, + broadcastWithAcknowledgement, + broadcastWithStrictAcknowledgement, + sendToThreadWithStrictAcknowledgement, +} = require('./manageThreads.js'); module.exports = { sendItcEvent, + sendItcEventStrict, validateEvent, SchemaEventMsg, UserEventMsg, }; let serverItcHandlers; onMessageFromWorkers(async (event, sender) => { - serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - validateEvent(event); - if (serverItcHandlers[event.type]) { - await serverItcHandlers[event.type](event); + const requestId = event?.requestId; + let handlerError; + try { + serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); + validateEvent(event); + if (serverItcHandlers[event.type]) { + await serverItcHandlers[event.type](event); + } + if (event.relayStrictToWorkers && isMainThread) { + const relayedEvent = { ...event, relayStrictToWorkers: false, requestId: undefined }; + await broadcastWithStrictAcknowledgement(relayedEvent); + } + } catch (error) { + handlerError = error; + } + if (handlerError) harperLogger.error('ITC event handler failed', handlerError); + if (requestId && sender) { + try { + sender.postMessage({ + type: 'ack', + id: requestId, + ...(handlerError && { + error: { + message: handlerError.message ?? String(handlerError), + code: handlerError.code, + }, + }), + }); + } catch (error) { + harperLogger.error('Unable to acknowledge ITC event', error); + } } - if (event.requestId && sender) - sender.postMessage({ - type: 'ack', - id: event.requestId, - }); }); /** @@ -31,11 +60,22 @@ onMessageFromWorkers(async (event, sender) => { * @param event */ function sendItcEvent(event) { + stampOriginator(event); + return broadcastWithAcknowledgement(event); +} + +function sendItcEventStrict(event) { + stampOriginator(event); + if (isMainThread) return broadcastWithStrictAcknowledgement(event); + event.relayStrictToWorkers = true; + return sendToThreadWithStrictAcknowledgement(0, event); +} + +function stampOriginator(event) { // Always stamp originator so handlers can send direct responses back. // The main thread's threadId is 0 (worker_threads convention); parentPort.threadId // is set to 0 in workers, so sendToThread(0, ...) routes back to main. if (event.message) event.message.originator = threadId; - return broadcastWithAcknowledgement(event); } /** diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 87d829c9c3..64c07fdf19 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -17,7 +17,7 @@ const { setHeapSnapshotNearHeapLimit } = typeof globalThis.Bun !== 'undefined' ? const hdbTerms = require('../../utility/hdbTerms.ts'); const envMgr = require('../../utility/environment/environmentManager.ts'); const harperLogger = require('../../utility/logging/harper_logger.ts'); -const { randomBytes } = require('crypto'); +const { randomBytes, randomUUID } = require('crypto'); const { _assignPackageExport } = require('../../globals.js'); const { PACKAGE_ROOT } = require('../../utility/packageUtils.js'); const { resolvePreloadModules } = require('./resolvePreload.ts'); @@ -47,6 +47,11 @@ const isBun = typeof globalThis.Bun !== 'undefined'; const MB = 1024 * 1024; const workers = []; // these are our child workers that we are managing const connectedPorts = []; // these are all known connected worker ports (siblings, children, parents) +const PROCESS_INSTANCE_ENV = 'HARPER_INTERNAL_PROCESS_INSTANCE_ID'; +const processInstanceId = isMainThread + ? randomUUID() + : workerData?.processInstanceId || process.env[PROCESS_INSTANCE_ENV] || randomUUID(); +if (isMainThread) process.env[PROCESS_INSTANCE_ENV] = processInstanceId; const MAX_UNEXPECTED_RESTARTS = 50; // Threads get 10s to die before they're forced. In dev (`harper dev`) we widen this: a reload's old // worker may be disposing a native runtime (e.g. @harperfast/vite's rolldown dev server) and forcing it @@ -142,8 +147,11 @@ module.exports = { onMessageByType, broadcast, broadcastWithAcknowledgement, + broadcastWithStrictAcknowledgement, + sendToThreadWithStrictAcknowledgement, getWorkerIndex, getWorkerCount, + getProcessInstanceId, getTicketKeys, setMainIsWorker, setTerminateTimeout, @@ -189,6 +197,9 @@ function setTerminateTimeout(newTimeout) { function getWorkerIndex() { return workerData ? workerData.workerIndex : isMainWorker ? 0 : undefined; } +function getProcessInstanceId() { + return processInstanceId; +} function getWorkerCount() { return workerData ? workerData.workerCount : isMainWorker ? 1 : undefined; } @@ -209,6 +220,7 @@ const RESERVED_WORKER_DATA_KEYS = [ 'workerCount', 'name', 'restartNumber', + 'processInstanceId', 'ticketKeys', 'noServerStart', '__proto__', // never a legitimate payload name; spread would define it as an own property @@ -387,6 +399,7 @@ function startWorker(path, options = {}) { workerCount: (workerCount = options.threadCount), name: options.name, restartNumber: module.exports.restartNumber, + processInstanceId, ticketKeys: getTicketKeys(), }, transferList: portsToSend, @@ -695,43 +708,89 @@ let nextId = 1; // worker (its port close fires the same ack handlers), so on timeout we proceed best-effort. const DEFAULT_ACK_TIMEOUT_MS = 30000; function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return new Promise((resolve) => { + return broadcastAwaitingAcknowledgements(message, timeout, false, false); +} + +// Destructive work uses the strict variant: every connected thread, including jobs, must finish +// its handler successfully. A timeout, disconnect, handler error, or post failure rejects so the +// caller can leave its durable recovery marker in place without touching storage. +function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { + return broadcastAwaitingAcknowledgements(message, timeout, true, true); +} + +function sendToThreadWithStrictAcknowledgement(threadId, message, timeout = DEFAULT_ACK_TIMEOUT_MS) { + const port = connectedPorts.find((port) => port.threadId === threadId); + if (!port) { + const error = new Error(`Worker thread ${threadId} is not connected`); + error.code = 'ERR_ITC_THREAD_NOT_CONNECTED'; + return Promise.reject(error); + } + return broadcastAwaitingAcknowledgements(message, timeout, true, true, [port]); +} + +function broadcastAwaitingAcknowledgements(message, timeout, strict, includeJobWorkers, ports = connectedPorts) { + return new Promise((resolve, reject) => { let waitingCount = 0; let timer; + let setupComplete = false; + let finished = false; + const failures = []; // Tracks the handlers still awaiting an ack for THIS broadcast. Doubles as an // idempotency guard: a port's handler runs at most once whether it's driven by an ack, // the close listener, or the timeout below. const pending = new Set(); const finish = () => { + if (finished || !setupComplete || waitingCount !== 0) return; + finished = true; if (timer) { clearTimeout(timer); timer = undefined; } - resolve(); + if (strict && failures.length) { + const error = new Error( + `ITC broadcast (type ${message.type}) failed on ${failures.length} worker thread(s): ${failures + .map(({ threadId, message }) => `${threadId}: ${message}`) + .join('; ')}` + ); + error.code = 'ERR_ITC_ACKNOWLEDGEMENT'; + error.failures = failures; + reject(error); + } else resolve(); }; - for (let port of connectedPorts) { + for (let port of ports) { // Job workers run a single isolated task and exit; they don't participate in // schema-change gossip. Including them causes a deadlock: the broadcast waits for // the job worker's ACK while the job worker's event loop is busy waiting for the // same broadcast to complete (re-entrant schema change triggered by the job op). - if (port.isJobWorker) continue; - try { - let requestId = nextId++; - const ackHandler = () => { - if (!pending.delete(ackHandler)) return; // already settled for this port - awaitingResponses.delete(requestId); - if (--waitingCount === 0) { - finish(); + if (port.isJobWorker && !includeJobWorkers) continue; + let referenced = false; + let requestId = nextId++; + const ackHandler = (acknowledgement) => { + if (!pending.delete(ackHandler)) return; // already settled for this port + awaitingResponses.delete(requestId); + if (strict) { + const ackError = acknowledgement?.error; + if (ackError) { + failures.push({ + threadId: port.threadId, + message: ackError.message ?? String(ackError), + }); + } else if (!acknowledgement) { + failures.push({ threadId: port.threadId, message: 'worker disconnected before acknowledging' }); } - if (port !== parentPort && --port.refCount === 0) { - port.unref(); - } - }; - ackHandler.port = port; - pending.add(ackHandler); + } + waitingCount--; + if (referenced && port !== parentPort && --port.refCount === 0) port.unref(); + finish(); + }; + ackHandler.port = port; + pending.add(ackHandler); + waitingCount++; + awaitingResponses.set((message.requestId = requestId), ackHandler); + try { port.ref(); port.refCount = (port.refCount || 0) + 1; - awaitingResponses.set((message.requestId = requestId), ackHandler); + referenced = true; if (!port.hasAckCloseListener) { // just set a single close listener that can clean up all the ack handlers for a port that is closed port.hasAckCloseListener = true; @@ -744,22 +803,25 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) }); } port.postMessage(message); - waitingCount++; } catch (error) { harperLogger.error(`Unable to send message to worker`, error); + ackHandler({ error: { message: error.message ?? String(error) } }); } } - if (waitingCount === 0) return resolve(); + setupComplete = true; + if (waitingCount === 0) return finish(); if (timeout > 0) { timer = setTimeout(() => { timer = undefined; const stuck = []; for (let ackHandler of [...pending]) { stuck.push(ackHandler.port?.threadId); - ackHandler(); // same cleanup path as an ack/close; drives waitingCount to 0 and resolves + ackHandler({ error: { message: `no acknowledgement within ${timeout}ms` } }); } harperLogger.warn( - `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` + strict + ? `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; refusing destructive work` + : `ITC broadcast (type ${message.type}) not acknowledged by worker thread(s) ${stuck.join(', ')} within ${timeout}ms; proceeding best-effort` ); }, timeout); timer.unref?.(); @@ -1157,7 +1219,7 @@ function addPort(port, keepRef, isJobWorker) { } else if (message.type === ACKNOWLEDGEMENT) { let completion = awaitingResponses.get(message.id); if (completion) { - completion(); + completion(message); } } else if (message.type === REMOVE_PORT) { const idx = connectedPorts.findIndex((p) => p.threadId === message.threadId); diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js new file mode 100644 index 0000000000..346745d67b --- /dev/null +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -0,0 +1,123 @@ +'use strict'; + +require('../testUtils'); +const { parentPort } = require('node:worker_threads'); +const { setupTestDBPath } = require('../testUtils'); +const { table, closeLoadedDatabases } = require('#src/resources/databases'); +const { onMessageByType, getProcessInstanceId } = require('#js/server/threads/manageThreads'); + +const MESSAGE_TYPE = 'drop-table-quiescence-test'; +const CONTROL_TYPE = 'drop-table-quiescence-control'; +let TestTable; +let releaseEmbed; + +function report(event, details = {}) { + parentPort.postMessage({ type: MESSAGE_TYPE, event, ...details }); +} + +function runWorkerFixture() { + onMessageByType(CONTROL_TYPE, () => {}); + setupTestDBPath(); + + process.on('unhandledRejection', (error) => { + report('unhandled-rejection', { error: error?.stack ?? String(error) }); + }); + + parentPort + .on('message', async (message) => { + if (message.type !== CONTROL_TYPE) return; + try { + switch (message.command) { + case 'initialize': { + TestTable = table({ + table: message.table, + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name' }, + { name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }, + ], + }); + const embedGate = new Promise((resolve) => { + releaseEmbed = resolve; + }); + TestTable.setEmbedAttribute('vector', async () => { + report('embed-entered'); + await embedGate; + return [1, 2, 3]; + }); + TestTable.sourcedFrom({ + get: async (id) => ({ id, name: 'gated' }), + available: () => true, + }); + + if (typeof TestTable._prepareDrop === 'function') { + const prepareDrop = TestTable._prepareDrop; + TestTable._prepareDrop = async function (options) { + report('prepare-entered'); + await prepareDrop.call(this, options); + let handlesClosed = false; + try { + TestTable.primaryStore.getSync('__drop-close-probe__'); + } catch { + handlesClosed = true; + } + report('prepare-finished', { handlesClosed }); + }; + } + report('ready', { processInstanceId: getProcessInstanceId() }); + break; + } + case 'begin-source-read': + TestTable.get(message.id, {}).then( + () => report('source-read-resolved'), + (error) => report('source-read-rejected', { error: error?.stack ?? String(error) }) + ); + break; + case 'release-embed': + releaseEmbed(); + break; + case 'reject-prepare': + TestTable._prepareDrop = async () => { + report('prepare-entered'); + throw new Error('injected worker quiescence failure'); + }; + report('reject-prepare-armed'); + break; + case 'drop-table': { + const originalDropSync = TestTable.primaryStore.dropSync; + if (message.interruptAfterColumnFamilyDrop) { + TestTable.primaryStore.dropSync = function (...args) { + originalDropSync.apply(this, args); + throw new Error('injected interruption after column-family drop'); + }; + } + try { + await TestTable.dropTable(); + report('drop-result', { outcome: 'resolved' }); + } catch (error) { + report('drop-result', { + outcome: 'rejected', + error: error?.stack ?? String(error), + }); + } finally { + TestTable.primaryStore.dropSync = originalDropSync; + } + break; + } + case 'shutdown': + closeLoadedDatabases(); + report('shutdown-complete'); + parentPort.unref(); + break; + } + } catch (error) { + report('command-error', { command: message.command, error: error?.stack ?? String(error) }); + } + }) + .ref(); + + report('booted'); +} + +if (parentPort) runWorkerFixture(); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js new file mode 100644 index 0000000000..c7ec068997 --- /dev/null +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -0,0 +1,297 @@ +'use strict'; + +require('../testUtils'); +const assert = require('node:assert'); +const path = require('node:path'); +const { setupTestDBPath } = require('../testUtils'); +const { table, database, databases, getDatabases, resetDatabases } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); +const { + startWorker, + onMessageByType, + setMainIsWorker, + getProcessInstanceId, +} = require('#js/server/threads/manageThreads'); + +const WORKER_FIXTURE = path.join(__dirname, 'dropTableQuiescence-worker.js'); +const MESSAGE_TYPE = 'drop-table-quiescence-test'; +const CONTROL_TYPE = 'drop-table-quiescence-control'; + +function defineTable(name, withEmbed = false) { + const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; + if (withEmbed) attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); + return table({ table: name, database: 'test', attributes }); +} + +function startDropWorker(workerIndex, threadCount) { + const queued = new Map(); + const waiting = new Map(); + const errors = []; + let fatalError; + const nextEvent = (event) => { + if (fatalError) return Promise.reject(fatalError); + const prior = queued.get(event); + if (prior?.length) return Promise.resolve(prior.shift()); + return new Promise((resolve, reject) => { + let eventWaiters = waiting.get(event); + if (!eventWaiters) waiting.set(event, (eventWaiters = [])); + eventWaiters.push({ resolve, reject }); + }); + }; + const fail = (error) => { + fatalError = error instanceof Error ? error : new Error(String(error)); + for (const eventWaiters of waiting.values()) { + for (const waiter of eventWaiters.splice(0)) waiter.reject(fatalError); + } + }; + const receive = (message) => { + if (message.type !== MESSAGE_TYPE) return; + if (message.event === 'command-error' || message.event === 'unhandled-rejection') { + errors.push(message); + fail(new Error(message.error)); + return; + } + const eventWaiters = waiting.get(message.event); + if (eventWaiters?.length) eventWaiters.shift().resolve(message); + else { + let eventQueue = queued.get(message.event); + if (!eventQueue) queued.set(message.event, (eventQueue = [])); + eventQueue.push(message); + } + }; + const booted = nextEvent('booted'); + const worker = startWorker(WORKER_FIXTURE, { + name: 'drop-table-quiescence-test', + workerIndex, + threadCount, + autoRestart: false, + onStarted(spawnedWorker) { + spawnedWorker.on('message', receive); + spawnedWorker.once('error', fail); + }, + }); + const send = (command, details = {}) => worker.postMessage({ type: CONTROL_TYPE, command, ...details }); + return { + worker, + booted, + errors, + nextEvent, + send, + async shutdown() { + const shutdown = nextEvent('shutdown-complete').catch(() => undefined); + const exited = new Promise((resolve) => worker.once('exit', resolve)); + send('shutdown'); + await Promise.race([shutdown, exited]); + worker.wasShutdown = true; + await worker.terminate(); + }, + }; +} + +describe('dropTable worker quiescence', function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + + before(() => { + setupTestDBPath(); + setMainIsWorker(true); + onMessageByType(MESSAGE_TYPE, () => {}); + }); + + after(() => { + setMainIsWorker(false); + }); + + it('drains an ordinary staged transaction before touching the column families', async function () { + const Table = defineTable(`DropStagedTxn_${process.pid}_${Date.now()}`); + const context = {}; + let staged; + const stagedPromise = new Promise((resolve) => { + staged = resolve; + }); + let releaseTransaction; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + const transactionPromise = transaction(context, async () => { + await Table.put({ id: 'held', name: 'pending' }, context); + staged(); + await transactionGate; + }); + await stagedPromise; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + let earlyError; + try { + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must suspend on the staged transaction before dropping its primary column family' + ); + } catch (error) { + earlyError = error; + } finally { + if (destructivePhaseStarted) transaction.abort(context); + releaseTransaction(); + } + const [transactionResult, dropResult] = await Promise.allSettled([transactionPromise, dropPromise]); + if (earlyError) throw earlyError; + if (transactionResult.status === 'rejected') throw transactionResult.reason; + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(destructivePhaseStarted, true, 'dropTable() should continue after the transaction settles'); + }); + + it('defers an unquiesced tombstone until the process that could hold stale handles is gone', function () { + const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + const meta = dbisDb.getSync(`${tableName}/`); + meta.dropping = true; + meta.dropGeneration = 'unquiesced-test'; + meta.dropQuiesced = false; + meta.dropProcessInstance = getProcessInstanceId(); + dbisDb.putSync(`${tableName}/`, meta); + + resetDatabases(); + assert.strictEqual(getDatabases().test?.[tableName], undefined, 'an unquiesced table must stay unloaded'); + assert.strictEqual( + dbisDb.getSync(`${tableName}/`)?.dropping, + true, + 'recovery must preserve the tombstone while stale handles can still exist in this process' + ); + + for (const index of Object.values(Table.indices)) index.close(); + Table.primaryStore.close(); + const priorProcessMeta = dbisDb.getSync(`${tableName}/`); + priorProcessMeta.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, priorProcessMeta); + delete databases.test?.[tableName]; + resetDatabases(); + assert.strictEqual(getDatabases().test?.[tableName], undefined); + assert.strictEqual( + database({ database: 'test', table: null }).dbisDb.getSync(`${tableName}/`), + undefined, + 'a tombstone from a prior process should complete on restart' + ); + }); + + it('fails closed when a worker cannot quiesce', async function () { + this.timeout(30000); + const tableName = `DropQuiescenceFailure_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: 'test', table: null }).dbisDb; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName }); + const ready = await remote.nextEvent('ready'); + assert.strictEqual(ready.processInstanceId, getProcessInstanceId()); + remote.send('reject-prepare'); + await remote.nextEvent('reject-prepare-armed'); + + const dropResult = await Table.dropTable().then( + () => ({ outcome: 'resolved' }), + (error) => ({ outcome: 'rejected', error }) + ); + assert.strictEqual(dropResult.outcome, 'rejected', 'a worker NACK must reject the live drop'); + assert.match(dropResult.error.message, /injected worker quiescence failure/); + assert.ok( + rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'no table column family may be dropped after a worker NACK' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true); + assert.strictEqual(tombstone?.dropQuiesced, false); + assert.deepStrictEqual(remote.errors, [], 'the expected NACK must not become an unhandled rejection'); + } finally { + await remote?.shutdown(); + const tombstone = dbisDb.getSync(`${tableName}/`); + if (tombstone?.dropping) { + tombstone.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, tombstone); + resetDatabases(); + getDatabases(); + } + } + }); + + it('quiesces a remote source-cache write and recovers after the column family is already gone', async function () { + this.timeout(30000); + const tableName = `DropWorkerRace_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName, true); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + const rootStore = Table.primaryStore.rootStore; + let origin; + let remote; + + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName }); + const readyWorkers = await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + assert.deepStrictEqual( + readyWorkers.map((message) => message.processInstanceId), + [getProcessInstanceId(), getProcessInstanceId()], + 'every worker must share the main thread process-incarnation marker' + ); + remote.send('begin-source-read', { id: 'remote-source' }); + await remote.nextEvent('embed-entered'); + + const prepareEntered = remote.nextEvent('prepare-entered').then(() => 'prepare'); + const dropResultPromise = origin.nextEvent('drop-result'); + origin.send('drop-table', { interruptAfterColumnFamilyDrop: true }); + const firstOutcome = await Promise.race([prepareEntered, dropResultPromise.then(() => 'drop-settled')]); + assert.strictEqual( + firstOutcome, + 'prepare', + 'the main-thread coordinator must put the remote worker in the pre-drop drain before the worker-originated drop settles' + ); + + remote.send('release-embed'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true, 'the remote worker must close its table handle before ACK'); + await remote.nextEvent('source-read-resolved'); + + const dropResult = await dropResultPromise; + assert.strictEqual( + dropResult.outcome, + 'rejected', + 'the injected interruption must fail the live drop after removing the primary column family' + ); + assert.match(dropResult.error, /injected interruption/); + + assert.ok( + !rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'the table column families must already be absent before recovery begins' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true, 'the failed catalog cleanup must retain the tombstone'); + assert.strictEqual(tombstone?.dropQuiesced, true, 'the tombstone must record completed worker quiescence'); + + resetDatabases(); + const reloaded = getDatabases(); + assert.strictEqual(reloaded.test?.[tableName], undefined, 'recovery must not resurrect the table'); + assert.strictEqual( + database({ database: 'test', table: null }).dbisDb.getSync(`${tableName}/`), + undefined, + 'recovery must remove the tombstone after confirming the column families are absent' + ); + assert.deepStrictEqual( + [...origin.errors, ...remote.errors], + [], + 'no worker rejection should escape the quiescence or recovery path' + ); + } finally { + await Promise.all([origin?.shutdown(), remote?.shutdown()]); + } + }); +}); diff --git a/unitTests/server/threads/workerDataProviders.test.js b/unitTests/server/threads/workerDataProviders.test.js index 9423784fd8..fa3e1a8b59 100644 --- a/unitTests/server/threads/workerDataProviders.test.js +++ b/unitTests/server/threads/workerDataProviders.test.js @@ -13,6 +13,7 @@ describe('registerWorkerDataProvider', () => { it('rejects reserved workerData keys, duplicate names, and non-function providers', () => { assert.throws(() => registerWorkerDataProvider('ticketKeys', () => 1), /already in use/); assert.throws(() => registerWorkerDataProvider('addPorts', () => 1), /already in use/); + assert.throws(() => registerWorkerDataProvider('processInstanceId', () => 'other'), /already in use/); // consumed by threadServer.js, not spread by startWorker — must be reserved all the same assert.throws(() => registerWorkerDataProvider('noServerStart', () => true), /already in use/); assert.throws(() => registerWorkerDataProvider('__proto__', () => ({})), /already in use/); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 7828ffde61..1f00e4ca4a 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -951,6 +951,10 @@ export const ITC_EVENT_TYPES = { OPERATION_EXECUTE_RESPONSE: 'operation_execute_response', } as const; +// Internal schema-event phase used to quiesce every worker before a RocksDB table's column +// families are dropped. It is not an operations-API verb. +export const TABLE_DROP_PREPARE_OPERATION = 'prepare_drop_table'; + /** Supported thread types */ export const THREAD_TYPES = { HTTP: 'http', diff --git a/utility/signalling.ts b/utility/signalling.ts index aae59d81cd..a748af7630 100644 --- a/utility/signalling.ts +++ b/utility/signalling.ts @@ -4,7 +4,7 @@ import * as hdbTerms from './hdbTerms.ts'; import hdbLogger from '../utility/logging/harper_logger.ts'; import ITCEventObject from '../server/itc/utility/ITCEventObject.js'; let serverItcHandlers; -import { sendItcEvent } from '../server/threads/itc.js'; +import { sendItcEvent, sendItcEventStrict } from '../server/threads/itc.js'; // Await BOTH the local handler and the cross-worker broadcast. The local handler is what // rebuilds THIS thread's cache; firing it un-awaited let the originating worker return success @@ -23,6 +23,15 @@ export async function signalSchemaChange(message: any) { } } +/** + * Quiesce a table on every connected thread before its RocksDB handles are dropped. Unlike normal + * post-change gossip, this rejects unless every handler succeeds; the caller must fail closed. + */ +export function signalTableDropPreparation(message: any) { + const event = new ITCEventObject(hdbTerms.ITC_EVENT_TYPES.SCHEMA, message); + return sendItcEventStrict(event); +} + /** * Notify local listeners that JS resources have just been registered (resources.js loaded). This is * deliberately local-only — no ITC broadcast — because every worker registers its own JS resources, From b2f987045a549aae316400f00c54fba73e7e8946 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 12:47:34 -0600 Subject: [PATCH 02/40] fix: harden table drop quiescence retries Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 38 +++++- resources/Table.ts | 61 +++++++-- resources/databases.ts | 26 ++-- server/threads/itc.js | 5 +- server/threads/manageThreads.js | 7 +- .../resources/Resource-get-context.test.js | 24 +--- .../resources/dropTableQuiescence-worker.js | 86 +++++++++--- .../resources/dropTableQuiescence.test.js | 123 +++++++++++++++++- utility/hdbTerms.ts | 2 - 9 files changed, 303 insertions(+), 69 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 09477d3acd..0b1b808ea2 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -162,6 +162,16 @@ export function getPendingWriteResolutions(stores: Iterable): Promise } return resolutions; } + +export function getPendingReadResolutions(rootStore: any): Promise[] { + const resolutions: Promise[] = []; + for (const transaction of trackedTxns) { + if ((transaction.db as any)?.rootStore !== rootStore) continue; + const resolution = transaction.getPendingReadResolution(); + if (resolution) resolutions.push(resolution); + } + return resolutions; +} // Once per process: committing under open read iterators forces a write replay, so the warning is // about the caller's pattern, not the individual commit. let replayedWritesWarned = false; @@ -350,6 +360,9 @@ export class DatabaseTransaction implements Transaction { #context: Context; #pendingWriteResolution?: Promise; #resolvePendingWrites?: () => void; + #pendingReadResolution?: Promise; + #resolvePendingReads?: () => void; + #trackedForDropDrain = false; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) declare writesByKey?: Map>; @@ -513,6 +526,7 @@ export class DatabaseTransaction implements Transaction { // discards nothing — the replay re-staged the writes AND their audit/txn-log entries // into its own transaction; this handle's never-committed log batch dies with it. const transaction = this.detachOwnedTransaction(); + this.finishPendingReads(); try { transaction?.abort(); } catch (error) { @@ -539,6 +553,7 @@ export class DatabaseTransaction implements Transaction { } catch (error) { harperLogger.debug?.('releasing timed-out read transaction', error); } + this.finishPendingReads(); this.completeDeferredContextRelease(); } @@ -581,7 +596,10 @@ export class DatabaseTransaction implements Transaction { error.code = 'ERR_TABLE_DROPPING'; throw error; } - if (operation.store?.rootStore instanceof RocksDatabase) activeWriteTransactions.add(this); + if (!this.#trackedForDropDrain && operation.store?.rootStore instanceof RocksDatabase) { + this.#trackedForDropDrain = true; + activeWriteTransactions.add(this); + } if (operation.key === undefined) return; let writesForStore = (this.writesByKey ??= new Map()).get(operation.store); if (!writesForStore) this.writesByKey.set(operation.store, (writesForStore = new Map())); @@ -609,13 +627,28 @@ export class DatabaseTransaction implements Transaction { return this.#pendingWriteResolution; } + getPendingReadResolution(): Promise | undefined { + if (!this.transaction || !(this.readTxnsUsed > 0)) return; + this.#pendingReadResolution ??= new Promise((resolve) => { + this.#resolvePendingReads = resolve; + }); + return this.#pendingReadResolution; + } + private finishPendingWrites(): void { activeWriteTransactions.delete(this); + this.#trackedForDropDrain = false; this.#resolvePendingWrites?.(); this.#pendingWriteResolution = undefined; this.#resolvePendingWrites = undefined; } + private finishPendingReads(): void { + this.#resolvePendingReads?.(); + this.#pendingReadResolution = undefined; + this.#resolvePendingReads = undefined; + } + /** * Drop this transaction's back-reference from its context once completed (commit or abort), * so a long-lived context (e.g. an MQTT subscription context held open for the life of a @@ -938,6 +971,7 @@ export class DatabaseTransaction implements Transaction { } else { // no more reads need to be performed, just commit/abort based if there are any writes this.detachOwnedTransaction(); // any further operations operate immediately + this.finishPendingReads(); if (transaction) { this.writes = this.writes.filter((write) => write); // filter out removed entries if (this.writes.length > 0) { @@ -1223,6 +1257,7 @@ export class DatabaseTransaction implements Transaction { // loop cannot spin on a nulled handle. Not to avoid a double abort: rocksdb-js tolerates // abort-after-abort, and it is abort-after-COMMIT that throws. const detached = txn.detachOwnedTransaction(); + txn.finishPendingReads(); const committingTransaction = txn === this ? headTransaction : detached; try { committingTransaction?.abort(); @@ -1305,6 +1340,7 @@ export class DatabaseTransaction implements Transaction { } throw error; } finally { + this.finishPendingReads(); this.finishPendingWrites(); } this.detachOwnedTransaction(); diff --git a/resources/Table.ts b/resources/Table.ts index 56d30cdd2f..826f538533 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -39,6 +39,7 @@ import { priorStagedWrite, isReleasedTransaction, TRANSACTION_STATE, + getPendingReadResolutions, getPendingWriteResolutions, } from './DatabaseTransaction.ts'; import * as envMngr from '../utility/environment/environmentManager.ts'; @@ -412,11 +413,17 @@ export function makeTable(options) { const tableStores = () => [...Object.values(indices), primaryStore].filter(Boolean); const markTableDropping = () => { droppingTable = true; - for (const store of tableStores()) (store as any).dropping = true; + if (isRocksDB) { + for (const store of tableStores()) (store as any).dropping = true; + } delete databases[databaseName]?.[tableName]; }; - const drainTableWrites = async () => { - const pending = new Set>([...pendingSourceCommits, ...getPendingWriteResolutions(tableStores())]); + const drainTableOperations = async () => { + const pending = new Set>([ + ...pendingSourceCommits, + ...getPendingWriteResolutions(tableStores()), + ...getPendingReadResolutions(primaryStore.rootStore), + ]); if (!pending.size) return; let timer: NodeJS.Timeout; const timedOut = Symbol('timedOut'); @@ -429,7 +436,7 @@ export function makeTable(options) { clearTimeout(timer); if (result === timedOut) { throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight write(s) on ${tableName} to settle; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` ); } }; @@ -1395,12 +1402,21 @@ export function makeTable(options) { static async _prepareDrop({ closeStores = true } = {}) { markTableDropping(); - dropPreparation ??= drainTableWrites(); + dropPreparation ??= drainTableOperations().catch((error) => { + dropPreparation = undefined; + throw error; + }); await dropPreparation; if (closeStores && !coordinatingDrop) closeTableStores(); } static async dropTable() { + if (storesClosed) { + throw new ServerError( + `Cannot retry dropping ${databaseName}.${tableName} through closed handles; restart Harper to resume the durable drop`, + 503 + ); + } const rootStore = primaryStore.rootStore; const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; let dropGeneration: string | undefined; @@ -1425,14 +1441,24 @@ export function makeTable(options) { if (sharedRocksStore) { await prepareTableDrop(rootStore.path, tableName, dropGeneration, TableResource); locallyQuiesced = true; - await signalling.signalTableDropPreparation({ - originator: process.pid, - operation: TABLE_DROP_PREPARE_OPERATION, - schema: databaseName, - table: tableName, - path: rootStore.path, - dropGeneration, - }); + try { + await signalling.signalTableDropPreparation({ + originator: process.pid, + operation: TABLE_DROP_PREPARE_OPERATION, + schema: databaseName, + table: tableName, + path: rootStore.path, + dropGeneration, + }); + } catch (error) { + const quiescenceError: any = new ServerError( + `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable but its storage was not dropped. Restart Harper before retrying. ${error.message}`, + 503 + ); + quiescenceError.code = error.code; + quiescenceError.cause = error; + throw quiescenceError; + } } else { await TableResource._prepareDrop({ closeStores: false }); locallyQuiesced = true; @@ -1443,6 +1469,8 @@ export function makeTable(options) { } if (databaseName === databasePath) { + // Keep the tombstone until every column family is gone; reversing this order can orphan + // an undiscoverable "ghost" family when a drop fails partway through. const removeTombstonedCatalog = () => { const currentPrimary = (dbisDb as any).getSync(TableResource.tableName + '/'); if (!currentPrimary?.dropping || currentPrimary.dropGeneration !== dropGeneration) return false; @@ -1453,6 +1481,8 @@ export function makeTable(options) { return true; }; if (rootStore instanceof RocksDatabase) { + // Concurrent creates take this synchronous spin lock too. Nothing in the locked section + // may await, or this worker can deadlock the process while the lock owner needs its event loop. while (!rootStore.tryLock('update-attributes')) {} let removed = false; try { @@ -5590,6 +5620,11 @@ export function makeTable(options) { } } function txnForContext(context: Context) { + if (isRocksDB && droppingTable) { + const error: any = new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 409); + error.code = 'ERR_TABLE_DROPPING'; + throw error; + } let transaction = context?.transaction; if (isReleasedTransaction(transaction)) transaction = undefined; if (transaction) { diff --git a/resources/databases.ts b/resources/databases.ts index 8ad8591ee2..6e55f4e5f4 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -379,6 +379,7 @@ const MAX_INTERRUPTED_DROP_ATTEMPTS = 3; // resolved, so a resolution can only ever identify the outer path+table key, // never the specific spent generation to target. const interruptedDropAttempts = new Map>(); +const incompleteTableDropPreparations = new Map>(); const interruptedDropTableKey = (storePath: string, tableName: string) => `${storePath}\0${tableName}`; function getInterruptedDropAttempts(storePath: string, tableName: string, generation?: string): number { return interruptedDropAttempts.get(interruptedDropTableKey(storePath, tableName))?.get(generation ?? 'legacy') ?? 0; @@ -2398,27 +2399,36 @@ function canCompleteInterruptedDrop(primaryMeta): boolean { ); } -/** - * Mark and drain every worker-local class backed by this physical table. The drop origin keeps - * its own handles so it can perform the destructive phase; remote workers close theirs after the - * drain so no stale handle can later contaminate RocksDB's shared write path. - */ export async function prepareTableDrop( storePath: string, tableName: string, dropGeneration: string | undefined, preserveTable?: any ): Promise { - const matchingTables = new Set(); + const preparationKey = `${storePath}\0${tableName}\0${dropGeneration ?? 'legacy'}`; + let matchingTables = incompleteTableDropPreparations.get(preparationKey); + if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); + if (preserveTable) matchingTables.add(preserveTable); for (const databaseName of Object.getOwnPropertyNames(databases)) { const databaseTables = databases[databaseName]; const Table = databaseTables?.[tableName]; if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); - if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) continue; + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { + throw new ClientError( + `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, + 409 + ); + } matchingTables.add(Table); } - await Promise.all([...matchingTables].map((Table) => Table._prepareDrop({ closeStores: Table !== preserveTable }))); + await Promise.all( + [...matchingTables].map(async (Table) => { + await Table._prepareDrop({ closeStores: Table !== preserveTable }); + matchingTables.delete(Table); + }) + ); + if (!matchingTables.size) incompleteTableDropPreparations.delete(preparationKey); } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/server/threads/itc.js b/server/threads/itc.js index 5ad4bb0b77..1fab18f9c4 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -20,6 +20,7 @@ module.exports = { UserEventMsg, }; let serverItcHandlers; +const STRICT_COORDINATOR_ACK_TIMEOUT_MS = 60000; onMessageFromWorkers(async (event, sender) => { const requestId = event?.requestId; let handlerError; @@ -68,7 +69,9 @@ function sendItcEventStrict(event) { stampOriginator(event); if (isMainThread) return broadcastWithStrictAcknowledgement(event); event.relayStrictToWorkers = true; - return sendToThreadWithStrictAcknowledgement(0, event); + // The main thread first prepares itself and then runs its own 30-second worker broadcast. + // The worker-to-main deadline must cover both phases rather than racing the nested deadline. + return sendToThreadWithStrictAcknowledgement(0, event, STRICT_COORDINATOR_ACK_TIMEOUT_MS); } function stampOriginator(event) { diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 64c07fdf19..c8f5292590 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -758,10 +758,9 @@ function broadcastAwaitingAcknowledgements(message, timeout, strict, includeJobW } else resolve(); }; for (let port of ports) { - // Job workers run a single isolated task and exit; they don't participate in - // schema-change gossip. Including them causes a deadlock: the broadcast waits for - // the job worker's ACK while the job worker's event loop is busy waiting for the - // same broadcast to complete (re-entrant schema change triggered by the job op). + // Ordinary post-change gossip excludes transient job workers. Strict pre-change barriers + // include them because a job can hold the same native handles; the main-thread relay keeps + // a job-originated async schema operation re-entrant while it awaits its own ACK. if (port.isJobWorker && !includeJobWorkers) continue; let referenced = false; let requestId = nextId++; diff --git a/unitTests/resources/Resource-get-context.test.js b/unitTests/resources/Resource-get-context.test.js index c89ad1ab56..953e60f9ca 100644 --- a/unitTests/resources/Resource-get-context.test.js +++ b/unitTests/resources/Resource-get-context.test.js @@ -268,7 +268,7 @@ describe('dropTable waits for in-flight source-populated cache writes (harper#13 assert.strictEqual(dropResolved, true, 'dropTable() should resolve once the pending write has landed'); }); - it('does not start a new cache write once dropTable() has begun (late admission during the drain)', async function () { + it('does not admit a new cache read once dropTable() has begun', async function () { setupTestDBPath(); setMainIsWorker(true); @@ -306,23 +306,13 @@ describe('dropTable waits for in-flight source-populated cache writes (harper#13 const dropPromise = TestTable.dropTable(); - // A get() admitted while the drop is draining must still return fresh source data... - const lateResult = await TestTable.get('late', {}); - assert.ok(lateSourceCalled, 'the source should still be consulted for a late-admitted read'); - assert.strictEqual(lateResult.name, 'value'); - // ...but must not have started a new cache write into a table that's being dropped. The - // get() call itself resolves before its own cache write would land (that's the whole - // bug this file covers), so a correctly-blocked write and one that merely hasn't landed - // YET look identical immediately after the await above. Give a generous, bounded window - // for an (incorrectly) unblocked local write to land - the drop itself is still parked on - // the gated first write, so this checks storage well before any column family is - // touched, not a race against the drop. - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.strictEqual( - TestTable.primaryStore.getSync('late'), - undefined, - 'a get() admitted after dropTable() started must not cache its result' + assert.throws( + () => TestTable.get('late', {}), + (error) => error?.code === 'ERR_TABLE_DROPPING', + 'a read arriving after the drain boundary must fail before opening a transaction' ); + assert.strictEqual(lateSourceCalled, false, 'the source must not be consulted after drop preparation begins'); + assert.strictEqual(TestTable.primaryStore.getSync('late'), undefined, 'a rejected read must not cache a result'); releaseFirst(); await firstGetPromise; diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js index 346745d67b..67aa2cb30b 100644 --- a/unitTests/resources/dropTableQuiescence-worker.js +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -4,12 +4,15 @@ require('../testUtils'); const { parentPort } = require('node:worker_threads'); const { setupTestDBPath } = require('../testUtils'); const { table, closeLoadedDatabases } = require('#src/resources/databases'); +const { transaction } = require('#src/resources/transaction'); const { onMessageByType, getProcessInstanceId } = require('#js/server/threads/manageThreads'); const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; let TestTable; let releaseEmbed; +let releaseRead; +let releaseTransaction; function report(event, details = {}) { parentPort.postMessage({ type: MESSAGE_TYPE, event, ...details }); @@ -29,27 +32,29 @@ function runWorkerFixture() { try { switch (message.command) { case 'initialize': { + const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; + if (message.withEmbed) { + attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); + } TestTable = table({ table: message.table, database: 'test', - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'name' }, - { name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }, - ], - }); - const embedGate = new Promise((resolve) => { - releaseEmbed = resolve; - }); - TestTable.setEmbedAttribute('vector', async () => { - report('embed-entered'); - await embedGate; - return [1, 2, 3]; - }); - TestTable.sourcedFrom({ - get: async (id) => ({ id, name: 'gated' }), - available: () => true, + attributes, }); + if (message.withEmbed) { + const embedGate = new Promise((resolve) => { + releaseEmbed = resolve; + }); + TestTable.setEmbedAttribute('vector', async () => { + report('embed-entered'); + await embedGate; + return [1, 2, 3]; + }); + TestTable.sourcedFrom({ + get: async (id) => ({ id, name: 'gated' }), + available: () => true, + }); + } if (typeof TestTable._prepareDrop === 'function') { const prepareDrop = TestTable._prepareDrop; @@ -77,6 +82,53 @@ function runWorkerFixture() { case 'release-embed': releaseEmbed(); break; + case 'begin-transaction': { + const context = {}; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + transaction(context, async () => { + await TestTable.put({ id: message.id, name: 'pending' }, context); + report('transaction-staged'); + await transactionGate; + }).then( + () => report('transaction-resolved'), + (error) => report('transaction-rejected', { error: error?.stack ?? String(error) }) + ); + break; + } + case 'release-transaction': + releaseTransaction(); + break; + case 'begin-read': { + const context = {}; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + transaction(context, async (dbTransaction) => { + TestTable._readTxnForContext(context); + const readTransaction = dbTransaction.useReadTxn(); + const iterator = TestTable.primaryStore + .getRange({ start: false, transaction: readTransaction }) + [Symbol.iterator](); + iterator.next(); + report('read-open'); + await readGate; + try { + iterator.next(); + } finally { + iterator.return?.(); + dbTransaction.doneReadTxn(); + } + }).then( + () => report('read-resolved'), + (error) => report('read-rejected', { error: error?.stack ?? String(error) }) + ); + break; + } + case 'release-read': + releaseRead(); + break; case 'reject-prepare': TestTable._prepareDrop = async () => { report('prepare-entered'); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index c7ec068997..af0d987cc0 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -6,6 +6,7 @@ const path = require('node:path'); const { setupTestDBPath } = require('../testUtils'); const { table, database, databases, getDatabases, resetDatabases } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); +const { THREAD_TYPES } = require('#src/utility/hdbTerms'); const { startWorker, onMessageByType, @@ -23,7 +24,7 @@ function defineTable(name, withEmbed = false) { return table({ table: name, database: 'test', attributes }); } -function startDropWorker(workerIndex, threadCount) { +function startDropWorker(workerIndex, threadCount, name = 'drop-table-quiescence-test') { const queued = new Map(); const waiting = new Map(); const errors = []; @@ -46,7 +47,12 @@ function startDropWorker(workerIndex, threadCount) { }; const receive = (message) => { if (message.type !== MESSAGE_TYPE) return; - if (message.event === 'command-error' || message.event === 'unhandled-rejection') { + if ( + message.event === 'command-error' || + message.event === 'unhandled-rejection' || + message.event === 'read-rejected' || + message.event === 'transaction-rejected' + ) { errors.push(message); fail(new Error(message.error)); return; @@ -61,7 +67,7 @@ function startDropWorker(workerIndex, threadCount) { }; const booted = nextEvent('booted'); const worker = startWorker(WORKER_FIXTURE, { - name: 'drop-table-quiescence-test', + name, workerIndex, threadCount, autoRestart: false, @@ -126,6 +132,7 @@ describe('dropTable worker quiescence', function () { return originalDropSync.apply(this, args); }; const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); let earlyError; try { assert.strictEqual( @@ -180,6 +187,34 @@ describe('dropTable worker quiescence', function () { ); }); + it('retries preparation on a class already removed from the live schema', async function () { + const tableName = `DropPreparationRetry_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const rootStore = Table.primaryStore.rootStore; + const originalPrepareDrop = Table._prepareDrop; + let firstPreparation = true; + Table._prepareDrop = async function (options) { + await originalPrepareDrop.call(this, options); + if (firstPreparation) { + firstPreparation = false; + throw new Error('injected local preparation failure'); + } + }; + + await assert.rejects(() => Table.dropTable(), /injected local preparation failure/); + assert.strictEqual(databases.test?.[tableName], undefined); + assert.ok(rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); + assert.throws( + () => Table._readTxnForContext({}), + (error) => error?.code === 'ERR_TABLE_DROPPING', + 'a stale table-class reference must not admit a new read after preparation starts' + ); + + Table._prepareDrop = originalPrepareDrop; + await Table.dropTable(); + assert.ok(!rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); + }); + it('fails closed when a worker cannot quiesce', async function () { this.timeout(30000); const tableName = `DropQuiescenceFailure_${process.pid}_${Date.now()}`; @@ -222,6 +257,82 @@ describe('dropTable worker quiescence', function () { } }); + it('drains a remote staged transaction before a worker-originated drop', async function () { + this.timeout(30000); + const tableName = `DropRemoteTransaction_${process.pid}_${Date.now()}`; + defineTable(tableName); + let origin; + let remote; + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName }); + await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + + remote.send('begin-transaction', { id: 'remote-staged' }); + await remote.nextEvent('transaction-staged'); + const dropResultPromise = origin.nextEvent('drop-result'); + let dropSettled = false; + dropResultPromise.then(() => { + dropSettled = true; + }); + origin.send('drop-table'); + await remote.nextEvent('prepare-entered'); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(dropSettled, false, 'the drop must wait for the remote transaction'); + + remote.send('release-transaction'); + await remote.nextEvent('transaction-resolved'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true); + const dropResult = await dropResultPromise; + assert.strictEqual(dropResult.outcome, 'resolved'); + assert.deepStrictEqual([...origin.errors, ...remote.errors], []); + } finally { + await Promise.all([origin?.shutdown(), remote?.shutdown()]); + } + }); + + it('drains a remote read iterator before closing its handles', async function () { + this.timeout(30000); + const tableName = `DropRemoteRead_${process.pid}_${Date.now()}`; + defineTable(tableName); + let origin; + let remote; + try { + remote = startDropWorker(1, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); + await Promise.all([remote.booted, origin.booted]); + remote.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName }); + await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); + + remote.send('begin-read'); + await remote.nextEvent('read-open'); + const dropResultPromise = origin.nextEvent('drop-result'); + let dropSettled = false; + dropResultPromise.then(() => { + dropSettled = true; + }); + origin.send('drop-table'); + await remote.nextEvent('prepare-entered'); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(dropSettled, false, 'the drop must wait for the remote iterator'); + + remote.send('release-read'); + await remote.nextEvent('read-resolved'); + const prepared = await remote.nextEvent('prepare-finished'); + assert.strictEqual(prepared.handlesClosed, true); + const dropResult = await dropResultPromise; + assert.strictEqual(dropResult.outcome, 'resolved'); + assert.deepStrictEqual([...origin.errors, ...remote.errors], []); + } finally { + await Promise.all([origin?.shutdown(), remote?.shutdown()]); + } + }); + it('quiesces a remote source-cache write and recovers after the column family is already gone', async function () { this.timeout(30000); const tableName = `DropWorkerRace_${process.pid}_${Date.now()}`; @@ -233,10 +344,10 @@ describe('dropTable worker quiescence', function () { try { remote = startDropWorker(1, 3); - origin = startDropWorker(2, 3); + origin = startDropWorker(2, 3, THREAD_TYPES.JOB); await Promise.all([remote.booted, origin.booted]); - remote.send('initialize', { table: tableName }); - origin.send('initialize', { table: tableName }); + remote.send('initialize', { table: tableName, withEmbed: true }); + origin.send('initialize', { table: tableName, withEmbed: true }); const readyWorkers = await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); assert.deepStrictEqual( readyWorkers.map((message) => message.processInstanceId), diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 1f00e4ca4a..7b164aeaa9 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -951,8 +951,6 @@ export const ITC_EVENT_TYPES = { OPERATION_EXECUTE_RESPONSE: 'operation_execute_response', } as const; -// Internal schema-event phase used to quiesce every worker before a RocksDB table's column -// families are dropped. It is not an operations-API verb. export const TABLE_DROP_PREPARE_OPERATION = 'prepare_drop_table'; /** Supported thread types */ From b6b057c473372c5dfdc4ca949d3cef86aedc8ee8 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 12:56:06 -0600 Subject: [PATCH 03/40] test: tighten table drop drain scope Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 20 +++- resources/Table.ts | 8 +- .../resources/dropTableQuiescence.test.js | 108 +++++++++++++++--- 3 files changed, 119 insertions(+), 17 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 0b1b808ea2..3b42ee0e88 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -163,10 +163,11 @@ export function getPendingWriteResolutions(stores: Iterable): Promise return resolutions; } -export function getPendingReadResolutions(rootStore: any): Promise[] { +export function getPendingReadResolutions(stores: Iterable): Promise[] { + const targetStores = new Set(stores); const resolutions: Promise[] = []; for (const transaction of trackedTxns) { - if ((transaction.db as any)?.rootStore !== rootStore) continue; + if (!transaction.usesAnyStore(targetStores)) continue; const resolution = transaction.getPendingReadResolution(); if (resolution) resolutions.push(resolution); } @@ -362,6 +363,8 @@ export class DatabaseTransaction implements Transaction { #resolvePendingWrites?: () => void; #pendingReadResolution?: Promise; #resolvePendingReads?: () => void; + // `db` identifies the first table; allocate only when one native transaction spans more tables. + #additionalStores?: Set; #trackedForDropDrain = false; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) @@ -635,6 +638,18 @@ export class DatabaseTransaction implements Transaction { return this.#pendingReadResolution; } + trackStore(store: any): void { + if (this.db !== store) (this.#additionalStores ??= new Set()).add(store); + } + + usesAnyStore(stores: Set): boolean { + if (stores.has(this.db)) return true; + for (const store of this.#additionalStores ?? []) { + if (stores.has(store)) return true; + } + return false; + } + private finishPendingWrites(): void { activeWriteTransactions.delete(this); this.#trackedForDropDrain = false; @@ -647,6 +662,7 @@ export class DatabaseTransaction implements Transaction { this.#resolvePendingReads?.(); this.#pendingReadResolution = undefined; this.#resolvePendingReads = undefined; + this.#additionalStores = undefined; } /** diff --git a/resources/Table.ts b/resources/Table.ts index 826f538533..5f2461255d 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -422,7 +422,7 @@ export function makeTable(options) { const pending = new Set>([ ...pendingSourceCommits, ...getPendingWriteResolutions(tableStores()), - ...getPendingReadResolutions(primaryStore.rootStore), + ...getPendingReadResolutions(tableStores()), ]); if (!pending.size) return; let timer: NodeJS.Timeout; @@ -5636,7 +5636,11 @@ export function makeTable(options) { } do { // See if this is a transaction for our database and if so, use it - if (transaction.db?.path === primaryStore.path) return transaction; + if (transaction.db?.path === primaryStore.path) { + // Tracked reads must join here so the drop drain records every table store they borrow. + transaction.trackStore(primaryStore); + return transaction; + } // try the next one: const nextTxn = transaction.next; if (!nextTxn) { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index af0d987cc0..376fdea430 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -4,6 +4,7 @@ require('../testUtils'); const assert = require('node:assert'); const path = require('node:path'); const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); const { table, database, databases, getDatabases, resetDatabases } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); const { THREAD_TYPES } = require('#src/utility/hdbTerms'); @@ -153,6 +154,62 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(destructivePhaseStarted, true, 'dropTable() should continue after the transaction settles'); }); + it('does not wait for a read iterator on another table in the same database', async function () { + const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); + const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); + const context = {}; + let readOpened; + const readOpenedPromise = new Promise((resolve) => { + readOpened = resolve; + }); + let releaseRead; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + let readFinished = false; + const readPromise = transaction(context, async (dbTransaction) => { + otherTable._readTxnForContext(context); + const readTransaction = dbTransaction.useReadTxn(); + const iterator = otherTable.primaryStore + .getRange({ start: false, transaction: readTransaction }) + [Symbol.iterator](); + iterator.next(); + readOpened(); + await readGate; + try { + iterator.next(); + } finally { + iterator.return?.(); + dbTransaction.doneReadTxn(); + readFinished = true; + } + }); + await readOpenedPromise; + + const originalDropSync = droppedTable.primaryStore.dropSync; + let destructivePhaseStarted = false; + droppedTable.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = droppedTable.dropTable(); + let earlyError; + try { + await waitFor(() => destructivePhaseStarted, { + message: 'an unrelated table reader must not widen the drop drain to the whole database', + }); + assert.strictEqual(readFinished, false, 'the unrelated iterator must still be open when the drop proceeds'); + } catch (error) { + earlyError = error; + } finally { + releaseRead(); + } + const [readResult, dropResult] = await Promise.allSettled([readPromise, dropPromise]); + if (earlyError) throw earlyError; + if (readResult.status === 'rejected') throw readResult.reason; + if (dropResult.status === 'rejected') throw dropResult.reason; + }); + it('defers an unquiesced tombstone until the process that could hold stale handles is gone', function () { const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); @@ -187,21 +244,28 @@ describe('dropTable worker quiescence', function () { ); }); - it('retries preparation on a class already removed from the live schema', async function () { + it('retries a timed-out preparation on a class already removed from the live schema', async function () { + this.timeout(25000); const tableName = `DropPreparationRetry_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); const rootStore = Table.primaryStore.rootStore; - const originalPrepareDrop = Table._prepareDrop; - let firstPreparation = true; - Table._prepareDrop = async function (options) { - await originalPrepareDrop.call(this, options); - if (firstPreparation) { - firstPreparation = false; - throw new Error('injected local preparation failure'); - } - }; + const context = {}; + let staged; + const stagedPromise = new Promise((resolve) => { + staged = resolve; + }); + let releaseTransaction; + const transactionGate = new Promise((resolve) => { + releaseTransaction = resolve; + }); + const transactionPromise = transaction(context, async () => { + await Table.put({ id: 'held-for-retry', name: 'pending' }, context); + staged(); + await transactionGate; + }); + await stagedPromise; - await assert.rejects(() => Table.dropTable(), /injected local preparation failure/); + await assert.rejects(() => Table.dropTable(), /timed out after 10000ms/); assert.strictEqual(databases.test?.[tableName], undefined); assert.ok(rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); assert.throws( @@ -210,8 +274,26 @@ describe('dropTable worker quiescence', function () { 'a stale table-class reference must not admit a new read after preparation starts' ); - Table._prepareDrop = originalPrepareDrop; - await Table.dropTable(); + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const retryPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'the retry must start a fresh drain'); + } catch (error) { + earlyError = error; + } finally { + releaseTransaction(); + } + const [transactionResult, retryResult] = await Promise.allSettled([transactionPromise, retryPromise]); + if (earlyError) throw earlyError; + if (transactionResult.status === 'rejected') throw transactionResult.reason; + if (retryResult.status === 'rejected') throw retryResult.reason; assert.ok(!rootStore.columns.some((column) => column.startsWith(`${tableName}/`))); }); From 25b3b3c4853d8518dd1f4c15c65aa628e263f68e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 13:28:52 -0600 Subject: [PATCH 04/40] fix: close table drop quiescence gaps Track transaction-less table scans and clears so drop waits for their native handles. Gate strict worker broadcasts on ITC readiness and reject malformed events instead of acknowledging them. Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 7 +- resources/Table.ts | 388 ++++++++++-------- server/threads/itc.js | 6 +- server/threads/manageThreads.js | 16 +- .../resources/dropTableQuiescence.test.js | 86 +++- .../resources/dropTableUnready-worker.js | 6 + utility/hdbTerms.ts | 1 + 7 files changed, 343 insertions(+), 167 deletions(-) create mode 100644 unitTests/resources/dropTableUnready-worker.js diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 3b42ee0e88..bde8f72a2d 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -365,6 +365,7 @@ export class DatabaseTransaction implements Transaction { #resolvePendingReads?: () => void; // `db` identifies the first table; allocate only when one native transaction spans more tables. #additionalStores?: Set; + #lastTrackedStore?: any; #trackedForDropDrain = false; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) @@ -639,7 +640,10 @@ export class DatabaseTransaction implements Transaction { } trackStore(store: any): void { - if (this.db !== store) (this.#additionalStores ??= new Set()).add(store); + if (this.db !== store && this.#lastTrackedStore !== store) { + this.#lastTrackedStore = store; + (this.#additionalStores ??= new Set()).add(store); + } } usesAnyStore(stores: Set): boolean { @@ -663,6 +667,7 @@ export class DatabaseTransaction implements Transaction { this.#pendingReadResolution = undefined; this.#resolvePendingReads = undefined; this.#additionalStores = undefined; + this.#lastTrackedStore = undefined; } /** diff --git a/resources/Table.ts b/resources/Table.ts index 5f2461255d..d3e9bfb8ff 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -406,13 +406,40 @@ export function makeTable(options) { // in-flight commit promises here so dropTable() can drain them first, and stop admitting // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); + const pendingTableOperations = new Set>(); let droppingTable = false; let coordinatingDrop = false; let storesClosed = false; let dropPreparation: Promise | undefined; const tableStores = () => [...Object.values(indices), primaryStore].filter(Boolean); + const tableDroppingError = () => { + const error: any = new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 409); + error.code = 'ERR_TABLE_DROPPING'; + return error; + }; + const beginTableOperation = () => { + if (!isRocksDB) return () => {}; + if (droppingTable) throw tableDroppingError(); + let resolve: () => void; + const completion = new Promise((scanResolve) => { + resolve = scanResolve; + }); + pendingTableOperations.add(completion); + let active = true; + return () => { + if (!active) return; + active = false; + pendingTableOperations.delete(completion); + resolve(); + }; + }; + const stopBackgroundScans = () => { + if (cleanupTimer) clearTimeout(cleanupTimer); + if (recordExpirationInterval) clearInterval(recordExpirationInterval); + }; const markTableDropping = () => { droppingTable = true; + stopBackgroundScans(); if (isRocksDB) { for (const store of tableStores()) (store as any).dropping = true; } @@ -421,6 +448,7 @@ export function makeTable(options) { const drainTableOperations = async () => { const pending = new Set>([ ...pendingSourceCommits, + ...pendingTableOperations, ...getPendingWriteResolutions(tableStores()), ...getPendingReadResolutions(tableStores()), ]); @@ -463,6 +491,7 @@ export function makeTable(options) { let cleanupPriority = 0; let lastCleanupInterval: number; let cleanupTimer: NodeJS.Timeout; + let recordExpirationInterval: NodeJS.Timeout; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; // set on the first expiring write so the unscheduled-expiration warning is evaluated at most once per table @@ -1167,6 +1196,7 @@ export function makeTable(options) { const asyncIdExpansionThreshold = type === 'Int' ? 0x200 : 0x100000; if (nextId + asyncIdExpansionThreshold >= idIncrementer.maxSafeId) { const updateEnd = (inTxn) => { + if (droppingTable) return; // we update the end of the allocation range after verifying we don't have any conflicting ids in front of us idIncrementer.maxSafeId = nextId + (type === 'Int' ? 0x3ff : 0x3fffff); let idAfter = (type === 'Int' ? Math.pow(2, 31) : Math.pow(2, 49)) - 1; @@ -3986,6 +4016,7 @@ export function makeTable(options) { // #section: pub-sub async subscribe(request: SubscriptionRequest): Promise> { + if (isRocksDB && droppingTable) throw tableDroppingError(); if (!request) request = {} as any; const loadAsInstance = (this.constructor as any).loadAsInstance; if (loadAsInstance === false && (request as any).checkPermission) { @@ -4052,6 +4083,7 @@ export function makeTable(options) { return evaluateFilter(rowFilter, event.value, 'rowFilter'); } : null; + if (isRocksDB && droppingTable) throw tableDroppingError(); const subscription = addSubscription( TableResource, thisId, @@ -4116,6 +4148,7 @@ export function makeTable(options) { // in subscription.queue. Without this, the IIFE can fill the queue past // EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists. if (request.listener) subscription!.on('data', request.listener); + const finishInitialScan = beginTableOperation(); const result = (async () => { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { @@ -4361,6 +4394,7 @@ export function makeTable(options) { pendingRealTimeQueue = null; } })(); + result.then(finishInitialScan, finishInitialScan); result.catch(failSubscription); function failSubscription(error: any) { if (subscription.closed) return; @@ -4419,7 +4453,9 @@ export function makeTable(options) { // aftercommit path holds an inter-thread lock that must not span event-loop turns. async function runReloadResnapshot() { reloadResnapshotRunning = true; + let finishReloadScan: (() => void) | undefined; try { + finishReloadScan = beginTableOperation(); await rest(); // defer off the broadcast listener's stack before scanning while (reloadResnapshotPending) { reloadResnapshotPending = false; @@ -4438,6 +4474,7 @@ export function makeTable(options) { } catch (error) { harperLogger.error?.('Error in reload re-snapshot:', error); } finally { + finishReloadScan?.(); reloadResnapshotRunning = false; // A marker that landed after the last pending-check but before we cleared the flag would // otherwise be dropped — re-arm if so (unless the subscription has since closed). @@ -4445,7 +4482,7 @@ export function makeTable(options) { } } function scheduleReloadResnapshot() { - if (!subscription.subscriptions) return; + if (droppingTable || !subscription.subscriptions) return; reloadResnapshotPending = true; if (!reloadResnapshotRunning) runReloadResnapshot(); } @@ -4828,91 +4865,96 @@ export function makeTable(options) { return getStorageSpaceStats(primaryStore.path); } static async getRecordCount(options?: any) { - // iterate through the metadata entries to exclude their count and exclude the deletion counts - const exactCount = options?.exactCount; - const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms - const start = performance.now(); - // `entryCount` (the exact key count) is only needed once the scan blows the time budget -- - // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full - // key-only scan, so we defer it: tables that finish within budget (the common case) and - // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed. - let entryCount = 0; - let halfway = 0; - let counted = false; - let completeForExact = false; - let recordCount = 0; - let entriesScanned = 0; - let limit: number; - for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) { - if (value != null) recordCount++; - entriesScanned++; - await rest(); - if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { - if (!counted) { - counted = true; - entryCount = isRocksDB - ? primaryStore.getKeysCount({ start: undefined }) - : primaryStore.getStats().entryCount; - halfway = Math.floor(entryCount / 2); - } - if (entriesScanned < halfway) { - // it is taking too long, so we will just take this sample and a sample from the end to estimate - limit = entriesScanned; - break; - } - // Past the halfway point already: finishing the scan for an exact count is cheaper - // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration. - completeForExact = true; - } - } - if (limit) { - // in this case we are going to make an estimate of the table count using the first thousand - // entries and last thousand entries - const firstRecordCount = recordCount; - recordCount = 0; - // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but - // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate` - // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source - // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB - // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint. - let reverseScanned = 0; - for (const { value } of primaryStore.getRange({ - start: '\uffff', - reverse: true, - lazy: true, - limit, - snapshot: false, - })) { + const finishRecordCountScan = beginTableOperation(); + try { + // iterate through the metadata entries to exclude their count and exclude the deletion counts + const exactCount = options?.exactCount; + const TIME_LIMIT = options?.timeLimit ?? 1000 / 2; // one second time limit, enforced by seeing if we are halfway through at 500ms + const start = performance.now(); + // `entryCount` (the exact key count) is only needed once the scan blows the time budget -- + // to decide whether to estimate and as the extrapolation base. On RocksDB it is a full + // key-only scan, so we defer it: tables that finish within budget (the common case) and + // `exact_count` requests never pay for it. `halfway`/`entryCount` stay 0 until first computed. + let entryCount = 0; + let halfway = 0; + let counted = false; + let completeForExact = false; + let recordCount = 0; + let entriesScanned = 0; + let limit: number; + for (const { value } of primaryStore.getRange({ start: true, lazy: true, snapshot: false })) { if (value != null) recordCount++; - reverseScanned++; + entriesScanned++; await rest(); - if (reverseScanned >= limit) break; + if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { + if (!counted) { + counted = true; + entryCount = isRocksDB + ? primaryStore.getKeysCount({ start: undefined }) + : primaryStore.getStats().entryCount; + halfway = Math.floor(entryCount / 2); + } + if (entriesScanned < halfway) { + // it is taking too long, so we will just take this sample and a sample from the end to estimate + limit = entriesScanned; + break; + } + // Past the halfway point already: finishing the scan for an exact count is cheaper + // than estimating. Set the flag so we stop re-evaluating the budget on each remaining iteration. + completeForExact = true; + } + } + if (limit) { + // in this case we are going to make an estimate of the table count using the first thousand + // entries and last thousand entries + const firstRecordCount = recordCount; + recordCount = 0; + // Bound the reverse scan explicitly. The getRange `limit` option is honored by lmdb-js but + // ignored by rocksdb-js; without this break the scan reads the whole table, so `recordRate` + // blows up to ~entryCount/(2*limit) and the estimate scales with entryCount^2 -- the source + // of the wildly inflated `record_count` (e.g. 20,000,000 for ~105k rows) on large RocksDB + // tables. The early-exit above guarantees limit < entryCount/2, so the two samples stay disjoint. + let reverseScanned = 0; + for (const { value } of primaryStore.getRange({ + start: '\uffff', + reverse: true, + lazy: true, + limit, + snapshot: false, + })) { + if (value != null) recordCount++; + reverseScanned++; + await rest(); + if (reverseScanned >= limit) break; + } + // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` + // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting + // those un-scanned slots would inflate the denominator and underestimate the rate. + const sampleSize = limit + reverseScanned; + const recordRate = (recordCount + firstRecordCount) / sampleSize; + const variance = + Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples + (recordRate * (1 - recordRate)) / sampleSize; + const sd = Math.max(Math.sqrt(variance) * entryCount, 1); + const estimatedRecordCount = Math.round(recordRate * entryCount); + // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using + // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement). + const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount); + const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount); + let significantUnit = Math.pow(10, Math.round(Math.log10(sd))); + if (significantUnit > estimatedRecordCount) significantUnit = significantUnit / 10; + recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit; + return { + recordCount, + estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)], + }; } - // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` - // (concurrent deletions under snapshot:false, or an overestimated entryCount), and counting - // those un-scanned slots would inflate the denominator and underestimate the rate. - const sampleSize = limit + reverseScanned; - const recordRate = (recordCount + firstRecordCount) / sampleSize; - const variance = - Math.pow((recordCount - firstRecordCount + 1) / limit / 2, 2) + // variance between samples - (recordRate * (1 - recordRate)) / sampleSize; - const sd = Math.max(Math.sqrt(variance) * entryCount, 1); - const estimatedRecordCount = Math.round(recordRate * entryCount); - // TODO: This uses a normal/Wald interval, but a binomial confidence interval is probably better calculated using - // Wilson score interval or Agresti-Coull interval (I think the latter is a little easier to calculate/implement). - const lowerCiLimit = Math.max(estimatedRecordCount - 1.96 * sd, recordCount + firstRecordCount); - const upperCiLimit = Math.min(estimatedRecordCount + 1.96 * sd, entryCount); - let significantUnit = Math.pow(10, Math.round(Math.log10(sd))); - if (significantUnit > estimatedRecordCount) significantUnit = significantUnit / 10; - recordCount = Math.round(estimatedRecordCount / significantUnit) * significantUnit; return { recordCount, - estimatedRange: [Math.round(lowerCiLimit), Math.round(upperCiLimit)], }; + } finally { + finishRecordCountScan(); } - return { - recordCount, - }; } /** * When attributes have been changed, we update the accessors that are assigned to this table @@ -5183,98 +5225,119 @@ export function makeTable(options) { this.userSetEmbedders.add(attribute_name); } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise { - let completion: Promise; - let entriesDeleted = 0; - for (const auditRecord of auditStore.getRange({ - start: 0, - end: endTime, - })) { - await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId) continue; - completion = removeAuditEntry(auditStore, auditRecord); - entriesDeleted++; - } - if (cleanupDeletedRecords) { - // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't - // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records - for (const entry of primaryStore.getRange({ start: 0, versions: true })) { - const { value, localTime } = entry; + const finishHistoryScan = beginTableOperation(); + try { + let completion: Promise; + let entriesDeleted = 0; + for (const auditRecord of auditStore.getRange({ + start: 0, + end: endTime, + })) { await rest(); // yield to other async operations - if (value === null && localTime < endTime) { - completion = removeEntry(primaryStore, entry); + if (auditRecord.tableId !== tableId) continue; + completion = removeAuditEntry(auditStore, auditRecord); + entriesDeleted++; + } + if (cleanupDeletedRecords) { + // this is separate procedure we can do if the records are not being cleaned up by the audit log. This shouldn't + // ever happen, but if there are cleanup failures for some reason, we can run this to clean up the records + for (const entry of primaryStore.getRange({ start: 0, versions: true })) { + const { value, localTime } = entry; + await rest(); // yield to other async operations + if (value === null && localTime < endTime) { + completion = removeEntry(primaryStore, entry); + } } } + await completion; + return entriesDeleted; + } finally { + finishHistoryScan(); } - await completion; - return entriesDeleted; } static async *getHistory(startTime = 0, endTime = Infinity) { - for (const auditRecord of auditStore.getRange({ - start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols - end: endTime, - })) { - await rest(); // yield to other async operations - if (auditRecord.tableId !== tableId) continue; - yield { - id: auditRecord.recordId, - localTime: auditRecord.version, - version: auditRecord.version, - type: auditRecord.type, - value: auditRecord.getValue(primaryStore, true, auditRecord.version), - user: auditRecord.user, - operation: auditRecord.originatingOperation, - }; + const finishHistoryScan = beginTableOperation(); + try { + for (const auditRecord of auditStore.getRange({ + start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols + end: endTime, + })) { + await rest(); // yield to other async operations + if (auditRecord.tableId !== tableId) continue; + yield { + id: auditRecord.recordId, + localTime: auditRecord.version, + version: auditRecord.version, + type: auditRecord.type, + value: auditRecord.getValue(primaryStore, true, auditRecord.version), + user: auditRecord.user, + operation: auditRecord.originatingOperation, + }; + } + } finally { + finishHistoryScan(); } } static async getHistoryOfRecord(id) { - const history = []; - if (id == undefined) throw new Error('An id is required'); - const entry = primaryStore.getEntry(id); - if (!entry) return history; - let nextVersion = entry.localTime; - if (!nextVersion) throw new Error('The entry does not have a local audit time'); - const count = 0; - const auditWindow = 100; - do { - await rest(); // yield to other async operations - let insertionPoint = history.length; - let highestPreviousVersion = 0; - const start = nextVersion - auditWindow; - for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { - if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) { - history.splice(insertionPoint, 0, { - id: auditRecord.recordId, - localTime: auditRecord.version, - version: auditRecord.version, - type: auditRecord.type, - // reconstruct each entry's record image as of its own version, not the audit - // window boundary (nextVersion), matching getHistory (issue #1330) - value: auditRecord.getValue(primaryStore, true, auditRecord.version), - user: auditRecord.user, - operation: auditRecord.originatingOperation, - }); - if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) { - highestPreviousVersion = auditRecord.previousVersion; + const finishHistoryScan = beginTableOperation(); + try { + const history = []; + if (id == undefined) throw new Error('An id is required'); + const entry = primaryStore.getEntry(id); + if (!entry) return history; + let nextVersion = entry.localTime; + if (!nextVersion) throw new Error('The entry does not have a local audit time'); + const count = 0; + const auditWindow = 100; + do { + await rest(); // yield to other async operations + let insertionPoint = history.length; + let highestPreviousVersion = 0; + const start = nextVersion - auditWindow; + for (const auditRecord of auditStore.getRange({ start, end: nextVersion + 0.001 })) { + if (auditRecord.tableId === tableId && compareKeys(auditRecord.recordId, id) === 0) { + history.splice(insertionPoint, 0, { + id: auditRecord.recordId, + localTime: auditRecord.version, + version: auditRecord.version, + type: auditRecord.type, + // reconstruct each entry's record image as of its own version, not the audit + // window boundary (nextVersion), matching getHistory (issue #1330) + value: auditRecord.getValue(primaryStore, true, auditRecord.version), + user: auditRecord.user, + operation: auditRecord.originatingOperation, + }); + if (auditRecord.previousVersion > highestPreviousVersion && auditRecord.previousVersion < start) { + highestPreviousVersion = auditRecord.previousVersion; + } } } - } - nextVersion = highestPreviousVersion; - } while (count < 1000 && nextVersion); - return history.reverse(); + nextVersion = highestPreviousVersion; + } while (count < 1000 && nextVersion); + return history.reverse(); + } finally { + finishHistoryScan(); + } } - static clear() { - // clear the primary store and every secondary index dbi (same pattern used by - // runIndexing when rebuilding from scratch), so clear() doesn't leave stale - // index entries pointing at records that no longer exist. - const promises = [primaryStore.clear()]; - for (const key in indices) { - const index = indices[key]; - promises.push(index.clearAsync ? index.clearAsync() : index.clear()); + static async clear() { + const finishClear = beginTableOperation(); + try { + // clear the primary store and every secondary index dbi (same pattern used by + // runIndexing when rebuilding from scratch), so clear() doesn't leave stale + // index entries pointing at records that no longer exist. + const promises = [primaryStore.clear()]; + for (const key in indices) { + const index = indices[key]; + promises.push(index.clearAsync ? index.clearAsync() : index.clear()); + } + return await Promise.all(promises); + } finally { + finishClear(); } - return Promise.all(promises); } static cleanup() { deleteCallbackHandle?.remove(); + stopBackgroundScans(); } static _readTxnForContext(context) { return txnForContext(context).getReadTxn(); @@ -5620,11 +5683,7 @@ export function makeTable(options) { } } function txnForContext(context: Context) { - if (isRocksDB && droppingTable) { - const error: any = new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 409); - error.code = 'ERR_TABLE_DROPPING'; - throw error; - } + if (isRocksDB && droppingTable) throw tableDroppingError(); let transaction = context?.transaction; if (isReleasedTransaction(transaction)) transaction = undefined; if (transaction) { @@ -6337,7 +6396,9 @@ export function makeTable(options) { return false; } + let finishCleanupScan: (() => void) | undefined; try { + finishCleanupScan = beginTableOperation(); let count = 0; let removeDeletedRecords = !audit || isRocksDB; // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize @@ -6386,6 +6447,8 @@ export function makeTable(options) { logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); } catch (error) { logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + } finally { + finishCleanupScan?.(); } resolve(undefined); cleanupPriority = 0; // reset the priority @@ -6406,12 +6469,14 @@ export function makeTable(options) { // Periodically evict expired records, searching for records who expiresAt timestamp is before now if (getWorkerIndex() === 0) { // we want to run the pruning of expired records on only one thread so we don't have conflicts in evicting - setInterval(async () => { + recordExpirationInterval = setInterval(async () => { // go through each database and table and then search for expired entries // find any entries that are set to expire before now if (runningRecordExpiration) return; runningRecordExpiration = true; + let finishExpirationScan: (() => void) | undefined; try { + finishExpirationScan = beginTableOperation(); const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); @@ -6436,6 +6501,7 @@ export function makeTable(options) { } catch (error) { logger.error?.('Error in evicting old records', error); } finally { + finishExpirationScan?.(); runningRecordExpiration = false; } }, RECORD_PRUNING_INTERVAL).unref(); diff --git a/server/threads/itc.js b/server/threads/itc.js index 1fab18f9c4..5f6386ff2a 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,7 +3,7 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { isMainThread, threadId } = require('worker_threads'); +const { isMainThread, parentPort, threadId } = require('worker_threads'); const harperLogger = require('../../utility/logging/harper_logger.ts'); const { onMessageFromWorkers, @@ -26,7 +26,8 @@ onMessageFromWorkers(async (event, sender) => { let handlerError; try { serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - validateEvent(event); + const validationError = validateEvent(event); + if (validationError) throw new Error(validationError); if (serverItcHandlers[event.type]) { await serverItcHandlers[event.type](event); } @@ -55,6 +56,7 @@ onMessageFromWorkers(async (event, sender) => { } } }); +if (!isMainThread) parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); /** * Emits an ITC event to the ITC server. diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index c8f5292590..14d98cb34a 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -715,7 +715,7 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) // its handler successfully. A timeout, disconnect, handler error, or post failure rejects so the // caller can leave its durable recovery marker in place without touching storage. function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return broadcastAwaitingAcknowledgements(message, timeout, true, true); + return broadcastAwaitingAcknowledgements(message, timeout, true, true, connectedPorts, true); } function sendToThreadWithStrictAcknowledgement(threadId, message, timeout = DEFAULT_ACK_TIMEOUT_MS) { @@ -728,7 +728,14 @@ function sendToThreadWithStrictAcknowledgement(threadId, message, timeout = DEFA return broadcastAwaitingAcknowledgements(message, timeout, true, true, [port]); } -function broadcastAwaitingAcknowledgements(message, timeout, strict, includeJobWorkers, ports = connectedPorts) { +function broadcastAwaitingAcknowledgements( + message, + timeout, + strict, + includeJobWorkers, + ports = connectedPorts, + skipUnready = false +) { return new Promise((resolve, reject) => { let waitingCount = 0; let timer; @@ -758,6 +765,9 @@ function broadcastAwaitingAcknowledgements(message, timeout, strict, includeJobW } else resolve(); }; for (let port of ports) { + // Loading table storage registers the ITC listener before opening any table handle. Until then, + // the worker is safe to omit and could not acknowledge this barrier anyway. + if (skipUnready && !port.itcReady) continue; // Ordinary post-change gossip excludes transient job workers. Strict pre-change barriers // include them because a job can hold the same native handles; the main-thread relay keeps // a job-originated async schema operation re-entrant while it awaits its own ACK. @@ -1212,6 +1222,8 @@ function addPort(port, keepRef, isJobWorker) { addProcessGroup(portThreadId, message.processGroupId); } else if (message.type === UNREGISTER_PROCESS_GROUP) { removeProcessGroup(portThreadId, message.processGroupId); + } else if (message.type === hdbTerms.ITC_EVENT_TYPES.ITC_READY) { + port.itcReady = true; } else if (message.type === ADDED_PORT) { message.port.threadId = message.threadId; addPort(message.port, false, message.isJobWorker); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 376fdea430..bd10d6d9fa 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -7,8 +7,9 @@ const { setupTestDBPath } = require('../testUtils'); const { waitFor } = require('../waitFor'); const { table, database, databases, getDatabases, resetDatabases } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); -const { THREAD_TYPES } = require('#src/utility/hdbTerms'); +const { ITC_EVENT_TYPES, TABLE_DROP_PREPARE_OPERATION, THREAD_TYPES } = require('#src/utility/hdbTerms'); const { + broadcastWithStrictAcknowledgement, startWorker, onMessageByType, setMainIsWorker, @@ -16,6 +17,7 @@ const { } = require('#js/server/threads/manageThreads'); const WORKER_FIXTURE = path.join(__dirname, 'dropTableQuiescence-worker.js'); +const UNREADY_WORKER_FIXTURE = path.join(__dirname, 'dropTableUnready-worker.js'); const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; @@ -210,6 +212,88 @@ describe('dropTable worker quiescence', function () { if (dropResult.status === 'rejected') throw dropResult.reason; }); + it('drains a transaction-less range scan before closing the table stores', async function () { + const Table = defineTable(`DropDirectScan_${process.pid}_${Date.now()}`); + await Table.put({ id: 'scan', name: 'held' }); + + const originalSetImmediate = global.setImmediate; + let releaseScan; + global.setImmediate = (callback, ...args) => { + global.setImmediate = originalSetImmediate; + releaseScan = () => originalSetImmediate(callback, ...args); + }; + let scanPromise; + try { + scanPromise = Table.getRecordCount({ exactCount: true }); + await waitFor(() => releaseScan, { message: 'getRecordCount() did not enter its yielded range scan' }); + } finally { + global.setImmediate = originalSetImmediate; + } + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(originalSetImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the direct range scan'); + } catch (error) { + earlyError = error; + } finally { + releaseScan(); + } + const [scanResult, dropResult] = await Promise.allSettled([scanPromise, dropPromise]); + if (earlyError) throw earlyError; + if (scanResult.status === 'rejected') throw scanResult.reason; + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(destructivePhaseStarted, true); + }); + + it('omits a worker until its ITC listener is ready', async function () { + const worker = startWorker(UNREADY_WORKER_FIXTURE, { + name: THREAD_TYPES.JOB, + workerIndex: 1, + threadCount: 2, + autoRestart: false, + }); + try { + await new Promise((resolve, reject) => { + worker.once('online', resolve); + worker.once('error', reject); + }); + assert.notStrictEqual(worker.itcReady, true); + await broadcastWithStrictAcknowledgement({ type: ITC_EVENT_TYPES.SCHEMA, message: { originator: 0 } }, 50); + } finally { + worker.wasShutdown = true; + await worker.terminate(); + } + }); + + it('NACKs a malformed strict schema event', async function () { + const worker = startDropWorker(1, 2); + try { + await worker.booted; + assert.strictEqual(worker.worker.itcReady, true); + await assert.rejects( + () => + broadcastWithStrictAcknowledgement( + { + type: ITC_EVENT_TYPES.SCHEMA, + message: { operation: TABLE_DROP_PREPARE_OPERATION }, + }, + 1000 + ), + /originator/i + ); + } finally { + await worker.shutdown(); + } + }); + it('defers an unquiesced tombstone until the process that could hold stale handles is gone', function () { const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); diff --git a/unitTests/resources/dropTableUnready-worker.js b/unitTests/resources/dropTableUnready-worker.js new file mode 100644 index 0000000000..8b1df11d8d --- /dev/null +++ b/unitTests/resources/dropTableUnready-worker.js @@ -0,0 +1,6 @@ +'use strict'; + +const { parentPort } = require('node:worker_threads'); + +// Deliberately never loads Table.ts or server/threads/itc.js. +parentPort.on('message', () => {}); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 7b164aeaa9..3ff00e9849 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -923,6 +923,7 @@ export const JWT_ENUM = { /** ITC Channel Event types */ export const ITC_EVENT_TYPES = { SHUTDOWN: 'shutdown', + ITC_READY: 'itc_ready', CHILD_STARTED: 'child_started', CHILD_STOPPED: 'child_stopped', SCHEMA: 'schema', From 182e0cee45084ec2c0f784f09401e4704520af70 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 13:49:33 -0600 Subject: [PATCH 05/40] fix: make drop quiescence cancellation-safe Cancel client-paced and abandoned direct scans when a RocksDB table begins dropping, while retaining labeled drain tokens for diagnostics. Scope ITC validation to owned events and publish worker readiness through an atomic signal so coordinator observation cannot lag store opening. Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 104 +++++++++++++----- server/threads/itc.js | 11 +- server/threads/manageThreads.js | 22 ++-- .../resources/dropTableQuiescence-worker.js | 10 +- .../resources/dropTableQuiescence.test.js | 47 +++++++- 5 files changed, 155 insertions(+), 39 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index d3e9bfb8ff..5236e3998b 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -406,7 +406,11 @@ export function makeTable(options) { // in-flight commit promises here so dropTable() can drain them first, and stop admitting // new ones (droppingTable) once a drop has actually started. const pendingSourceCommits = new Set>(); - const pendingTableOperations = new Set>(); + const pendingTableOperations = new Set<{ + completion: Promise; + label: string; + cancel?: () => void; + }>(); let droppingTable = false; let coordinatingDrop = false; let storesClosed = false; @@ -417,38 +421,56 @@ export function makeTable(options) { error.code = 'ERR_TABLE_DROPPING'; return error; }; - const beginTableOperation = () => { + // Rocks operations that bypass txnForContext must hold this token across every yield so drop can see them. + const beginTableOperation = (label: string, cancel?: () => void) => { if (!isRocksDB) return () => {}; if (droppingTable) throw tableDroppingError(); let resolve: () => void; const completion = new Promise((scanResolve) => { resolve = scanResolve; }); - pendingTableOperations.add(completion); + const operation = { completion, label, cancel }; + pendingTableOperations.add(operation); let active = true; return () => { if (!active) return; active = false; - pendingTableOperations.delete(completion); + pendingTableOperations.delete(operation); resolve(); }; }; + const stopCleanupTimer = () => { + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = undefined; + cleanupTimerCompletion?.resolve(); + cleanupTimerCompletion = undefined; + } + }; const stopBackgroundScans = () => { - if (cleanupTimer) clearTimeout(cleanupTimer); + stopCleanupTimer(); if (recordExpirationInterval) clearInterval(recordExpirationInterval); }; const markTableDropping = () => { droppingTable = true; stopBackgroundScans(); + for (const operation of [...pendingTableOperations]) { + try { + operation.cancel?.(); + } catch (error) { + logger.warn?.(`Unable to cancel ${operation.label} while dropping ${databaseName}.${tableName}`, error); + } + } if (isRocksDB) { for (const store of tableStores()) (store as any).dropping = true; } delete databases[databaseName]?.[tableName]; }; const drainTableOperations = async () => { + const directOperations = [...pendingTableOperations]; const pending = new Set>([ ...pendingSourceCommits, - ...pendingTableOperations, + ...directOperations.map(({ completion }) => completion), ...getPendingWriteResolutions(tableStores()), ...getPendingReadResolutions(tableStores()), ]); @@ -463,8 +485,11 @@ export function makeTable(options) { ]); clearTimeout(timer); if (result === timedOut) { + const directOperationLabels = directOperations + .filter((operation) => pendingTableOperations.has(operation)) + .map(({ label }) => label); throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` ); } }; @@ -490,7 +515,8 @@ export function makeTable(options) { let cleanupInterval = 86400000; let cleanupPriority = 0; let lastCleanupInterval: number; - let cleanupTimer: NodeJS.Timeout; + let cleanupTimer: NodeJS.Timeout | undefined; + let cleanupTimerCompletion: { resolve: () => void } | undefined; let recordExpirationInterval: NodeJS.Timeout; // true once a table-level expiration/eviction/scanInterval has armed the periodic cleanup scan at setup let expirationScanScheduled = false; @@ -4148,7 +4174,7 @@ export function makeTable(options) { // in subscription.queue. Without this, the IIFE can fill the queue past // EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists. if (request.listener) subscription!.on('data', request.listener); - const finishInitialScan = beginTableOperation(); + const finishInitialScan = beginTableOperation('subscription replay', () => subscription.close()); const result = (async () => { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { @@ -4455,7 +4481,7 @@ export function makeTable(options) { reloadResnapshotRunning = true; let finishReloadScan: (() => void) | undefined; try { - finishReloadScan = beginTableOperation(); + finishReloadScan = beginTableOperation('subscription reload scan', () => subscription.close()); await rest(); // defer off the broadcast listener's stack before scanning while (reloadResnapshotPending) { reloadResnapshotPending = false; @@ -4865,7 +4891,7 @@ export function makeTable(options) { return getStorageSpaceStats(primaryStore.path); } static async getRecordCount(options?: any) { - const finishRecordCountScan = beginTableOperation(); + const finishRecordCountScan = beginTableOperation('record count scan'); try { // iterate through the metadata entries to exclude their count and exclude the deletion counts const exactCount = options?.exactCount; @@ -4886,6 +4912,7 @@ export function makeTable(options) { if (value != null) recordCount++; entriesScanned++; await rest(); + if (droppingTable) throw tableDroppingError(); if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { if (!counted) { counted = true; @@ -4925,6 +4952,7 @@ export function makeTable(options) { if (value != null) recordCount++; reverseScanned++; await rest(); + if (droppingTable) throw tableDroppingError(); if (reverseScanned >= limit) break; } // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` @@ -5225,7 +5253,7 @@ export function makeTable(options) { this.userSetEmbedders.add(attribute_name); } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise { - const finishHistoryScan = beginTableOperation(); + const finishHistoryScan = beginTableOperation('history deletion scan'); try { let completion: Promise; let entriesDeleted = 0; @@ -5234,6 +5262,7 @@ export function makeTable(options) { end: endTime, })) { await rest(); // yield to other async operations + if (droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; completion = removeAuditEntry(auditStore, auditRecord); entriesDeleted++; @@ -5244,6 +5273,7 @@ export function makeTable(options) { for (const entry of primaryStore.getRange({ start: 0, versions: true })) { const { value, localTime } = entry; await rest(); // yield to other async operations + if (droppingTable) throw tableDroppingError(); if (value === null && localTime < endTime) { completion = removeEntry(primaryStore, entry); } @@ -5256,13 +5286,24 @@ export function makeTable(options) { } } static async *getHistory(startTime = 0, endTime = Infinity) { - const finishHistoryScan = beginTableOperation(); + let iterator: Iterator | undefined; + let finishHistoryScan: () => void; + finishHistoryScan = beginTableOperation('history iterator', () => { + if (!iterator?.return) return; + iterator.return(); + finishHistoryScan(); + }); try { - for (const auditRecord of auditStore.getRange({ - start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols - end: endTime, - })) { + iterator = auditStore + .getRange({ + start: startTime || 1, // if startTime is 0, we actually want to shift to 1 because 0 is encoded as all zeros with audit store's special encoder, and will include symbols + end: endTime, + }) + [Symbol.iterator](); + for (let next = iterator.next(); !next.done; next = iterator.next()) { + const auditRecord = next.value; await rest(); // yield to other async operations + if (droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; yield { id: auditRecord.recordId, @@ -5275,11 +5316,12 @@ export function makeTable(options) { }; } } finally { + iterator?.return?.(); finishHistoryScan(); } } static async getHistoryOfRecord(id) { - const finishHistoryScan = beginTableOperation(); + const finishHistoryScan = beginTableOperation('record history scan'); try { const history = []; if (id == undefined) throw new Error('An id is required'); @@ -5291,6 +5333,7 @@ export function makeTable(options) { const auditWindow = 100; do { await rest(); // yield to other async operations + if (droppingTable) throw tableDroppingError(); let insertionPoint = history.length; let highestPreviousVersion = 0; const start = nextVersion - auditWindow; @@ -5320,7 +5363,7 @@ export function makeTable(options) { } } static async clear() { - const finishClear = beginTableOperation(); + const finishClear = beginTableOperation('table clear'); try { // clear the primary store and every secondary index dbi (same pattern used by // runIndexing when rebuilding from scratch), so clear() doesn't leave stale @@ -6310,6 +6353,9 @@ export function makeTable(options) { } return { + cancel(): void { + pending = []; + }, add(type: 'evict' | 'tombstone', key: any, version: number): Promise | void { pending.push({ type, key, version }); if (pending.length >= EVICTION_BATCH_SIZE) { @@ -6340,9 +6386,11 @@ export function makeTable(options) { lastCleanupInterval = cleanupInterval; if (getWorkerIndex() === getWorkerCount() - 1) { // run on the last thread so we aren't overloading lower-numbered threads - if (cleanupTimer) clearTimeout(cleanupTimer); + stopCleanupTimer(); if (!cleanupInterval) return; - return new Promise((resolve) => { + return new Promise((resolve) => { + const thisCleanupCompletion = { resolve }; + cleanupTimerCompletion = thisCleanupCompletion; const startOfYear = new Date(); startOfYear.setMonth(0); startOfYear.setDate(1); @@ -6398,7 +6446,7 @@ export function makeTable(options) { let finishCleanupScan: (() => void) | undefined; try { - finishCleanupScan = beginTableOperation(); + finishCleanupScan = beginTableOperation('cleanup scan'); let count = 0; let removeDeletedRecords = !audit || isRocksDB; // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize @@ -6442,14 +6490,19 @@ export function makeTable(options) { } } await rest(); + if (droppingTable) { + batcher?.cancel(); + return; + } } if (batcher) await batcher.drain(); logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); } catch (error) { - logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); + if (!droppingTable) logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); } finally { finishCleanupScan?.(); } + if (cleanupTimerCompletion === thisCleanupCompletion) cleanupTimerCompletion = undefined; resolve(undefined); cleanupPriority = 0; // reset the priority })), @@ -6476,7 +6529,7 @@ export function makeTable(options) { runningRecordExpiration = true; let finishExpirationScan: (() => void) | undefined; try { - finishExpirationScan = beginTableOperation(); + finishExpirationScan = beginTableOperation('expiration scan'); const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); @@ -6497,9 +6550,10 @@ export function makeTable(options) { } } await rest(); + if (droppingTable) return; } } catch (error) { - logger.error?.('Error in evicting old records', error); + if (!droppingTable) logger.error?.('Error in evicting old records', error); } finally { finishExpirationScan?.(); runningRecordExpiration = false; diff --git a/server/threads/itc.js b/server/threads/itc.js index 5f6386ff2a..1aa732f96b 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -3,7 +3,7 @@ const hdbUtils = require('../../utility/common_utils.ts'); const hdbTerms = require('../../utility/hdbTerms.ts'); const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts'); -const { isMainThread, parentPort, threadId } = require('worker_threads'); +const { isMainThread, parentPort, threadId, workerData } = require('worker_threads'); const harperLogger = require('../../utility/logging/harper_logger.ts'); const { onMessageFromWorkers, @@ -26,9 +26,9 @@ onMessageFromWorkers(async (event, sender) => { let handlerError; try { serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js'); - const validationError = validateEvent(event); - if (validationError) throw new Error(validationError); if (serverItcHandlers[event.type]) { + const validationError = validateEvent(event); + if (validationError) throw new Error(validationError); await serverItcHandlers[event.type](event); } if (event.relayStrictToWorkers && isMainThread) { @@ -56,7 +56,10 @@ onMessageFromWorkers(async (event, sender) => { } } }); -if (!isMainThread) parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); +if (!isMainThread) { + if (workerData?.itcReadyBuffer) Atomics.store(new Int32Array(workerData.itcReadyBuffer), 0, 1); + parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); +} /** * Emits an ITC event to the ITC server. diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 14d98cb34a..cbcd745aeb 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -216,6 +216,8 @@ let workerCount = 1; // should be assigned when workers are created const RESERVED_WORKER_DATA_KEYS = [ 'addPorts', 'addThreadIds', + 'addItcReadyBuffers', + 'itcReadyBuffer', 'workerIndex', 'workerCount', 'name', @@ -353,6 +355,7 @@ function startWorker(path, options = {}) { channelsToConnect.push(channel); portsToSend.push(channel.port2); } + const itcReadyBuffer = new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT); if (!extname(path)) path += '.js'; @@ -395,6 +398,8 @@ function startWorker(path, options = {}) { ...collectProvidedWorkerData(options), addPorts: portsToSend, addThreadIds: channelsToConnect.map((channel) => channel.existingPort.threadId), + addItcReadyBuffers: channelsToConnect.map((channel) => channel.existingPort.itcReadySignal?.buffer), + itcReadyBuffer, workerIndex: options.workerIndex, workerCount: (workerCount = options.threadCount), name: options.name, @@ -415,11 +420,12 @@ function startWorker(path, options = {}) { port: port1, threadId: worker.threadId, isJobWorker, + itcReadyBuffer, }, [port1] ); } - addPort(worker, true, isJobWorker); + addPort(worker, true, isJobWorker, itcReadyBuffer); worker.unexpectedRestarts = options.unexpectedRestarts || 0; worker.startCopy = () => { // in a shutdown sequence we use overlapping restarts, starting the new thread while waiting for the old thread @@ -765,9 +771,10 @@ function broadcastAwaitingAcknowledgements( } else resolve(); }; for (let port of ports) { - // Loading table storage registers the ITC listener before opening any table handle. Until then, - // the worker is safe to omit and could not acknowledge this barrier anyway. - if (skipUnready && !port.itcReady) continue; + // The worker publishes readiness before Table.ts can continue opening stores, so the coordinator + // does not depend on when its event loop handles the matching ITC_READY message. + const itcReady = port.itcReady || (port.itcReadySignal && Atomics.load(port.itcReadySignal, 0) === 1); + if (skipUnready && !itcReady) continue; // Ordinary post-change gossip excludes transient job workers. Strict pre-change barriers // include them because a job can hold the same native handles; the main-thread relay keeps // a job-originated async schema operation re-entrant while it awaits its own ACK. @@ -913,7 +920,7 @@ if (parentPort && workerData?.addPorts) { for (let i = 0, l = workerData.addPorts.length; i < l; i++) { let port = workerData.addPorts[i]; port.threadId = workerData.addThreadIds[i]; - addPort(port); + addPort(port, false, false, workerData.addItcReadyBuffers?.[i]); } setInterval(() => { // post our memory usage as a resource report, reporting our memory usage @@ -1211,8 +1218,9 @@ function removePort(port, deadThreadId) { } } -function addPort(port, keepRef, isJobWorker) { +function addPort(port, keepRef, isJobWorker, itcReadyBuffer) { if (isJobWorker) port.isJobWorker = true; + if (itcReadyBuffer) port.itcReadySignal = new Int32Array(itcReadyBuffer); connectedPorts.push(port); // Capture threadId now — Bun resets port.threadId to -1 by the time 'exit' fires. const portThreadId = port.threadId; @@ -1226,7 +1234,7 @@ function addPort(port, keepRef, isJobWorker) { port.itcReady = true; } else if (message.type === ADDED_PORT) { message.port.threadId = message.threadId; - addPort(message.port, false, message.isJobWorker); + addPort(message.port, false, message.isJobWorker, message.itcReadyBuffer); } else if (message.type === ACKNOWLEDGEMENT) { let completion = awaitingResponses.get(message.id); if (completion) { diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js index 67aa2cb30b..6017e71759 100644 --- a/unitTests/resources/dropTableQuiescence-worker.js +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -5,7 +5,11 @@ const { parentPort } = require('node:worker_threads'); const { setupTestDBPath } = require('../testUtils'); const { table, closeLoadedDatabases } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); -const { onMessageByType, getProcessInstanceId } = require('#js/server/threads/manageThreads'); +const { + onMessageByType, + getProcessInstanceId, + sendToThreadWithStrictAcknowledgement, +} = require('#js/server/threads/manageThreads'); const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; @@ -136,6 +140,10 @@ function runWorkerFixture() { }; report('reject-prepare-armed'); break; + case 'send-foreign-strict': + await sendToThreadWithStrictAcknowledgement(0, { type: 'resource_report', heapUsed: 1 }, 1000); + report('foreign-strict-acknowledged'); + break; case 'drop-table': { const originalDropSync = TestTable.primaryStore.dropSync; if (message.interruptAfterColumnFamilyDrop) { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index bd10d6d9fa..77ae743054 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -212,7 +212,7 @@ describe('dropTable worker quiescence', function () { if (dropResult.status === 'rejected') throw dropResult.reason; }); - it('drains a transaction-less range scan before closing the table stores', async function () { + it('cancels and drains a transaction-less range scan before closing the table stores', async function () { const Table = defineTable(`DropDirectScan_${process.pid}_${Date.now()}`); await Table.put({ id: 'scan', name: 'held' }); @@ -248,11 +248,41 @@ describe('dropTable worker quiescence', function () { } const [scanResult, dropResult] = await Promise.allSettled([scanPromise, dropPromise]); if (earlyError) throw earlyError; - if (scanResult.status === 'rejected') throw scanResult.reason; + assert.strictEqual(scanResult.status, 'rejected'); + assert.strictEqual(scanResult.reason.code, 'ERR_TABLE_DROPPING'); if (dropResult.status === 'rejected') throw dropResult.reason; assert.strictEqual(destructivePhaseStarted, true); }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { + const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); + for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); + const subscription = await Table.subscribe({ isCollection: true }); + await waitFor(() => subscription.currentDrainResolver, { + message: 'subscription replay did not pause on client backpressure', + }); + + await Table.dropTable(); + assert.strictEqual(subscription.closed, true); + }); + + it('closes an abandoned history iterator before dropping its stores', async function () { + const tableName = `DropHistoryIterator_${process.pid}_${Date.now()}`; + const Table = table({ + table: tableName, + database: 'test', + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'history', name: 'held' }); + const history = Table.getHistory()[Symbol.asyncIterator](); + const first = await history.next(); + assert.strictEqual(first.done, false); + + await Table.dropTable(); + await history.return?.(); + }); + it('omits a worker until its ITC listener is ready', async function () { const worker = startWorker(UNREADY_WORKER_FIXTURE, { name: THREAD_TYPES.JOB, @@ -278,6 +308,8 @@ describe('dropTable worker quiescence', function () { try { await worker.booted; assert.strictEqual(worker.worker.itcReady, true); + worker.worker.itcReady = false; + assert.strictEqual(Atomics.load(worker.worker.itcReadySignal, 0), 1); await assert.rejects( () => broadcastWithStrictAcknowledgement( @@ -294,6 +326,17 @@ describe('dropTable worker quiescence', function () { } }); + it('does not validate unrelated traffic handled by the shared worker listener', async function () { + const worker = startDropWorker(1, 2); + try { + await worker.booted; + worker.send('send-foreign-strict'); + await worker.nextEvent('foreign-strict-acknowledged'); + } finally { + await worker.shutdown(); + } + }); + it('defers an unquiesced tombstone until the process that could hold stale handles is gone', function () { const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); From 69b6a1155a71c69a6f3d1a60b7220e4ab724f9be Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 13:53:54 -0600 Subject: [PATCH 06/40] fix: guard cancelled history iterator resumption Reject a post-drop next() before the generator can re-enter its closed native iterator, and document the tracked-iterator attribution invariant. Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 3 ++- unitTests/resources/dropTableQuiescence.test.js | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 5236e3998b..6d0c953c91 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -5314,6 +5314,7 @@ export function makeTable(options) { user: auditRecord.user, operation: auditRecord.originatingOperation, }; + if (droppingTable) throw tableDroppingError(); } } finally { iterator?.return?.(); @@ -5739,7 +5740,7 @@ export function makeTable(options) { do { // See if this is a transaction for our database and if so, use it if (transaction.db?.path === primaryStore.path) { - // Tracked reads must join here so the drop drain records every table store they borrow. + // Every tracked iterator must join here so the drop drain can attribute every borrowed table store. transaction.trackStore(primaryStore); return transaction; } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 77ae743054..3d75da4d5d 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -280,7 +280,10 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(first.done, false); await Table.dropTable(); - await history.return?.(); + await assert.rejects( + () => history.next(), + (error) => error.code === 'ERR_TABLE_DROPPING' + ); }); it('omits a worker until its ITC listener is ready', async function () { From 2e694c2c0cc2fa1dc7ccc6499b341df295c56fb7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 14:04:21 -0600 Subject: [PATCH 07/40] fix: scope ITC readiness to managed workers Avoid sending the internal readiness envelope to raw Node workers, where it can be mistaken for an application reply. Keep resource-test worker fixtures safe when Mocha imports them without a parent port. Co-Authored-By: GPT-5 Codex --- server/threads/itc.js | 4 ++-- unitTests/resources/dropTableUnready-worker.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/threads/itc.js b/server/threads/itc.js index 1aa732f96b..485973643e 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -56,8 +56,8 @@ onMessageFromWorkers(async (event, sender) => { } } }); -if (!isMainThread) { - if (workerData?.itcReadyBuffer) Atomics.store(new Int32Array(workerData.itcReadyBuffer), 0, 1); +if (!isMainThread && workerData?.itcReadyBuffer) { + Atomics.store(new Int32Array(workerData.itcReadyBuffer), 0, 1); parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); } diff --git a/unitTests/resources/dropTableUnready-worker.js b/unitTests/resources/dropTableUnready-worker.js index 8b1df11d8d..2451ae532b 100644 --- a/unitTests/resources/dropTableUnready-worker.js +++ b/unitTests/resources/dropTableUnready-worker.js @@ -3,4 +3,4 @@ const { parentPort } = require('node:worker_threads'); // Deliberately never loads Table.ts or server/threads/itc.js. -parentPort.on('message', () => {}); +parentPort?.on('message', () => {}); From bd38b7565fefdd9a2a3adac338abaed09813cbdb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 14:20:16 -0600 Subject: [PATCH 08/40] fix(deps): sync structon root lock metadata Co-Authored-By: GPT-5 Codex --- package-lock.json | 84 ++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 48 deletions(-) diff --git a/package-lock.json b/package-lock.json index b8986f2b12..394a8a28f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1017,11 +1017,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-darwin-x64": { "version": "2.2.2", @@ -1030,11 +1032,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm": { "version": "2.2.2", @@ -1043,11 +1047,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm64": { "version": "2.2.2", @@ -1056,11 +1062,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-x64": { "version": "2.2.2", @@ -1069,11 +1077,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-win32-x64": { "version": "2.2.2", @@ -1082,11 +1092,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@colors/colors": { "version": "1.5.0", @@ -2560,9 +2572,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2579,9 +2588,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2598,9 +2604,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2617,9 +2620,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3336,11 +3336,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { "version": "3.0.4", @@ -3349,11 +3351,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { "version": "3.0.4", @@ -3362,11 +3366,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { "version": "3.0.4", @@ -3375,11 +3381,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { "version": "3.0.4", @@ -3388,11 +3396,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { "version": "3.0.4", @@ -3401,11 +3411,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@noble/hashes": { "version": "1.8.0", @@ -3574,9 +3586,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3594,9 +3603,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3614,9 +3620,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3634,9 +3637,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3654,9 +3654,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3674,9 +3671,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3694,9 +3688,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3714,9 +3705,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ From 0dda484014cab3b93b6ab2c93d8e0ca8582f4c65 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 14:23:50 -0600 Subject: [PATCH 09/40] test: make drop worker teardown failure-safe Co-Authored-By: GPT-5 Codex --- unitTests/resources/dropTableQuiescence.test.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 3d75da4d5d..866275f61e 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -97,6 +97,10 @@ function startDropWorker(workerIndex, threadCount, name = 'drop-table-quiescence }; } +async function shutdownWorkers(...workers) { + await Promise.all(workers.filter(Boolean).map((worker) => worker.shutdown().catch(() => undefined))); +} + describe('dropTable worker quiescence', function () { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; @@ -458,7 +462,7 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(tombstone?.dropQuiesced, false); assert.deepStrictEqual(remote.errors, [], 'the expected NACK must not become an unhandled rejection'); } finally { - await remote?.shutdown(); + await shutdownWorkers(remote); const tombstone = dbisDb.getSync(`${tableName}/`); if (tombstone?.dropping) { tombstone.dropProcessInstance = `${getProcessInstanceId()}-prior`; @@ -503,7 +507,7 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(dropResult.outcome, 'resolved'); assert.deepStrictEqual([...origin.errors, ...remote.errors], []); } finally { - await Promise.all([origin?.shutdown(), remote?.shutdown()]); + await shutdownWorkers(origin, remote); } }); @@ -541,7 +545,7 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(dropResult.outcome, 'resolved'); assert.deepStrictEqual([...origin.errors, ...remote.errors], []); } finally { - await Promise.all([origin?.shutdown(), remote?.shutdown()]); + await shutdownWorkers(origin, remote); } }); @@ -614,7 +618,7 @@ describe('dropTable worker quiescence', function () { 'no worker rejection should escape the quiescence or recovery path' ); } finally { - await Promise.all([origin?.shutdown(), remote?.shutdown()]); + await shutdownWorkers(origin, remote); } }); }); From 7e3a2f9d45c06bf5da147cfdf6f5e2f8f2a56298 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 14:29:32 -0600 Subject: [PATCH 10/40] fix: accept confirmed worker exits during drop barriers Co-Authored-By: GPT-5 Codex --- server/threads/manageThreads.js | 24 +++++++----- .../resources/dropTableQuiescence.test.js | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index cbcd745aeb..55397a064d 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -718,8 +718,9 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) } // Destructive work uses the strict variant: every connected thread, including jobs, must finish -// its handler successfully. A timeout, disconnect, handler error, or post failure rejects so the -// caller can leave its durable recovery marker in place without touching storage. +// its handler successfully or exit completely. A timeout, ambiguous MessagePort disconnect, +// handler error, or post failure rejects so the caller can leave its durable recovery marker in +// place without touching storage. function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { return broadcastAwaitingAcknowledgements(message, timeout, true, true, connectedPorts, true); } @@ -779,20 +780,21 @@ function broadcastAwaitingAcknowledgements( // include them because a job can hold the same native handles; the main-thread relay keeps // a job-originated async schema operation re-entrant while it awaits its own ACK. if (port.isJobWorker && !includeJobWorkers) continue; + const targetThreadId = port.threadId; let referenced = false; let requestId = nextId++; - const ackHandler = (acknowledgement) => { + const ackHandler = (acknowledgement, threadExited = false) => { if (!pending.delete(ackHandler)) return; // already settled for this port awaitingResponses.delete(requestId); if (strict) { const ackError = acknowledgement?.error; if (ackError) { failures.push({ - threadId: port.threadId, + threadId: targetThreadId, message: ackError.message ?? String(ackError), }); - } else if (!acknowledgement) { - failures.push({ threadId: port.threadId, message: 'worker disconnected before acknowledging' }); + } else if (!acknowledgement && !threadExited) { + failures.push({ threadId: targetThreadId, message: 'worker disconnected before acknowledging' }); } } waitingCount--; @@ -800,6 +802,7 @@ function broadcastAwaitingAcknowledgements( finish(); }; ackHandler.port = port; + ackHandler.threadId = targetThreadId; pending.add(ackHandler); waitingCount++; awaitingResponses.set((message.requestId = requestId), ackHandler); @@ -810,10 +813,13 @@ function broadcastAwaitingAcknowledgements( if (!port.hasAckCloseListener) { // just set a single close listener that can clean up all the ack handlers for a port that is closed port.hasAckCloseListener = true; - port.on(port.close ? 'close' : 'exit', () => { + const disconnectEvent = port.close ? 'close' : 'exit'; + port.on(disconnectEvent, () => { for (let [, ackHandler] of awaitingResponses) { if (ackHandler.port === port) { - ackHandler(); + // A Worker exit means the thread can no longer use its stale handles. A sibling + // MessagePort can close while its owning thread remains alive, so that stays a NACK. + ackHandler(undefined, disconnectEvent === 'exit'); } } }); @@ -831,7 +837,7 @@ function broadcastAwaitingAcknowledgements( timer = undefined; const stuck = []; for (let ackHandler of [...pending]) { - stuck.push(ackHandler.port?.threadId); + stuck.push(ackHandler.threadId); ackHandler({ error: { message: `no acknowledgement within ${timeout}ms` } }); } harperLogger.warn( diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 866275f61e..6145e3b8cf 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -473,6 +473,44 @@ describe('dropTable worker quiescence', function () { } }); + it('continues after a worker fully exits during preparation', async function () { + this.timeout(30000); + const tableName = `DropWorkerExit_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: 'test', table: null }).dbisDb; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName }); + await remote.nextEvent('ready'); + remote.send('begin-transaction', { id: 'exiting-worker-staged' }); + await remote.nextEvent('transaction-staged'); + + const dropPromise = Table.dropTable(); + await remote.nextEvent('prepare-entered'); + remote.worker.wasShutdown = true; + await remote.worker.terminate(); + remote = undefined; + await dropPromise; + + assert.ok( + !rootStore.columns.some((column) => column.startsWith(`${tableName}/`)), + 'a fully exited worker can no longer issue writes through its stale handles' + ); + } finally { + await shutdownWorkers(remote); + const tombstone = dbisDb.getSync(`${tableName}/`); + if (tombstone?.dropping) { + tombstone.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, tombstone); + resetDatabases(); + getDatabases(); + } + } + }); + it('drains a remote staged transaction before a worker-originated drop', async function () { this.timeout(30000); const tableName = `DropRemoteTransaction_${process.pid}_${Date.now()}`; From a076039a0cd0c1c7e692311639c6e7f378c7612a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 14:55:02 -0600 Subject: [PATCH 11/40] fix: reject self-draining table drops Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 7 +++++++ resources/Table.ts | 15 ++++++++++++++- .../resources/dropTableQuiescence.test.js | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index bde8f72a2d..68fea93c6b 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -654,6 +654,13 @@ export class DatabaseTransaction implements Transaction { return false; } + hasWritesForAnyStore(stores: Set): boolean { + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + if (transaction.writes.some((write) => write && stores.has(write.store))) return true; + } + return false; + } + private finishPendingWrites(): void { activeWriteTransactions.delete(this); this.#trackedForDropDrain = false; diff --git a/resources/Table.ts b/resources/Table.ts index 6d0c953c91..6df577cb13 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1475,6 +1475,19 @@ export function makeTable(options) { } const rootStore = primaryStore.rootStore; const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; + const activeTransaction = contextStorage.getStore()?.transaction; + if ( + sharedRocksStore && + activeTransaction instanceof DatabaseTransaction && + activeTransaction.hasWritesForAnyStore(new Set(tableStores())) + ) { + const error: any = new ClientError( + `Cannot drop ${databaseName}.${tableName} from a transaction with staged writes to that table; commit or abort the transaction first`, + 409 + ); + error.code = 'ERR_TABLE_DROP_IN_TRANSACTION'; + throw error; + } let dropGeneration: string | undefined; if (databaseName === databasePath) { const primaryCatalogKey = TableResource.tableName + '/'; @@ -1591,7 +1604,7 @@ export function makeTable(options) { try { closeTableStores(); } catch (error) { - logger.warn(`Failed to close table handles for ${databaseName}.${tableName}`, error); + logger.warn?.(`Failed to close table handles for ${databaseName}.${tableName}`, error); } } } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 6145e3b8cf..a684dc497f 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -160,6 +160,24 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(destructivePhaseStarted, true, 'dropTable() should continue after the transaction settles'); }); + it('rejects a drop from its own staged-write transaction before tombstoning', async function () { + const tableName = `DropOwnTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await assert.rejects( + () => + transaction(async () => { + await Table.put({ id: 'staged', name: 'pending' }); + await Table.dropTable(); + }), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + it('does not wait for a read iterator on another table in the same database', async function () { const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); From 88b44d85a848359fd4e190cff81f431b06574c12 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 15:11:33 -0600 Subject: [PATCH 12/40] test: isolate forced worker-exit database Co-Authored-By: GPT-5 Codex --- .../resources/dropTableQuiescence-worker.js | 2 +- .../resources/dropTableQuiescence.test.js | 31 ++++++++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js index 6017e71759..d452c4becc 100644 --- a/unitTests/resources/dropTableQuiescence-worker.js +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -42,7 +42,7 @@ function runWorkerFixture() { } TestTable = table({ table: message.table, - database: 'test', + database: message.database ?? 'test', attributes, }); if (message.withEmbed) { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index a684dc497f..2ef6b2213a 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -21,10 +21,10 @@ const UNREADY_WORKER_FIXTURE = path.join(__dirname, 'dropTableUnready-worker.js' const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; -function defineTable(name, withEmbed = false) { +function defineTable(name, withEmbed = false, databaseName = 'test') { const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; if (withEmbed) attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); - return table({ table: name, database: 'test', attributes }); + return table({ table: name, database: databaseName, attributes }); } function startDropWorker(workerIndex, threadCount, name = 'drop-table-quiescence-test') { @@ -178,6 +178,23 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('rejects a drop from its own read transaction before tombstoning', async function () { + const tableName = `DropOwnReadTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await transaction(async (dbTransaction) => { + Table._readTxnForContext({ transaction: dbTransaction }); + await assert.rejects( + () => Table.dropTable(), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + }); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + it('does not wait for a read iterator on another table in the same database', async function () { const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); @@ -493,15 +510,19 @@ describe('dropTable worker quiescence', function () { it('continues after a worker fully exits during preparation', async function () { this.timeout(30000); + // Force-terminating a worker intentionally bypasses closeLoadedDatabases(), so rocksdb-js keeps + // that worker's process-global handle and snapshot watermark. Isolate the test's sacrificial + // database so the leaked snapshot cannot pin later blob-reclamation tests on the shared test DB. + const databaseName = `DropWorkerExitDb_${process.pid}_${Date.now()}`; const tableName = `DropWorkerExit_${process.pid}_${Date.now()}`; - const Table = defineTable(tableName); + const Table = defineTable(tableName, false, databaseName); const rootStore = Table.primaryStore.rootStore; - const dbisDb = database({ database: 'test', table: null }).dbisDb; + const dbisDb = database({ database: databaseName, table: null }).dbisDb; let remote; try { remote = startDropWorker(1, 2); await remote.booted; - remote.send('initialize', { table: tableName }); + remote.send('initialize', { table: tableName, database: databaseName }); await remote.nextEvent('ready'); remote.send('begin-transaction', { id: 'exiting-worker-staged' }); await remote.nextEvent('transaction-staged'); From 8c94dac07176582550e55b359aff01eea4c9facb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 15:13:55 -0600 Subject: [PATCH 13/40] fix: reject self-draining read drops Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 7 +++++++ resources/Table.ts | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 68fea93c6b..2d075a91e9 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -661,6 +661,13 @@ export class DatabaseTransaction implements Transaction { return false; } + hasOpenReadsForAnyStore(stores: Set): boolean { + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + if (transaction.transaction && transaction.readTxnsUsed > 0 && transaction.usesAnyStore(stores)) return true; + } + return false; + } + private finishPendingWrites(): void { activeWriteTransactions.delete(this); this.#trackedForDropDrain = false; diff --git a/resources/Table.ts b/resources/Table.ts index 6df577cb13..584940c723 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1476,13 +1476,15 @@ export function makeTable(options) { const rootStore = primaryStore.rootStore; const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; const activeTransaction = contextStorage.getStore()?.transaction; + const currentTableStores = new Set(tableStores()); if ( sharedRocksStore && activeTransaction instanceof DatabaseTransaction && - activeTransaction.hasWritesForAnyStore(new Set(tableStores())) + (activeTransaction.hasWritesForAnyStore(currentTableStores) || + activeTransaction.hasOpenReadsForAnyStore(currentTableStores)) ) { const error: any = new ClientError( - `Cannot drop ${databaseName}.${tableName} from a transaction with staged writes to that table; commit or abort the transaction first`, + `Cannot drop ${databaseName}.${tableName} from a transaction with active reads or staged writes to that table; complete the transaction first`, 409 ); error.code = 'ERR_TABLE_DROP_IN_TRANSACTION'; From 1eb8a62a09e7af939a7b8f9f89199fac569b8a60 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 15:25:51 -0600 Subject: [PATCH 14/40] fix: quiesce tombstoned classes during reconcile Co-Authored-By: GPT-5 Codex --- resources/databases.ts | 23 +++- .../resources/dropTableQuiescence.test.js | 105 ++++++++++++------ 2 files changed, 94 insertions(+), 34 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 6e55f4e5f4..ce629152b1 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -381,6 +381,19 @@ const MAX_INTERRUPTED_DROP_ATTEMPTS = 3; const interruptedDropAttempts = new Map>(); const incompleteTableDropPreparations = new Map>(); const interruptedDropTableKey = (storePath: string, tableName: string) => `${storePath}\0${tableName}`; +const tableDropPreparationKey = (storePath: string, tableName: string, generation?: string) => + `${storePath}\0${tableName}\0${generation ?? 'legacy'}`; + +function retainTableForDropPreparation(storePath: string, tableName: string, generation: string | undefined, Table) { + const preparationKey = tableDropPreparationKey(storePath, tableName, generation); + let matchingTables = incompleteTableDropPreparations.get(preparationKey); + if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); + matchingTables.add(Table); + // Marking and cancellation happen synchronously before _prepareDrop's first await. Keep the class + // in the preparation set after the drain so the later strict barrier can close its handles. + return Table._prepareDrop({ closeStores: false }); +} + function getInterruptedDropAttempts(storePath: string, tableName: string, generation?: string): number { return interruptedDropAttempts.get(interruptedDropTableKey(storePath, tableName))?.get(generation ?? 'legacy') ?? 0; } @@ -778,6 +791,14 @@ function initStores( // its dropped handles reachable after this reconcile pass. definedTables?.delete(tableName); if (!canCompleteInterruptedDrop(tableDef.primary)) { + // The tombstone can become visible here before this worker receives the coordinator's + // strict barrier. Prepare the still-live class now; otherwise the registry cleanup below + // makes it undiscoverable to prepareTableDrop while its timers and handles remain active. + const liveTable = tables[tableName]; + if (liveTable) + retainTableForDropPreparation(rootStore.path, tableName, tableDef.primary.dropGeneration, liveTable).catch( + (error) => logger.warn(`Failed to quiesce ${databaseName}.${tableName} during schema reconciliation`, error) + ); logger.debug( `Deferring interrupted drop of table ${databaseName}.${tableName} until worker quiescence or a clean process start` ); @@ -2405,7 +2426,7 @@ export async function prepareTableDrop( dropGeneration: string | undefined, preserveTable?: any ): Promise { - const preparationKey = `${storePath}\0${tableName}\0${dropGeneration ?? 'legacy'}`; + const preparationKey = tableDropPreparationKey(storePath, tableName, dropGeneration); let matchingTables = incompleteTableDropPreparations.get(preparationKey); if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); if (preserveTable) matchingTables.add(preserveTable); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 2ef6b2213a..d9b0558d1f 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -2,12 +2,22 @@ require('../testUtils'); const assert = require('node:assert'); +const { mkdirSync } = require('node:fs'); const path = require('node:path'); const { setupTestDBPath } = require('../testUtils'); const { waitFor } = require('../waitFor'); -const { table, database, databases, getDatabases, resetDatabases } = require('#src/resources/databases'); +const { + table, + database, + databases, + getDatabases, + prepareTableDrop, + resetDatabases, +} = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); -const { ITC_EVENT_TYPES, TABLE_DROP_PREPARE_OPERATION, THREAD_TYPES } = require('#src/utility/hdbTerms'); +const env = require('#src/utility/environment/environmentManager'); +const terms = require('#src/utility/hdbTerms'); +const { ITC_EVENT_TYPES, TABLE_DROP_PREPARE_OPERATION, THREAD_TYPES } = terms; const { broadcastWithStrictAcknowledgement, startWorker, @@ -20,6 +30,7 @@ const WORKER_FIXTURE = path.join(__dirname, 'dropTableQuiescence-worker.js'); const UNREADY_WORKER_FIXTURE = path.join(__dirname, 'dropTableUnready-worker.js'); const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; +let testPath; function defineTable(name, withEmbed = false, databaseName = 'test') { const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; @@ -105,7 +116,7 @@ describe('dropTable worker quiescence', function () { if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; before(() => { - setupTestDBPath(); + testPath = setupTestDBPath(); setMainIsWorker(true); onMessageByType(MESSAGE_TYPE, () => {}); }); @@ -379,38 +390,66 @@ describe('dropTable worker quiescence', function () { } }); - it('defers an unquiesced tombstone until the process that could hold stale handles is gone', function () { + it('quiesces a live class before deferring its unquiesced tombstone', async function () { + const databaseName = `DropReconcileDb_${process.pid}_${Date.now()}`; const tableName = `DropUnquiesced_${process.pid}_${Date.now()}`; - const Table = defineTable(tableName); - const dbisDb = database({ database: 'test', table: null }).dbisDb; - const meta = dbisDb.getSync(`${tableName}/`); - meta.dropping = true; - meta.dropGeneration = 'unquiesced-test'; - meta.dropQuiesced = false; - meta.dropProcessInstance = getProcessInstanceId(); - dbisDb.putSync(`${tableName}/`, meta); - - resetDatabases(); - assert.strictEqual(getDatabases().test?.[tableName], undefined, 'an unquiesced table must stay unloaded'); - assert.strictEqual( - dbisDb.getSync(`${tableName}/`)?.dropping, - true, - 'recovery must preserve the tombstone while stale handles can still exist in this process' - ); + const storagePath = path.join(testPath, 'reconcile-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const Table = defineTable(tableName, false, databaseName); + const rootStore = Table.primaryStore.rootStore; + const dbisDb = database({ database: databaseName, table: null }).dbisDb; + const meta = dbisDb.getSync(`${tableName}/`); + meta.dropping = true; + meta.dropGeneration = 'unquiesced-test'; + meta.dropQuiesced = false; + meta.dropProcessInstance = getProcessInstanceId(); + dbisDb.putSync(`${tableName}/`, meta); - for (const index of Object.values(Table.indices)) index.close(); - Table.primaryStore.close(); - const priorProcessMeta = dbisDb.getSync(`${tableName}/`); - priorProcessMeta.dropProcessInstance = `${getProcessInstanceId()}-prior`; - dbisDb.putSync(`${tableName}/`, priorProcessMeta); - delete databases.test?.[tableName]; - resetDatabases(); - assert.strictEqual(getDatabases().test?.[tableName], undefined); - assert.strictEqual( - database({ database: 'test', table: null }).dbisDb.getSync(`${tableName}/`), - undefined, - 'a tombstone from a prior process should complete on restart' - ); + resetDatabases(); + assert.strictEqual( + getDatabases()[databaseName]?.[tableName], + undefined, + 'an unquiesced table must stay unloaded' + ); + assert.strictEqual( + dbisDb.getSync(`${tableName}/`)?.dropping, + true, + 'recovery must preserve the tombstone while stale handles can still exist in this process' + ); + + assert.strictEqual(Table.primaryStore.dropping, true, 'reconcile must mark the removed class as dropping'); + await prepareTableDrop(rootStore.path, tableName, meta.dropGeneration); + await waitFor( + () => { + try { + Table.primaryStore.getSync('__reconcile-close-probe__'); + return false; + } catch { + return true; + } + }, + { message: 'the strict barrier must close a reconciled class retained for preparation' } + ); + + const priorProcessMeta = dbisDb.getSync(`${tableName}/`); + priorProcessMeta.dropProcessInstance = `${getProcessInstanceId()}-prior`; + dbisDb.putSync(`${tableName}/`, priorProcessMeta); + delete databases[databaseName]?.[tableName]; + resetDatabases(); + assert.strictEqual(getDatabases()[databaseName]?.[tableName], undefined); + assert.strictEqual( + database({ database: databaseName, table: null }).dbisDb.getSync(`${tableName}/`), + undefined, + 'a tombstone from a prior process should complete on restart' + ); + } finally { + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } }); it('retries a timed-out preparation on a class already removed from the live schema', async function () { From a44ccc130d5cb827fcdad1ce5f1b14caebc7b065 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 15:52:47 -0600 Subject: [PATCH 15/40] fix: drain cleanup writes before table drop Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 18 ++++-- .../resources/dropTableQuiescence.test.js | 61 +++++++++++++++++++ 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 584940c723..99d2d4e6e6 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -6461,14 +6461,16 @@ export function makeTable(options) { } let finishCleanupScan: (() => void) | undefined; + let batcher: ReturnType | undefined; + let count = 0; + let scanCompleted = false; try { finishCleanupScan = beginTableOperation('cleanup scan'); - let count = 0; let removeDeletedRecords = !audit || isRocksDB; // RocksDB coalesces eviction/tombstone removals into shared transactions to amortize // the per-record commit cost; LMDB keeps the per-record path (eventTurnBatching already // coalesces async writes per event turn). - const batcher = isRocksDB ? createEvictionBatcher() : undefined; + batcher = isRocksDB ? createEvictionBatcher() : undefined; // iterate through all entries to find expired records and deleted records for (const entry of primaryStore.getRange({ start: false, @@ -6511,13 +6513,19 @@ export function makeTable(options) { return; } } - if (batcher) await batcher.drain(); - logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); + scanCompleted = true; } catch (error) { if (!droppingTable) logger.warn?.(`Error in cleanup scan for ${tableName}:`, error); } finally { - finishCleanupScan?.(); + try { + if (droppingTable) batcher?.cancel(); + if (batcher) await batcher.drain(); + await Promise.all(outstandingCleanupOperations.filter(Boolean)); + } finally { + finishCleanupScan?.(); + } } + if (scanCompleted) logger.debug?.(`Finished cleanup scan for ${tableName}, evicted ${count} entries`); if (cleanupTimerCompletion === thisCleanupCompletion) cleanupTimerCompletion = undefined; resolve(undefined); cleanupPriority = 0; // reset the priority diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index d9b0558d1f..4aefc5d34e 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -304,6 +304,67 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(destructivePhaseStarted, true); }); + it('drains an in-flight cleanup batch before dropping the table stores', async function () { + this.timeout(15000); + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropCleanupBatch_${process.pid}_${Date.now()}`); + Table.setTTLExpiration({ expiration: 3600, eviction: 3600, scanInterval: 3600 }); + for (let id = 0; id < 100; id++) { + await Table.put(id, { name: `expired-${id}` }, { expiresAt: 1 }); + } + + const originalCommit = Transaction.prototype.commit; + const originalGetEntry = Table.primaryStore.getEntry; + const cleanupTransactions = new WeakSet(); + Table.primaryStore.getEntry = function (...args) { + const [, options] = args; + if (options?.transaction) cleanupTransactions.add(options.transaction); + return originalGetEntry.apply(this, args); + }; + let cleanupCommitEntered; + const cleanupCommitStarted = new Promise((resolve) => (cleanupCommitEntered = resolve)); + let releaseCleanupCommit; + const cleanupCommitGate = new Promise((resolve) => (releaseCleanupCommit = resolve)); + let blockedCommit = false; + Transaction.prototype.commit = async function (...args) { + if (!blockedCommit && cleanupTransactions.has(this)) { + blockedCommit = true; + cleanupCommitEntered(); + await cleanupCommitGate; + } + return originalCommit.apply(this, args); + }; + + try { + Table.setTTLExpiration({ expiration: 0.001, eviction: 0.001, scanInterval: 0.001 }); + await cleanupCommitStarted; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + let earlyError; + try { + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the cleanup batch commit'); + } catch (error) { + earlyError = error; + } finally { + releaseCleanupCommit(); + } + await dropPromise; + if (earlyError) throw earlyError; + assert.strictEqual(destructivePhaseStarted, true); + } finally { + releaseCleanupCommit(); + Transaction.prototype.commit = originalCommit; + Table.primaryStore.getEntry = originalGetEntry; + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From 5a4cc65d5a4a14707bb16708a0fc4111a86a1efe Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 16:12:41 -0600 Subject: [PATCH 16/40] fix: release all drop drain blockers on abort Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 15 ++++- resources/Table.ts | 15 +++-- .../resources/dropTableQuiescence.test.js | 62 +++++++++++++++++++ 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 2d075a91e9..8fa56d5e1d 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1243,6 +1243,17 @@ export class DatabaseTransaction implements Transaction { ); } abort(): void { + let firstError: unknown; + for (let transaction: DatabaseTransaction = this; transaction; transaction = transaction.next) { + try { + transaction.abortLink(); + } catch (error) { + firstError ??= error; + } + } + if (firstError) throw firstError; + } + private abortLink(): void { while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it // Defensively release any native handle whose reference bookkeeping was already consumed. if (this.transaction) this.releaseReadTxn(); @@ -1312,7 +1323,7 @@ export class DatabaseTransaction implements Transaction { // abort() synchronously walks savedBlobs and can call write.store.getEntry(), which can throw // (closed store, decode error). Catch and continue so one link's wrapper-cleanup failure can't // strand later links' native handles — they were already detached/aborted above regardless. - txn.abort(); + txn.abortLink(); } catch (abortError) { harperLogger.debug?.('cleaning up conflicted transaction in chain after exhausting retries', abortError); } @@ -1348,7 +1359,7 @@ export class DatabaseTransaction implements Transaction { } for (let txn: DatabaseTransaction = this; txn; txn = txn.next) { try { - txn.abort(); + txn.abortLink(); } catch (error) { harperLogger.debug?.(`Error aborting timed-out transaction in chain: ${error.message}`); } diff --git a/resources/Table.ts b/resources/Table.ts index 99d2d4e6e6..9abeb3ad36 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -5304,9 +5304,11 @@ export function makeTable(options) { let iterator: Iterator | undefined; let finishHistoryScan: () => void; finishHistoryScan = beginTableOperation('history iterator', () => { - if (!iterator?.return) return; - iterator.return(); - finishHistoryScan(); + try { + iterator?.return?.(); + } finally { + finishHistoryScan(); + } }); try { iterator = auditStore @@ -5332,8 +5334,11 @@ export function makeTable(options) { if (droppingTable) throw tableDroppingError(); } } finally { - iterator?.return?.(); - finishHistoryScan(); + try { + iterator?.return?.(); + } finally { + finishHistoryScan(); + } } } static async getHistoryOfRecord(id) { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4aefc5d34e..afb344a85b 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -15,6 +15,7 @@ const { resetDatabases, } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); +const { DatabaseTransaction } = require('#src/resources/DatabaseTransaction'); const env = require('#src/utility/environment/environmentManager'); const terms = require('#src/utility/hdbTerms'); const { ITC_EVENT_TYPES, TABLE_DROP_PREPARE_OPERATION, THREAD_TYPES } = terms; @@ -189,6 +190,26 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('aborts staged writes on linked transactions before dropping the table', async function () { + const Table = defineTable(`DropAbortedLinkedTxn_${process.pid}_${Date.now()}`); + const expectedError = new Error('abort linked transaction'); + let linkedTransaction; + + await assert.rejects( + () => + transaction(async (transactionHead) => { + linkedTransaction = transactionHead.next = new DatabaseTransaction(); + linkedTransaction.db = Table.primaryStore; + await Table.put({ id: 'aborted', name: 'not committed' }, { transaction: linkedTransaction }); + throw expectedError; + }), + expectedError + ); + assert.strictEqual(linkedTransaction.writes.length, 0, 'abort must clear every linked write set'); + assert.strictEqual(await Table.get('aborted'), null); + await Table.dropTable(); + }); + it('rejects a drop from its own read transaction before tombstoning', async function () { const tableName = `DropOwnReadTxn_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); @@ -397,6 +418,47 @@ describe('dropTable worker quiescence', function () { ); }); + it('releases a history operation token when iterator cancellation throws', async function () { + const Table = table({ + table: `DropThrowingHistoryIterator_${process.pid}_${Date.now()}`, + database: 'test', + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'history', name: 'held' }); + + const originalGetRange = Table.auditStore.getRange; + let injectedError = false; + Table.auditStore.getRange = function (...args) { + const range = originalGetRange.apply(this, args); + return { + [Symbol.iterator]() { + const iterator = range[Symbol.iterator](); + const originalReturn = iterator.return?.bind(iterator); + iterator.return = (...returnArgs) => { + const result = originalReturn?.(...returnArgs); + if (!injectedError) { + injectedError = true; + throw new Error('forced history iterator cancellation failure'); + } + return result; + }; + return iterator; + }, + }; + }; + + const history = Table.getHistory()[Symbol.asyncIterator](); + try { + assert.strictEqual((await history.next()).done, false); + await Table.dropTable(); + assert.strictEqual(injectedError, true, 'the test must exercise the throwing cancellation path'); + } finally { + Table.auditStore.getRange = originalGetRange; + await history.return?.(); + } + }); + it('omits a worker until its ITC listener is ready', async function () { const worker = startWorker(UNREADY_WORKER_FIXTURE, { name: THREAD_TYPES.JOB, From 7024115fb76b0706aac6d10e8d0db49dc56d2fef Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 16:33:45 -0600 Subject: [PATCH 17/40] fix: drain audit delete removals before table drop Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 18 ++++- .../resources/dropTableQuiescence.test.js | 65 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 9abeb3ad36..d97dd16532 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -415,6 +415,7 @@ export function makeTable(options) { let coordinatingDrop = false; let storesClosed = false; let dropPreparation: Promise | undefined; + let deleteCallbackHandle: { remove: () => void } | undefined; const tableStores = () => [...Object.values(indices), primaryStore].filter(Boolean); const tableDroppingError = () => { const error: any = new ServerError(`Table ${databaseName}.${tableName} is being dropped`, 409); @@ -450,6 +451,7 @@ export function makeTable(options) { const stopBackgroundScans = () => { stopCleanupTimer(); if (recordExpirationInterval) clearInterval(recordExpirationInterval); + deleteCallbackHandle?.remove(); }; const markTableDropping = () => { droppingTable = true; @@ -507,7 +509,6 @@ export function makeTable(options) { if (attribute.expiresAt) expiresAtProperty = attribute; if (attribute.isPrimaryKey) primaryKeyAttribute = attribute; } - let deleteCallbackHandle: { remove: () => void }; let prefetchIds = []; let prefetchCallbacks = []; let untilNextPrefetch = 1; @@ -6544,7 +6545,20 @@ export function makeTable(options) { } function addDeleteRemoval() { deleteCallbackHandle = auditStore?.addDeleteRemovalCallback(tableId, primaryStore, (id: Id, version: number) => { - primaryStore.remove(id, version); + const finishRemoval = beginTableOperation('audit delete removal'); + try { + const removal = primaryStore.remove(id, version); + if (removal?.then) { + removal.then(finishRemoval, (error) => { + finishRemoval(); + logger.warn?.(`Audit delete removal error for ${tableName}:`, error); + }); + } else finishRemoval(); + return removal; + } catch (error) { + finishRemoval(); + throw error; + } }); } function runRecordExpirationEviction() { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index afb344a85b..2e97bacea0 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -386,6 +386,71 @@ describe('dropTable worker quiescence', function () { } }); + it('drains an audit delete removal before dropping the primary store', async function () { + const storagePath = path.join(testPath, 'audit-delete-removal-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + let releaseRemoval; + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const databaseName = `DropAuditDeleteDb_${process.pid}_${Date.now()}`; + const Table = table({ + table: `DropAuditDeleteRemoval_${process.pid}_${Date.now()}`, + database: databaseName, + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'deleted', name: 'removed' }); + await Table.delete('deleted'); + + const originalRemove = Table.primaryStore.remove; + let removalStarted; + const removalStartedPromise = new Promise((resolve) => { + removalStarted = resolve; + }); + const removalGate = new Promise((resolve) => { + releaseRemoval = resolve; + }); + Table.primaryStore.remove = async function (...args) { + removalStarted(); + await removalGate; + return originalRemove.apply(this, args); + }; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const removeDeletedRecord = Table.auditStore.deleteCallbacks[Table.tableId]; + assert.strictEqual(typeof removeDeletedRecord, 'function'); + assert.strictEqual( + Table.auditStore.tableStores[Table.tableId], + Table.primaryStore, + 'the callback must belong to the table under test' + ); + const deleteRemovalPromise = removeDeletedRecord('deleted', Table.primaryStore.getEntry('deleted').version); + await removalStartedPromise; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must wait for the direct primary-store removal launched by audit pruning' + ); + releaseRemoval(); + await Promise.all([deleteRemovalPromise, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + Table.primaryStore.remove = originalRemove; + } finally { + releaseRemoval?.(); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From 178abfa717647e0fa2b852e88613e24d9df9df8d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 17:30:11 -0600 Subject: [PATCH 18/40] fix: preserve shared drop handles and removals Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 16 ++- resources/databases.ts | 2 +- .../resources/dropTableQuiescence.test.js | 98 +++++++++++++++++++ 3 files changed, 110 insertions(+), 6 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index d97dd16532..e3e60bb9fd 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -5270,8 +5270,8 @@ export function makeTable(options) { } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise { const finishHistoryScan = beginTableOperation('history deletion scan'); + const completions: Promise[] = []; try { - let completion: Promise; let entriesDeleted = 0; for (const auditRecord of auditStore.getRange({ start: 0, @@ -5280,7 +5280,7 @@ export function makeTable(options) { await rest(); // yield to other async operations if (droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; - completion = removeAuditEntry(auditStore, auditRecord); + completions.push(removeAuditEntry(auditStore, auditRecord)); entriesDeleted++; } if (cleanupDeletedRecords) { @@ -5291,14 +5291,20 @@ export function makeTable(options) { await rest(); // yield to other async operations if (droppingTable) throw tableDroppingError(); if (value === null && localTime < endTime) { - completion = removeEntry(primaryStore, entry); + const completion = removeEntry(primaryStore, entry); + if (completion) completions.push(completion); } } } - await completion; return entriesDeleted; } finally { - finishHistoryScan(); + try { + const settlements = await Promise.allSettled(completions); + const failure = settlements.find((settlement) => settlement.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + } finally { + finishHistoryScan(); + } } } static async *getHistory(startTime = 0, endTime = Infinity) { diff --git a/resources/databases.ts b/resources/databases.ts index ce629152b1..4b74f52cb2 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2445,7 +2445,7 @@ export async function prepareTableDrop( } await Promise.all( [...matchingTables].map(async (Table) => { - await Table._prepareDrop({ closeStores: Table !== preserveTable }); + await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); matchingTables.delete(Table); }) ); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 2e97bacea0..dba3d6288a 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -227,6 +227,46 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('preserves shared stores while preparing a database alias', async function () { + const tableName = `DropAliasedTable_${process.pid}_${Date.now()}`; + const databaseName = `DropAliasDb_${process.pid}_${Date.now()}`; + const aliasName = `${databaseName}_alias`; + const storePath = path.join(testPath, 'drop-alias-database'); + const dropGeneration = 'shared-store-test'; + const sharedPrimaryStore = { rootStore: { path: storePath } }; + const dbisDB = { + getSync() { + return { dropping: true, dropGeneration }; + }, + }; + let coordinatorCloseStores; + let aliasCloseStores; + const Table = { + primaryStore: sharedPrimaryStore, + dbisDB, + async _prepareDrop({ closeStores }) { + coordinatorCloseStores = closeStores; + }, + }; + const AliasTable = { + primaryStore: sharedPrimaryStore, + dbisDB, + async _prepareDrop({ closeStores }) { + aliasCloseStores = closeStores; + }, + }; + databases[databaseName] = { [tableName]: Table }; + databases[aliasName] = { [tableName]: AliasTable }; + try { + await prepareTableDrop(storePath, tableName, dropGeneration, Table); + assert.strictEqual(coordinatorCloseStores, false); + assert.strictEqual(aliasCloseStores, false, "an alias must not close the coordinator's shared stores"); + } finally { + delete databases[databaseName]; + delete databases[aliasName]; + } + }); + it('does not wait for a read iterator on another table in the same database', async function () { const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); @@ -451,6 +491,64 @@ describe('dropTable worker quiescence', function () { } }); + it('drains direct deleteHistory removals before dropping the primary store', async function () { + const storagePath = path.join(testPath, 'delete-history-removal-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + let releaseRemoval; + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + const Table = table({ + table: `DropDeleteHistoryRemoval_${process.pid}_${Date.now()}`, + database: `DropDeleteHistoryDb_${process.pid}_${Date.now()}`, + audit: true, + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + await Table.put({ id: 'deleted', name: 'removed' }); + await Table.delete('deleted'); + delete Table.auditStore.deleteCallbacks[Table.tableId]; + + const originalRemove = Table.primaryStore.remove; + let removalStarted; + const removalStartedPromise = new Promise((resolve) => { + removalStarted = resolve; + }); + const removalGate = new Promise((resolve) => { + releaseRemoval = resolve; + }); + Table.primaryStore.remove = async function (...args) { + removalStarted(); + await removalGate; + return originalRemove.apply(this, args); + }; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const deleteHistoryPromise = Table.deleteHistory(Date.now() + 1000, true); + await removalStartedPromise; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must wait for direct primary-store removals started by deleteHistory()' + ); + releaseRemoval(); + await Promise.all([deleteHistoryPromise, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + Table.primaryStore.remove = originalRemove; + } finally { + releaseRemoval?.(); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From 5f69c08f9f734762e8ae4948fc73daeff87bd088 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:05:59 -0600 Subject: [PATCH 19/40] fix: drain bounded maintenance removals Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 68 ++++++++--- .../resources/dropTableQuiescence.test.js | 110 ++++++++++++++++-- 2 files changed, 154 insertions(+), 24 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index e3e60bb9fd..6fa95db050 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -128,6 +128,7 @@ const RECORD_PRUNING_INTERVAL = 60000; // one minute // Each evict otherwise pays a full transaction commit, so batching amortizes that cost. LMDB already // coalesces async writes per event turn (eventTurnBatching), so it keeps the per-record path. const EVICTION_BATCH_SIZE = 100; +const MAX_INFLIGHT_MAINTENANCE_REMOVALS = 50; // Cap on eviction-batch commits in flight at once, so commit I/O overlaps scan/staging without // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; @@ -5270,9 +5271,26 @@ export function makeTable(options) { } static async deleteHistory(endTime = 0, cleanupDeletedRecords = false): Promise { const finishHistoryScan = beginTableOperation('history deletion scan'); - const completions: Promise[] = []; + const inFlightRemovals = new Set>(); + let removalFailure: unknown; + let removalFailed = false; + const trackRemoval = async (completion: Promise | void) => { + if (!completion || typeof completion.then !== 'function') return; + let tracked: Promise; + tracked = Promise.resolve(completion) + .then(undefined, (error) => { + if (!removalFailed) removalFailure = error; + removalFailed = true; + }) + .finally(() => inFlightRemovals.delete(tracked)); + inFlightRemovals.add(tracked); + if (inFlightRemovals.size >= MAX_INFLIGHT_MAINTENANCE_REMOVALS) await Promise.race(inFlightRemovals); + if (removalFailed) throw removalFailure; + }; + let entriesDeleted = 0; + let operationFailure: unknown; + let operationFailed = false; try { - let entriesDeleted = 0; for (const auditRecord of auditStore.getRange({ start: 0, end: endTime, @@ -5280,7 +5298,7 @@ export function makeTable(options) { await rest(); // yield to other async operations if (droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; - completions.push(removeAuditEntry(auditStore, auditRecord)); + await trackRemoval(removeAuditEntry(auditStore, auditRecord)); entriesDeleted++; } if (cleanupDeletedRecords) { @@ -5291,21 +5309,23 @@ export function makeTable(options) { await rest(); // yield to other async operations if (droppingTable) throw tableDroppingError(); if (value === null && localTime < endTime) { - const completion = removeEntry(primaryStore, entry); - if (completion) completions.push(completion); + await trackRemoval(removeEntry(primaryStore, entry)); } } } - return entriesDeleted; + } catch (error) { + operationFailure = error; + operationFailed = true; } finally { try { - const settlements = await Promise.allSettled(completions); - const failure = settlements.find((settlement) => settlement.status === 'rejected'); - if (failure?.status === 'rejected') throw failure.reason; + await Promise.all(inFlightRemovals); } finally { finishHistoryScan(); } } + if (operationFailed) throw operationFailure; + if (removalFailed) throw removalFailure; + return entriesDeleted; } static async *getHistory(startTime = 0, endTime = Infinity) { let iterator: Iterator | undefined; @@ -6577,22 +6597,38 @@ export function makeTable(options) { if (runningRecordExpiration) return; runningRecordExpiration = true; let finishExpirationScan: (() => void) | undefined; + const inFlightRemovals = new Set>(); + const trackRemoval = async (completion: Promise | void) => { + if (!completion || typeof completion.then !== 'function') return; + let tracked: Promise; + tracked = Promise.resolve(completion) + .catch((error) => { + if (!droppingTable) logger.error?.('Error removing expired index entry', error); + }) + .finally(() => inFlightRemovals.delete(tracked)); + inFlightRemovals.add(tracked); + if (inFlightRemovals.size >= MAX_INFLIGHT_MAINTENANCE_REMOVALS) await Promise.race(inFlightRemovals); + }; try { finishExpirationScan = beginTableOperation('expiration scan'); const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); - for (const key of index.getRange({ + for (const indexedEntry of index.getRange({ start: true, values: false, end: Date.now(), snapshot: false, })) { - for (const id of index.getValues(key)) { + const key = isRocksDB ? indexedEntry.key : indexedEntry; + const ids = isRocksDB ? [indexedEntry.value] : index.getValues(key); + for (const id of ids) { const recordEntry = primaryStore.getEntry(id); if (!recordEntry?.value) { // cleanup the index if the record is gone - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); + await trackRemoval( + primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)) + ); } else if (recordEntry.value[expiresAtName] < Date.now()) { // make sure the record hasn't changed and won't change while removing TableResource.evict(id, recordEntry.value, recordEntry.version); @@ -6604,8 +6640,12 @@ export function makeTable(options) { } catch (error) { if (!droppingTable) logger.error?.('Error in evicting old records', error); } finally { - finishExpirationScan?.(); - runningRecordExpiration = false; + try { + await Promise.all(inFlightRemovals); + } finally { + finishExpirationScan?.(); + runningRecordExpiration = false; + } } }, RECORD_PRUNING_INTERVAL).unref(); } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index dba3d6288a..bfc65536f2 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -491,7 +491,7 @@ describe('dropTable worker quiescence', function () { } }); - it('drains direct deleteHistory removals before dropping the primary store', async function () { + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { const storagePath = path.join(testPath, 'delete-history-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); let releaseRemoval; @@ -505,20 +505,19 @@ describe('dropTable worker quiescence', function () { audit: true, attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], }); - await Table.put({ id: 'deleted', name: 'removed' }); - await Table.delete('deleted'); + for (let id = 0; id < 51; id++) { + await Table.put({ id, name: 'removed' }); + await Table.delete(id); + } delete Table.auditStore.deleteCallbacks[Table.tableId]; const originalRemove = Table.primaryStore.remove; - let removalStarted; - const removalStartedPromise = new Promise((resolve) => { - removalStarted = resolve; - }); + let removalsStarted = 0; const removalGate = new Promise((resolve) => { releaseRemoval = resolve; }); Table.primaryStore.remove = async function (...args) { - removalStarted(); + removalsStarted++; await removalGate; return originalRemove.apply(this, args); }; @@ -530,16 +529,21 @@ describe('dropTable worker quiescence', function () { return originalDropSync.apply(this, args); }; const deleteHistoryPromise = Table.deleteHistory(Date.now() + 1000, true); - await removalStartedPromise; + await waitFor(() => removalsStarted === 50, { message: 'deleteHistory() did not fill its bounded removal window' }); const dropPromise = Table.dropTable(); for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(removalsStarted, 50, 'deleteHistory() must bound its pending removal promises'); assert.strictEqual( destructivePhaseStarted, false, 'dropTable() must wait for direct primary-store removals started by deleteHistory()' ); releaseRemoval(); - await Promise.all([deleteHistoryPromise, dropPromise]); + const [deleteHistoryResult, dropResult] = await Promise.allSettled([deleteHistoryPromise, dropPromise]); + assert.strictEqual(deleteHistoryResult.status, 'rejected'); + assert.strictEqual(deleteHistoryResult.reason.code, 'ERR_TABLE_DROPPING'); + if (dropResult.status === 'rejected') throw dropResult.reason; + assert.strictEqual(removalsStarted, 50); assert.strictEqual(destructivePhaseStarted, true); Table.primaryStore.remove = originalRemove; } finally { @@ -549,6 +553,92 @@ describe('dropTable worker quiescence', function () { } }); + it('drains an expiration index removal before dropping the table stores', async function () { + const storagePath = path.join(testPath, 'expiration-index-removal-databases'); + const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); + const originalSetInterval = global.setInterval; + const expirationCallbacks = []; + let releaseRemoval; + mkdirSync(storagePath, { recursive: true }); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); + try { + resetDatabases(); + global.setInterval = (callback, delay, ...args) => { + if (delay === 60000 && String(callback).includes('runningRecordExpiration')) { + expirationCallbacks.push(callback); + return originalSetInterval(() => {}, 0x7fffffff, ...args); + } + return originalSetInterval(callback, delay, ...args); + }; + const Table = table({ + table: `DropExpirationRemoval_${process.pid}_${Date.now()}`, + database: `DropExpirationDb_${process.pid}_${Date.now()}`, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'expiresAt', expiresAt: true, indexed: true }, + ], + }); + global.setInterval = originalSetInterval; + assert(expirationCallbacks.length > 0, 'table setup did not schedule an expiration scan'); + const expiresAt = Date.now() - 1000; + await Table.put({ id: 'orphaned-index', expiresAt }); + await Table.primaryStore.remove('orphaned-index'); + const expirationKeys = [...Table.indices.expiresAt.getRange({ start: true, values: false, end: Date.now() })]; + assert( + expirationKeys.some((entry) => entry.key === expiresAt && entry.value === 'orphaned-index'), + `missing expiration index key: ${JSON.stringify(expirationKeys)}` + ); + + const originalIfVersion = Table.primaryStore.ifVersion; + let removalStarted = false; + const removalGate = new Promise((resolve) => (releaseRemoval = resolve)); + Table.primaryStore.ifVersion = async function (...args) { + removalStarted = true; + await removalGate; + return originalIfVersion.apply(this, args); + }; + + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const expirationPromise = Promise.all(expirationCallbacks.map((callback) => callback())); + await waitFor(() => removalStarted, { timeout: 2000, message: 'expiration scan did not start the index removal' }); + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual( + destructivePhaseStarted, + false, + 'dropTable() must wait for a direct expiration-index removal' + ); + releaseRemoval(); + let expirationSettled = false; + let dropSettled = false; + expirationPromise.then( + () => (expirationSettled = true), + () => (expirationSettled = true) + ); + dropPromise.then( + () => (dropSettled = true), + () => (dropSettled = true) + ); + await waitFor(() => expirationSettled && dropSettled, { + timeout: 5000, + message: `expiration/drop did not settle (expiration=${expirationSettled}, drop=${dropSettled})`, + }); + await Promise.all([expirationPromise, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + Table.primaryStore.ifVersion = originalIfVersion; + } finally { + global.setInterval = originalSetInterval; + releaseRemoval?.(); + env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); + resetDatabases(); + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From 933d51e2608f6199969c84cdcdb54028e6043af7 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:13:45 -0600 Subject: [PATCH 20/40] fix: release eviction reads before table drain Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 48 +++------- .../resources/dropTableQuiescence.test.js | 90 ++----------------- 2 files changed, 19 insertions(+), 119 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 6fa95db050..c8d6bf01d3 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2180,15 +2180,13 @@ export function makeTable(options) { // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any).commit().catch((error) => { - // The commit failed, so the read-snapshot/transaction handle is still open — release it, as the - // batched-eviction path does on its own commit failures. committed===true skips the finally abort. - try { - (transaction as any).abort(); - } catch {} - if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); - else logger.warn?.('Error evicting record', id, error); - }); + return (transaction as any) + .commit() + .catch((error) => { + if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); + else logger.warn?.('Error evicting record', id, error); + }) + .finally(() => lmdbTransaction.releaseReadTxn()); } finally { if (!committed) { // Skip path or thrown error: abort instead of committing so we don't apply @@ -2196,7 +2194,7 @@ export function makeTable(options) { if (primaryStore.ifVersion) { (lmdbTransaction as any).abort?.(); } else { - (transaction as any)?.abort?.(); + lmdbTransaction.releaseReadTxn(); } } } @@ -6597,38 +6595,22 @@ export function makeTable(options) { if (runningRecordExpiration) return; runningRecordExpiration = true; let finishExpirationScan: (() => void) | undefined; - const inFlightRemovals = new Set>(); - const trackRemoval = async (completion: Promise | void) => { - if (!completion || typeof completion.then !== 'function') return; - let tracked: Promise; - tracked = Promise.resolve(completion) - .catch((error) => { - if (!droppingTable) logger.error?.('Error removing expired index entry', error); - }) - .finally(() => inFlightRemovals.delete(tracked)); - inFlightRemovals.add(tracked); - if (inFlightRemovals.size >= MAX_INFLIGHT_MAINTENANCE_REMOVALS) await Promise.race(inFlightRemovals); - }; try { finishExpirationScan = beginTableOperation('expiration scan'); const expiresAtName = expiresAtProperty.name; const index = indices[expiresAtName]; if (!index) throw new Error(`expiresAt attribute ${expiresAtProperty} must be indexed`); - for (const indexedEntry of index.getRange({ + for (const key of index.getRange({ start: true, values: false, end: Date.now(), snapshot: false, })) { - const key = isRocksDB ? indexedEntry.key : indexedEntry; - const ids = isRocksDB ? [indexedEntry.value] : index.getValues(key); - for (const id of ids) { + for (const id of index.getValues(key)) { const recordEntry = primaryStore.getEntry(id); if (!recordEntry?.value) { // cleanup the index if the record is gone - await trackRemoval( - primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)) - ); + primaryStore.ifVersion(id, recordEntry?.version, () => index.remove(key, id)); } else if (recordEntry.value[expiresAtName] < Date.now()) { // make sure the record hasn't changed and won't change while removing TableResource.evict(id, recordEntry.value, recordEntry.version); @@ -6640,12 +6622,8 @@ export function makeTable(options) { } catch (error) { if (!droppingTable) logger.error?.('Error in evicting old records', error); } finally { - try { - await Promise.all(inFlightRemovals); - } finally { - finishExpirationScan?.(); - runningRecordExpiration = false; - } + finishExpirationScan?.(); + runningRecordExpiration = false; } }, RECORD_PRUNING_INTERVAL).unref(); } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index bfc65536f2..941aa25d41 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -553,90 +553,12 @@ describe('dropTable worker quiescence', function () { } }); - it('drains an expiration index removal before dropping the table stores', async function () { - const storagePath = path.join(testPath, 'expiration-index-removal-databases'); - const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); - const originalSetInterval = global.setInterval; - const expirationCallbacks = []; - let releaseRemoval; - mkdirSync(storagePath, { recursive: true }); - env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, storagePath); - try { - resetDatabases(); - global.setInterval = (callback, delay, ...args) => { - if (delay === 60000 && String(callback).includes('runningRecordExpiration')) { - expirationCallbacks.push(callback); - return originalSetInterval(() => {}, 0x7fffffff, ...args); - } - return originalSetInterval(callback, delay, ...args); - }; - const Table = table({ - table: `DropExpirationRemoval_${process.pid}_${Date.now()}`, - database: `DropExpirationDb_${process.pid}_${Date.now()}`, - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'expiresAt', expiresAt: true, indexed: true }, - ], - }); - global.setInterval = originalSetInterval; - assert(expirationCallbacks.length > 0, 'table setup did not schedule an expiration scan'); - const expiresAt = Date.now() - 1000; - await Table.put({ id: 'orphaned-index', expiresAt }); - await Table.primaryStore.remove('orphaned-index'); - const expirationKeys = [...Table.indices.expiresAt.getRange({ start: true, values: false, end: Date.now() })]; - assert( - expirationKeys.some((entry) => entry.key === expiresAt && entry.value === 'orphaned-index'), - `missing expiration index key: ${JSON.stringify(expirationKeys)}` - ); - - const originalIfVersion = Table.primaryStore.ifVersion; - let removalStarted = false; - const removalGate = new Promise((resolve) => (releaseRemoval = resolve)); - Table.primaryStore.ifVersion = async function (...args) { - removalStarted = true; - await removalGate; - return originalIfVersion.apply(this, args); - }; - - const originalDropSync = Table.primaryStore.dropSync; - let destructivePhaseStarted = false; - Table.primaryStore.dropSync = function (...args) { - destructivePhaseStarted = true; - return originalDropSync.apply(this, args); - }; - const expirationPromise = Promise.all(expirationCallbacks.map((callback) => callback())); - await waitFor(() => removalStarted, { timeout: 2000, message: 'expiration scan did not start the index removal' }); - const dropPromise = Table.dropTable(); - for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); - assert.strictEqual( - destructivePhaseStarted, - false, - 'dropTable() must wait for a direct expiration-index removal' - ); - releaseRemoval(); - let expirationSettled = false; - let dropSettled = false; - expirationPromise.then( - () => (expirationSettled = true), - () => (expirationSettled = true) - ); - dropPromise.then( - () => (dropSettled = true), - () => (dropSettled = true) - ); - await waitFor(() => expirationSettled && dropSettled, { - timeout: 5000, - message: `expiration/drop did not settle (expiration=${expirationSettled}, drop=${dropSettled})`, - }); - await Promise.all([expirationPromise, dropPromise]); - assert.strictEqual(destructivePhaseStarted, true); - Table.primaryStore.ifVersion = originalIfVersion; - } finally { - global.setInterval = originalSetInterval; - releaseRemoval?.(); - env.setProperty(terms.CONFIG_PARAMS.STORAGE_PATH, previousStoragePath); - resetDatabases(); - } + it('releases an eviction read transaction before dropping the table', async function () { + const Table = defineTable(`DropAfterEvict_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); }); it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { From 2c8071a9bf309762434d331e132ceb7c615dbf17 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:19:34 -0600 Subject: [PATCH 21/40] fix: complete settled eviction transactions Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 7 +++++++ resources/Table.ts | 20 ++++++++++++------- .../resources/dropTableQuiescence.test.js | 19 ++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 8fa56d5e1d..eeade69ecf 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -561,6 +561,13 @@ export class DatabaseTransaction implements Transaction { this.completeDeferredContextRelease(); } + /** Finish wrapper bookkeeping after the caller has already settled the native handle. */ + completeReadTxn(): void { + this.detachOwnedTransaction(); + this.finishPendingReads(); + this.completeDeferredContextRelease(); + } + /** * Complete a context release that releaseContext() deferred because outstanding read iterators * were still using this transaction (see releaseContext()) — called once the last one drains, diff --git a/resources/Table.ts b/resources/Table.ts index c8d6bf01d3..de1bab3bfb 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -2180,13 +2180,19 @@ export function makeTable(options) { // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. - return (transaction as any) - .commit() - .catch((error) => { - if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); - else logger.warn?.('Error evicting record', id, error); - }) - .finally(() => lmdbTransaction.releaseReadTxn()); + const handleCommitFailure = (error) => { + lmdbTransaction.releaseReadTxn(); + if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); + else logger.warn?.('Error evicting record', id, error); + }; + let commitCompletion: Promise; + try { + commitCompletion = Promise.resolve((transaction as any).commit()); + } catch (error) { + handleCommitFailure(error); + return Promise.resolve(); + } + return commitCompletion.then(() => lmdbTransaction.completeReadTxn(), handleCommitFailure); } finally { if (!committed) { // Skip path or thrown error: abort instead of committing so we don't apply diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 941aa25d41..802c6c24ba 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -561,6 +561,25 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('releases an eviction read transaction when native commit throws synchronously', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterEvictThrow_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function () { + const error = new Error('Resource busy'); + error.code = 'ERR_BUSY'; + throw error; + }; + try { + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); + } finally { + Transaction.prototype.commit = originalCommit; + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From aa504a9af5f7d62ced3d588c9130b1ed7d1a6db2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:25:28 -0600 Subject: [PATCH 22/40] fix: release conflicted eviction transactions Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 16 +++++++++++----- unitTests/resources/dropTableQuiescence.test.js | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index de1bab3bfb..25e55a3b21 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -83,7 +83,7 @@ import { onStorageReclamation, getStorageSpaceStats } from '../server/storageRec import { RequestTarget } from './RequestTarget.ts'; import harperLogger from '../utility/logging/harper_logger.ts'; import { throttle } from '../server/throttle.ts'; -import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js'; +import { RocksDatabase, Transaction as RocksTransaction, constants as rocksConstants } from '@harperfast/rocksdb-js'; import { LMDBTransaction, ImmediateTransaction as ImmediateLMDBTransaction } from './LMDBTransaction'; import { contentTypes } from '../server/serverHelpers/contentTypes'; import { type JsonSchemaFragment, projectAttributesToProperties } from './jsonSchemaTypes.ts'; @@ -2177,9 +2177,8 @@ export function makeTable(options) { }); } // RocksDB: eviction writes went directly into the raw transaction via options; commit it directly, - // as DatabaseTransaction.commit() would abort it (no tracked writes). The raw commit bypasses - // DatabaseTransaction's ERR_BUSY retry, so a concurrent-write conflict rejects here — swallow it - // (abandon the eviction) and log anything unexpected, rather than letting it crash the process. + // as DatabaseTransaction.commit() would abort it (no tracked writes). A coordinated-retry conflict + // resolves with RETRY_NOW_VALUE; abandon that eviction and release its native transaction. const handleCommitFailure = (error) => { lmdbTransaction.releaseReadTxn(); if (error?.code === 'ERR_BUSY') logger.trace?.('Abandoned eviction of busy record', id); @@ -2192,7 +2191,14 @@ export function makeTable(options) { handleCommitFailure(error); return Promise.resolve(); } - return commitCompletion.then(() => lmdbTransaction.completeReadTxn(), handleCommitFailure); + return commitCompletion.then((result) => { + if (result === rocksConstants.RETRY_NOW_VALUE) { + lmdbTransaction.releaseReadTxn(); + logger.trace?.('Abandoned eviction of busy record', id); + return; + } + lmdbTransaction.completeReadTxn(); + }, handleCommitFailure); } finally { if (!committed) { // Skip path or thrown error: abort instead of committing so we don't apply diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 802c6c24ba..a51c285156 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -580,6 +580,21 @@ describe('dropTable worker quiescence', function () { } }); + it('releases an eviction read transaction on a coordinated-retry conflict', async function () { + const { Transaction, constants } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterEvictRetry_${process.pid}_${Date.now()}`); + await Table.put('expired', { name: 'evicted' }); + const entry = Table.primaryStore.getEntry('expired'); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = () => Promise.resolve(constants.RETRY_NOW_VALUE); + try { + await Table.evict('expired', entry.value, entry.version); + await Table.dropTable(); + } finally { + Transaction.prototype.commit = originalCommit; + } + }); + it('cancels a subscriber-paced replay instead of timing out the drop drain', async function () { const Table = defineTable(`DropSlowReplay_${process.pid}_${Date.now()}`); for (let i = 0; i < 110; i++) await Table.put(i, { name: `queued-${i}` }); From abc7517ef11a3be331f2c8d29f82bf975012ef4c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:48:07 -0600 Subject: [PATCH 23/40] fix: drain allocation and commit failure paths Co-Authored-By: GPT-5 Codex --- resources/Table.ts | 18 +++++- resources/transaction.ts | 18 ++++-- .../resources/dropTableQuiescence.test.js | 60 +++++++++++++++++++ 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 25e55a3b21..b6119e54ce 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1249,7 +1249,7 @@ export function makeTable(options) { return; } logger.info?.('New id allocation', nextId, idIncrementer.maxSafeId, version); - primaryStore.put( + return primaryStore.put( Symbol.for('id_allocation'), { start: updatedIdAllocation.start, @@ -1273,7 +1273,21 @@ export function makeTable(options) { } }; if (nextId + asyncIdExpansionThreshold === idIncrementer.maxSafeId) { - setImmediate(updateEnd); // if we are getting kind of close to the end, we try to update it asynchronously + const finishIdAllocationUpdate = beginTableOperation('id allocation update'); + setImmediate(() => { + try { + const completion = updateEnd(false); + if (completion?.then) { + completion.then(finishIdAllocationUpdate, (error) => { + finishIdAllocationUpdate(); + if (!droppingTable) logger.warn?.(`Error updating id allocation for ${tableName}`, error); + }); + } else finishIdAllocationUpdate(); + } catch (error) { + finishIdAllocationUpdate(); + if (!droppingTable) logger.warn?.(`Error updating id allocation for ${tableName}`, error); + } + }); } else if (nextId + 100 >= idIncrementer.maxSafeId) { logger.warn?.( `Synchronous id allocation required on table ${tableName}${ diff --git a/resources/transaction.ts b/resources/transaction.ts index b2dd2abcdf..3a65492833 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -66,15 +66,21 @@ export function transaction( return onComplete(result); // when the transaction function completes, run this to commit the transaction function onComplete(result) { - const committed = transaction.commit({ doneWriting: true }); - if ((committed as any).then) { - return (committed as any).then(() => { + try { + const committed = transaction.commit({ doneWriting: true }); + if ((committed as any).then) { + return (committed as any).then(() => result, onCommitError); + } else { return result; - }); - } else { - return result; + } + } catch (error) { + return onCommitError(error); } } + function onCommitError(error) { + transaction.abort(); + throw error; + } // if the transaction function throws an error, we abort function onError(error) { transaction.abort(); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index a51c285156..c9302a7a0f 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -491,6 +491,66 @@ describe('dropTable worker quiescence', function () { } }); + it('drains an asynchronous id-allocation update before dropping the primary store', async function () { + const Table = table({ + table: `DropIdAllocation_${process.pid}_${Date.now()}`, + database: 'test', + attributes: [{ name: 'id', type: 'Int', isPrimaryKey: true }], + }); + Table.getNewId(); + const originalPut = Table.primaryStore.put; + let allocationWriteStarted; + const allocationWriteStartedPromise = new Promise((resolve) => (allocationWriteStarted = resolve)); + let releaseAllocationWrite; + const allocationWriteGate = new Promise((resolve) => (releaseAllocationWrite = resolve)); + Table.primaryStore.put = async function (key, ...args) { + if (key === Symbol.for('id_allocation')) { + allocationWriteStarted(); + await allocationWriteGate; + } + return originalPut.call(this, key, ...args); + }; + try { + for (let count = 1; count < 512; count++) Table.getNewId(); + await allocationWriteStartedPromise; + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the id-allocation write'); + releaseAllocationWrite(); + await dropPromise; + assert.strictEqual(destructivePhaseStarted, true); + } finally { + releaseAllocationWrite(); + Table.primaryStore.put = originalPut; + } + }); + + it('aborts write tracking when the transaction wrapper commit throws synchronously', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterCommitThrow_${process.pid}_${Date.now()}`); + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function () { + throw new Error('forced synchronous commit failure'); + }; + try { + await assert.rejects( + transaction(async () => { + await Table.put('held', { name: 'pending' }); + }), + /forced synchronous commit failure/ + ); + } finally { + Transaction.prototype.commit = originalCommit; + } + await Table.dropTable(); + }); + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { const storagePath = path.join(testPath, 'delete-history-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); From 603afaf20930b64bf2e30844f6c6d5aa079e9956 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:59:04 -0600 Subject: [PATCH 24/40] fix: guarantee abort tracking cleanup Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 28 +++++++------------ resources/transaction.ts | 6 ++-- .../resources/dropTableQuiescence.test.js | 16 +++++++++++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index eeade69ecf..9e77471a16 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1263,33 +1263,25 @@ export class DatabaseTransaction implements Transaction { private abortLink(): void { while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it // Defensively release any native handle whose reference bookkeeping was already consumed. - if (this.transaction) this.releaseReadTxn(); - this.open = TRANSACTION_STATE.CLOSED; + if (this.transaction) this.releaseReadTxn(); + this.open = TRANSACTION_STATE.CLOSED; + try { this.drainCompletions(); - try { for (const write of this.writes) { if (write?.savedBlobs) cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value)); - } - } finally { + } + } finally { + // reset the transaction even if blob inspection fails this.clearWrites(); // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free": // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of // starting a fresh one) so the rest of the logical operation fails atomically via the // poison check in addWrite()/commit(), rather than silently landing a later write on a - // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would - // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. - this.releaseContext(!this.timedOut); - const next = this.next; - this.next = null; - if (next) { - try { - next.abort(); - } catch (error) { - harperLogger.debug?.('cleaning up a chained transaction during abort', error); - } - } - } + // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would + // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. + this.releaseContext(!this.timedOut); + } } /** * Give up on a chain of linked transactions after exhausting conflict retries: poison every link diff --git a/resources/transaction.ts b/resources/transaction.ts index 3a65492833..220f4f7e3f 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -69,7 +69,7 @@ export function transaction( try { const committed = transaction.commit({ doneWriting: true }); if ((committed as any).then) { - return (committed as any).then(() => result, onCommitError); + return (committed as any).then(() => result); } else { return result; } @@ -78,7 +78,9 @@ export function transaction( } } function onCommitError(error) { - transaction.abort(); + try { + transaction.abort(); + } catch {} throw error; } // if the transaction function throws an error, we abort diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index c9302a7a0f..a0cd974aa4 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -551,6 +551,22 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('clears write tracking when abort blob inspection throws', async function () { + const Table = defineTable(`DropAfterAbortCleanupThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.addWrite({ store: Table.primaryStore, key: 'held', savedBlobs: [], deferSave: true }); + const originalGetEntry = Table.primaryStore.getEntry; + Table.primaryStore.getEntry = () => { + throw new Error('forced blob inspection failure'); + }; + try { + assert.throws(() => txn.abort(), /forced blob inspection failure/); + } finally { + Table.primaryStore.getEntry = originalGetEntry; + } + await Table.dropTable(); + }); + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { const storagePath = path.join(testPath, 'delete-history-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); From 8461c4f98f39831b9c842c553f67f3fb6615e0fd Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 19:07:16 -0600 Subject: [PATCH 25/40] test: format drop quiescence coverage Co-Authored-By: GPT-5 Codex --- unitTests/resources/dropTableQuiescence.test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index a0cd974aa4..1111bfbfff 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -605,7 +605,9 @@ describe('dropTable worker quiescence', function () { return originalDropSync.apply(this, args); }; const deleteHistoryPromise = Table.deleteHistory(Date.now() + 1000, true); - await waitFor(() => removalsStarted === 50, { message: 'deleteHistory() did not fill its bounded removal window' }); + await waitFor(() => removalsStarted === 50, { + message: 'deleteHistory() did not fill its bounded removal window', + }); const dropPromise = Table.dropTable(); for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); assert.strictEqual(removalsStarted, 50, 'deleteHistory() must bound its pending removal promises'); From fd926473a34cf8b2329a3ad62f98265221cefe4f Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 19:25:12 -0600 Subject: [PATCH 26/40] fix: abort synchronous commit failures Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 25 +++++++++++++++++-- .../resources/dropTableQuiescence.test.js | 8 +++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 9e77471a16..d181728813 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -985,7 +985,11 @@ export class DatabaseTransaction implements Transaction { } // with options.transaction set this is a retry round — the save loop above already // re-staged the writes into it - commitResolution = transaction.commit() as Promise; + try { + commitResolution = transaction.commit() as Promise; + } catch (error) { + this.abortSynchronousCommit(transaction, error); + } recordCommitLatency(commitResolution, performance.now()); // Write-queue-depth accounting for this replay commit happens uniformly below, via // trackOutstandingCommit(commitResolution) — see that function's comment. Omitting @@ -1021,7 +1025,11 @@ export class DatabaseTransaction implements Transaction { // getReadTxn), so commit() can resolve to RETRY_NOW_VALUE. That // sentinel (a number) is why commitResolution is typed // Promise; it is handled in the resolve callback below. - commitResolution = transaction.commit(); + try { + commitResolution = transaction.commit(); + } catch (error) { + this.abortSynchronousCommit(transaction, error); + } // Record how long this commit stays outstanding (submit → settle) as a distribution // metric. This is the same clock the overload check uses (trackOutstandingCommit // stamps each attempt at submit), so a rising p99/p999 is the leading indicator for the @@ -1283,6 +1291,19 @@ export class DatabaseTransaction implements Transaction { this.releaseContext(!this.timedOut); } } + private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { + try { + transaction.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting native transaction after synchronous commit failure', abortError); + } + try { + this.abort(); + } catch (abortError) { + harperLogger.debug?.('cleaning up transaction after synchronous commit failure', abortError); + } + throw error; + } /** * Give up on a chain of linked transactions after exhausting conflict retries: poison every link * first, then abort each link's native transaction and release its DatabaseTransaction-level diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 1111bfbfff..b469c736a4 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -531,9 +531,10 @@ describe('dropTable worker quiescence', function () { } }); - it('aborts write tracking when the transaction wrapper commit throws synchronously', async function () { + it('aborts write tracking and the native snapshot when commit throws synchronously', async function () { const { Transaction } = require('@harperfast/rocksdb-js'); const Table = defineTable(`DropAfterCommitThrow_${process.pid}_${Date.now()}`); + assert.strictEqual(Table.primaryStore.getOldestSnapshotTimestamp(), 0); const originalCommit = Transaction.prototype.commit; Transaction.prototype.commit = function () { throw new Error('forced synchronous commit failure'); @@ -548,6 +549,11 @@ describe('dropTable worker quiescence', function () { } finally { Transaction.prototype.commit = originalCommit; } + assert.strictEqual( + Table.primaryStore.getOldestSnapshotTimestamp(), + 0, + 'a synchronous commit failure must not leave the detached native snapshot open' + ); await Table.dropTable(); }); From 226f1802a99741af8d8651d8847fd02b4823c99b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 19:32:51 -0600 Subject: [PATCH 27/40] fix: preserve iterators after replay commit failure Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 65 +++++++++++++++---- .../resources/dropTableQuiescence.test.js | 22 +++++++ .../resources/lingeringWriteCommit.test.js | 29 +++++++++ 3 files changed, 103 insertions(+), 13 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index d181728813..7239ae0148 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -988,7 +988,7 @@ export class DatabaseTransaction implements Transaction { try { commitResolution = transaction.commit() as Promise; } catch (error) { - this.abortSynchronousCommit(transaction, error); + this.abortSynchronousReplayCommit(transaction, error, !!options.doneWriting); } recordCommitLatency(commitResolution, performance.now()); // Write-queue-depth accounting for this replay commit happens uniformly below, via @@ -1269,27 +1269,66 @@ export class DatabaseTransaction implements Transaction { if (firstError) throw firstError; } private abortLink(): void { - while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it - // Defensively release any native handle whose reference bookkeeping was already consumed. - if (this.transaction) this.releaseReadTxn(); - this.open = TRANSACTION_STATE.CLOSED; - try { - this.drainCompletions(); + this.open = TRANSACTION_STATE.CLOSED; + try { + while (this.readTxnsUsed > 0) this.doneReadTxn(); // release the read snapshot when we abort, we assume we don't need it + // A write-only transaction never took a read reference (getReadTxn was never called), so the loop + // above releases nothing even though save() created a native handle; release it here instead of + // leaking the handle and its snapshot until GC. abortChainAfterRetries() detaches the handle + // before calling abort(), so this is a no-op there rather than a double-abort. + if (this.transaction) this.releaseReadTxn(); + this.drainCompletions(); for (const write of this.writes) { if (write?.savedBlobs) cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value)); - } - } finally { - // reset the transaction even if blob inspection fails + } + } finally { + // reset the transaction even if blob inspection fails this.clearWrites(); // A timeout-poisoned abort (abortDueToTimeout()) is the one abort that is NOT "reuse-free": // Resource.ts's dispatcher deliberately keeps joining a `timedOut` transaction (instead of // starting a fresh one) so the rest of the logical operation fails atomically via the // poison check in addWrite()/commit(), rather than silently landing a later write on a - // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would - // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. - this.releaseContext(!this.timedOut); + // brand-new transaction after an earlier one was rolled back (#1411). Releasing here would + // make that check see `undefined?.timedOut` and take the "start fresh" branch instead. + this.releaseContext(!this.timedOut); + } } + private abortSynchronousReplayCommit(transaction: RocksTransaction, error: unknown, final: boolean): never { + try { + transaction.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting replay transaction after synchronous commit failure', abortError); + } + if (!this.writesAbandoned) { + this.writesAbandoned = true; + try { + this.transaction?.abandonWrites?.(); + } catch (abandonError) { + harperLogger.debug?.('abandoning retained writes after synchronous replay commit failure', abandonError); + } + } + try { + for (const write of this.writes) { + if (write?.savedBlobs) + cleanupUnusedBlobs(write.savedBlobs, collectRetainedFileIds(write.store.getEntry(write.key)?.value)); + } + } catch (cleanupError) { + harperLogger.debug?.('cleaning up blobs after synchronous replay commit failure', cleanupError); + } finally { + this.clearWrites(); + this.releaseContext(final); + } + const nextTransaction = this.next; + this.next = null; + for (let linkedTransaction = nextTransaction; linkedTransaction; linkedTransaction = linkedTransaction.next) { + try { + linkedTransaction.abortLink(); + } catch (abortError) { + harperLogger.debug?.('aborting linked transaction after synchronous replay commit failure', abortError); + } + } + throw error; } private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { try { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index b469c736a4..4ec976316c 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -573,6 +573,28 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('clears write tracking when native read abort throws', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterNativeAbortThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.db = Table.primaryStore; + const nativeTransaction = txn.getReadTxn(); + txn.addWrite({ store: Table.primaryStore, key: 'held', deferSave: true }); + const originalAbort = Transaction.prototype.abort; + Transaction.prototype.abort = function (...args) { + if (this === nativeTransaction) throw new Error('forced native abort failure'); + return originalAbort.apply(this, args); + }; + try { + assert.throws(() => txn.abort(), /forced native abort failure/); + assert.strictEqual(txn.writes.length, 0, 'native abort failure must still clear the tracked writes'); + } finally { + Transaction.prototype.abort = originalAbort; + txn.releaseReadTxn(); + } + await Table.dropTable(); + }); + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { const storagePath = path.join(testPath, 'delete-history-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index e75b8ba6cc..f9fd9b0a53 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -160,4 +160,33 @@ describe('commit with open read iterators commits writes immediately on a replay await delay(100); assert.deepEqual(unhandled, [], 'a replay-commit failure must reject the awaited chain, never float unhandled'); }); + + it('a synchronous replay commit failure leaves the retained iterator usable', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const originalCommit = Transaction.prototype.commit; + const targetDb = LingerTable.primaryStore.store.db; + const context = {}; + let failedTransaction; + await transaction(context, async () => { + const results = await LingerTable.search({ conditions: [] }, context); + const iterator = results[Symbol.asyncIterator](); + await iterator.next(); + await LingerTable.put({ id: 'linger-sync-fail', v: 42 }, context); + Transaction.prototype.commit = function (...args) { + if (this.store?.db !== targetDb) return originalCommit.apply(this, args); + throw new Error('forced synchronous replay failure'); + }; + try { + assert.throws(() => context.transaction.commit(), /forced synchronous replay failure/); + } finally { + Transaction.prototype.commit = originalCommit; + } + failedTransaction = context.transaction; + assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); + assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); + while (!(await iterator.next()).done); + assert.strictEqual(failedTransaction.transaction, null, 'draining the iterator must release its read handle'); + }); + assert.equal(await LingerTable.get('linger-sync-fail'), null, 'the failed replay must not commit its record'); + }); }); From 9bba698a9108d8bf1eb4b272551da4b1c270ed6c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 20:04:18 -0600 Subject: [PATCH 28/40] fix: release audit table handles on drop Co-Authored-By: GPT-5 Codex --- resources/auditStore.ts | 3 ++- unitTests/resources/dropTableQuiescence.test.js | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/resources/auditStore.ts b/resources/auditStore.ts index b57b88665e..9ecd13d13c 100644 --- a/resources/auditStore.ts +++ b/resources/auditStore.ts @@ -153,7 +153,8 @@ export function openAuditStore(rootStore) { auditStore.deleteCallbacks = deleteCallbacks; return { remove() { - delete deleteCallbacks[tableId]; + if (deleteCallbacks[tableId] === callback) delete deleteCallbacks[tableId]; + if (auditStore.tableStores[tableId] === table) delete auditStore.tableStores[tableId]; }, }; }; diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4ec976316c..bac3a8f453 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -483,6 +483,16 @@ describe('dropTable worker quiescence', function () { releaseRemoval(); await Promise.all([deleteRemovalPromise, dropPromise]); assert.strictEqual(destructivePhaseStarted, true); + assert.strictEqual( + Table.auditStore.deleteCallbacks[Table.tableId], + undefined, + 'a dropped table must unregister its audit delete callback' + ); + assert.strictEqual( + Table.auditStore.tableStores[Table.tableId], + undefined, + 'audit retention must not retain the dropped table store' + ); Table.primaryStore.remove = originalRemove; } finally { releaseRemoval?.(); From b222e17113d25625c7efe7c84c98e4dd21021315 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 20:06:42 -0600 Subject: [PATCH 29/40] fix: preserve replay iterators through wrapper failures Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 9 ++-- .../resources/lingeringWriteCommit.test.js | 46 +++++++++++-------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 7239ae0148..0cddb103b0 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -988,7 +988,7 @@ export class DatabaseTransaction implements Transaction { try { commitResolution = transaction.commit() as Promise; } catch (error) { - this.abortSynchronousReplayCommit(transaction, error, !!options.doneWriting); + commitResolution = this.abortSynchronousReplayCommit(transaction, error, !!options.doneWriting); } recordCommitLatency(commitResolution, performance.now()); // Write-queue-depth accounting for this replay commit happens uniformly below, via @@ -1294,7 +1294,7 @@ export class DatabaseTransaction implements Transaction { this.releaseContext(!this.timedOut); } } - private abortSynchronousReplayCommit(transaction: RocksTransaction, error: unknown, final: boolean): never { + private abortSynchronousReplayCommit(transaction: RocksTransaction, error: unknown, final: boolean): Promise { try { transaction.abort(); } catch (abortError) { @@ -1328,7 +1328,10 @@ export class DatabaseTransaction implements Transaction { harperLogger.debug?.('aborting linked transaction after synchronous replay commit failure', abortError); } } - throw error; + // Normalize the native API's unexpected synchronous throw to its ordinary rejected-Promise + // contract. The transaction() wrapper only aborts on a thrown commit; returning a rejection keeps + // the retained read handle alive until its iterators drain, matching the normal failure path. + return Promise.reject(error); } private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { try { diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index f9fd9b0a53..17be4e129f 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -167,26 +167,32 @@ describe('commit with open read iterators commits writes immediately on a replay const targetDb = LingerTable.primaryStore.store.db; const context = {}; let failedTransaction; - await transaction(context, async () => { - const results = await LingerTable.search({ conditions: [] }, context); - const iterator = results[Symbol.asyncIterator](); - await iterator.next(); - await LingerTable.put({ id: 'linger-sync-fail', v: 42 }, context); - Transaction.prototype.commit = function (...args) { - if (this.store?.db !== targetDb) return originalCommit.apply(this, args); - throw new Error('forced synchronous replay failure'); - }; - try { - assert.throws(() => context.transaction.commit(), /forced synchronous replay failure/); - } finally { - Transaction.prototype.commit = originalCommit; - } - failedTransaction = context.transaction; - assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); - assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); - while (!(await iterator.next()).done); - assert.strictEqual(failedTransaction.transaction, null, 'draining the iterator must release its read handle'); - }); + let iterator; + let rejection; + try { + await transaction(context, async () => { + const results = await LingerTable.search({ conditions: [] }, context); + iterator = results[Symbol.asyncIterator](); + await iterator.next(); + await LingerTable.put({ id: 'linger-sync-fail', v: 42 }, context); + failedTransaction = context.transaction; + Transaction.prototype.commit = function (...args) { + if (this.store?.db !== targetDb) return originalCommit.apply(this, args); + throw new Error('forced synchronous replay failure'); + }; + }); + } catch (error) { + rejection = error; + } finally { + Transaction.prototype.commit = originalCommit; + } + assert.match(rejection?.message, /forced synchronous replay failure/); + assert.strictEqual(context.transaction, failedTransaction, 'context release must wait for the retained iterator'); + assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); + assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); + while (!(await iterator.next()).done); + assert.strictEqual(failedTransaction.transaction, null, 'draining the iterator must release its read handle'); + assert.strictEqual(context.transaction, null, 'draining the iterator must release the context back-reference'); assert.equal(await LingerTable.get('linger-sync-fail'), null, 'the failed replay must not commit its record'); }); }); From 90ec145512139e3c595dbef290748a88c11bacd6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 20:26:11 -0600 Subject: [PATCH 30/40] fix: drain indexing and failed transaction chains Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 27 ++++++- resources/Table.ts | 2 + .../resources/dropTableQuiescence.test.js | 77 +++++++++++++++++++ 3 files changed, 105 insertions(+), 1 deletion(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 0cddb103b0..c5d9c8ac48 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1215,7 +1215,32 @@ export class DatabaseTransaction implements Transaction { } catch (abortError) { harperLogger.debug?.('aborting transaction after failed commit', abortError); } - this.finishPendingWrites(); + const nextTransaction = this.next; + this.next = null; + for ( + let linkedTransaction = nextTransaction; + linkedTransaction; + linkedTransaction = linkedTransaction.next + ) { + try { + linkedTransaction.abortLink(); + } catch (abortError) { + harperLogger.debug?.('aborting linked transaction after failed commit', abortError); + } + } + try { + for (const write of this.writes) { + if (write?.savedBlobs) + cleanupUnusedBlobs( + write.savedBlobs, + collectRetainedFileIds(write.store.getEntry(write.key)?.value) + ); + } + } catch (cleanupError) { + harperLogger.debug?.('cleaning up writes after failed commit', cleanupError); + } finally { + this.clearWrites(); + } // A terminal failure is just as final as a success — release the context's // back-reference here too, or transaction.ts's onComplete() (which has no // rejection handler of its own) would leave a long-lived context pinning this diff --git a/resources/Table.ts b/resources/Table.ts index b6119e54ce..35dc9a0216 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -477,6 +477,8 @@ export function makeTable(options) { ...getPendingWriteResolutions(tableStores()), ...getPendingReadResolutions(tableStores()), ]); + const indexingOperation = (TableResource as any).indexingOperation; + if (indexingOperation) pending.add(indexingOperation); if (!pending.size) return; let timer: NodeJS.Timeout; const timedOut = Symbol('timedOut'); diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index bac3a8f453..4bad414633 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -210,6 +210,35 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('aborts linked writes when the head commit rejects terminally', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const HeadTable = defineTable(`DropTerminalHead_${process.pid}_${Date.now()}`); + const LinkedTable = defineTable(`DropTerminalLinked_${process.pid}_${Date.now()}`); + const headTransaction = new DatabaseTransaction(); + headTransaction.db = HeadTable.primaryStore; + const linkedTransaction = (headTransaction.next = new DatabaseTransaction()); + linkedTransaction.db = LinkedTable.primaryStore; + await HeadTable.put({ id: 'head', name: 'pending' }, { transaction: headTransaction }); + await LinkedTable.put({ id: 'linked', name: 'pending' }, { transaction: linkedTransaction }); + const headNativeTransaction = headTransaction.transaction; + const originalCommit = Transaction.prototype.commit; + Transaction.prototype.commit = function (...args) { + if (this === headNativeTransaction) + return Promise.reject(Object.assign(new Error('forced terminal commit failure'), { code: 'ERR_CORRUPTION' })); + return originalCommit.apply(this, args); + }; + try { + await assert.rejects(() => headTransaction.commit({ doneWriting: true }), /forced terminal commit failure/); + } finally { + Transaction.prototype.commit = originalCommit; + } + assert.strictEqual(headTransaction.writes.length, 0, 'the failed head must clear its own write set'); + assert.strictEqual(headTransaction.next, null, 'the failed head must detach its linked transaction chain'); + assert.strictEqual(linkedTransaction.writes.length, 0, 'the failed head must abort every linked write set'); + assert.strictEqual(linkedTransaction.transaction, null, 'the failed head must release every linked native handle'); + await Promise.all([HeadTable.dropTable(), LinkedTable.dropTable()]); + }); + it('rejects a drop from its own read transaction before tombstoning', async function () { const tableName = `DropOwnReadTxn_${process.pid}_${Date.now()}`; const Table = defineTable(tableName); @@ -426,6 +455,54 @@ describe('dropTable worker quiescence', function () { } }); + it('drains an active index backfill before dropping the table stores', async function () { + const tableName = `DropIndexBackfill_${process.pid}_${Date.now()}`; + let Table = table({ + table: tableName, + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'name' }], + }); + for (let id = 0; id < 20; id++) await Table.put({ id, name: `name-${id}` }); + Table = table({ + table: tableName, + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name', indexed: true }, + ], + }); + assert.ok(Table.indexingOperation, 'adding an index to existing records must start a backfill'); + const index = Table.indices.name; + const originalPut = index.put; + let indexingWriteStarted; + const indexingWriteStartedPromise = new Promise((resolve) => (indexingWriteStarted = resolve)); + let releaseIndexingWrite; + const indexingWriteGate = new Promise((resolve) => (releaseIndexingWrite = resolve)); + index.put = async function (...args) { + indexingWriteStarted(); + await indexingWriteGate; + return originalPut.apply(this, args); + }; + try { + await indexingWriteStartedPromise; + const originalDropSync = Table.primaryStore.dropSync; + let destructivePhaseStarted = false; + Table.primaryStore.dropSync = function (...args) { + destructivePhaseStarted = true; + return originalDropSync.apply(this, args); + }; + const dropPromise = Table.dropTable(); + for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the index backfill'); + releaseIndexingWrite(); + await Promise.all([Table.indexingOperation, dropPromise]); + assert.strictEqual(destructivePhaseStarted, true); + } finally { + releaseIndexingWrite(); + index.put = originalPut; + } + }); + it('drains an audit delete removal before dropping the primary store', async function () { const storagePath = path.join(testPath, 'audit-delete-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); From 34f447f787dd838312e0eec2b05f410be3439321 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 20:55:47 -0600 Subject: [PATCH 31/40] fix: close terminal drop recovery gaps Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 11 +++++--- resources/Table.ts | 7 +++-- resources/databases.ts | 15 +++++++--- resources/transaction.ts | 4 ++- .../resources/dropTableQuiescence.test.js | 28 +++++++++++++++++++ .../resources/lingeringWriteCommit.test.js | 2 +- unitTests/resources/transaction.test.js | 20 +++++++++++++ 7 files changed, 74 insertions(+), 13 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index c5d9c8ac48..382fdc13ac 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1353,10 +1353,13 @@ export class DatabaseTransaction implements Transaction { harperLogger.debug?.('aborting linked transaction after synchronous replay commit failure', abortError); } } - // Normalize the native API's unexpected synchronous throw to its ordinary rejected-Promise - // contract. The transaction() wrapper only aborts on a thrown commit; returning a rejection keeps - // the retained read handle alive until its iterators drain, matching the normal failure path. - return Promise.reject(error); + // Keep the retained read handle alive until its iterators drain, but do not let a retryable native + // code re-enter commit after the staged writes above were discarded. + return Promise.reject( + Object.assign(new Error(error instanceof Error ? error.message : String(error), { cause: error }), { + name: error instanceof Error ? error.name : 'Error', + }) + ); } private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { try { diff --git a/resources/Table.ts b/resources/Table.ts index 35dc9a0216..2ebe8aa3af 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -494,7 +494,7 @@ export function makeTable(options) { .filter((operation) => pendingTableOperations.has(operation)) .map(({ label }) => label); throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop tombstone is durable, so recovery can retry after a clean restart.` + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop request is durable: retry drop_table after the blocking operations settle, or restart Harper to complete the drop.` ); } }; @@ -1251,7 +1251,7 @@ export function makeTable(options) { return; } logger.info?.('New id allocation', nextId, idIncrementer.maxSafeId, version); - return primaryStore.put( + const completion = primaryStore.put( Symbol.for('id_allocation'), { start: updatedIdAllocation.start, @@ -1262,6 +1262,7 @@ export function makeTable(options) { Date.now(), version ); + return inTxn ? undefined : completion; } else { // indicate that we have run out of ids in the allocated range, so we need to allocate a new range logger.warn?.( @@ -1541,7 +1542,7 @@ export function makeTable(options) { }); } catch (error) { const quiescenceError: any = new ServerError( - `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable but its storage was not dropped. Restart Harper before retrying. ${error.message}`, + `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable and this durable drop will complete when Harper restarts. Retry drop_table in this process only after the blocking worker settles. ${error.message}`, 503 ); quiescenceError.code = error.code; diff --git a/resources/databases.ts b/resources/databases.ts index 4b74f52cb2..d86f14888b 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2443,13 +2443,20 @@ export async function prepareTableDrop( } matchingTables.add(Table); } - await Promise.all( + const preparations = await Promise.allSettled( [...matchingTables].map(async (Table) => { - await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); - matchingTables.delete(Table); + try { + await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); + } finally { + matchingTables.delete(Table); + } }) ); - if (!matchingTables.size) incompleteTableDropPreparations.delete(preparationKey); + incompleteTableDropPreparations.delete(preparationKey); + const failedPreparation = preparations.find( + (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' + ); + if (failedPreparation) throw failedPreparation.reason; } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/resources/transaction.ts b/resources/transaction.ts index 220f4f7e3f..d1264edac8 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -85,7 +85,9 @@ export function transaction( } // if the transaction function throws an error, we abort function onError(error) { - transaction.abort(); + try { + transaction.abort(); + } catch {} throw error; } } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4bad414633..bc5f89561b 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -296,6 +296,34 @@ describe('dropTable worker quiescence', function () { } }); + it('releases a failed preparation before a later retry', async function () { + const tableName = `DropFailedPreparation_${process.pid}_${Date.now()}`; + const databaseName = `DropFailedPreparationDb_${process.pid}_${Date.now()}`; + const storePath = path.join(testPath, 'drop-failed-preparation-database'); + const dropGeneration = 'failed-preparation-test'; + let attempts = 0; + const Table = { + primaryStore: { rootStore: { path: storePath } }, + dbisDB: { + getSync() { + return { dropping: true, dropGeneration }; + }, + }, + async _prepareDrop() { + attempts++; + throw new Error('forced preparation failure'); + }, + }; + databases[databaseName] = { [tableName]: Table }; + try { + await assert.rejects(prepareTableDrop(storePath, tableName, dropGeneration), /forced preparation failure/); + } finally { + delete databases[databaseName]; + } + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(attempts, 1, 'the failed table must not remain registered for later preparations'); + }); + it('does not wait for a read iterator on another table in the same database', async function () { const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index 17be4e129f..02f41e2a22 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -178,7 +178,7 @@ describe('commit with open read iterators commits writes immediately on a replay failedTransaction = context.transaction; Transaction.prototype.commit = function (...args) { if (this.store?.db !== targetDb) return originalCommit.apply(this, args); - throw new Error('forced synchronous replay failure'); + throw Object.assign(new Error('forced synchronous replay failure'), { code: 'ERR_BUSY' }); }; }); } catch (error) { diff --git a/unitTests/resources/transaction.test.js b/unitTests/resources/transaction.test.js index 38dd5024ca..613d611be1 100644 --- a/unitTests/resources/transaction.test.js +++ b/unitTests/resources/transaction.test.js @@ -189,6 +189,26 @@ describe('Transactions', () => { await committed; assert.deepEqual(order, ['completion', 'commit'], 'commit resolved only after the callback completion'); }); + it('preserves the application error when abort cleanup also fails', function () { + const applicationError = new Error('application failure'); + assert.throws( + () => + transaction({}, (txn) => { + txn.addWrite({ + store: { + getEntry() { + throw new Error('abort cleanup failure'); + }, + }, + key: 'pending', + savedBlobs: [], + deferSave: true, + }); + throw applicationError; + }), + (error) => error === applicationError + ); + }); it('Can run txn with three tables and two databases', async function () { const context = {}; let start = Date.now(); From 9fd985c65950b6a39f2aa125a878563d3b4c1354 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 21:26:33 -0600 Subject: [PATCH 32/40] fix: reject self-blocking table drops Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 55 +++++++++++++------ resources/Table.ts | 8 +-- resources/databases.ts | 55 ++++++++++--------- resources/transaction.ts | 12 +++- .../resources/dropTableQuiescence.test.js | 48 ++++++++++++++++ .../resources/lingeringWriteCommit.test.js | 2 + 6 files changed, 131 insertions(+), 49 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 382fdc13ac..eb2f8c26d1 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -26,7 +26,11 @@ const trackedTxns = new Set(); // is what the read-queue-depth metric counts, while this holds one entry per logical transaction — the // chain root — so a chain child can never become its own timeout root (issue #2231). const supervisedWriteRoots = new Set(); -const activeWriteTransactions = new Set(); +const activeWriteTransactions = new Set>(); +const activeWriteTransactionFinalizer = new FinalizationRegistry>((reference) => + activeWriteTransactions.delete(reference) +); +const terminalReplayCommitFailure = Symbol('terminalReplayCommitFailure'); const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); export const TRANSACTION_STATE = { @@ -154,10 +158,15 @@ export function getOutstandingCommits(): { count: number; oldestAgeMs: number | export function getPendingWriteResolutions(stores: Iterable): Promise[] { const targetStores = new Set(stores); const resolutions: Promise[] = []; - for (const transaction of activeWriteTransactions) { + for (const reference of activeWriteTransactions) { + const transaction = reference.deref(); + if (!transaction) { + activeWriteTransactions.delete(reference); + continue; + } if (transaction.writes.some((write) => write && targetStores.has(write.store))) { const resolution = transaction.getPendingWriteResolution(); - if (resolution) resolutions.push(resolution); + if (resolution) resolutions.push(resolution.finally(() => void transaction)); } } return resolutions; @@ -366,7 +375,7 @@ export class DatabaseTransaction implements Transaction { // `db` identifies the first table; allocate only when one native transaction spans more tables. #additionalStores?: Set; #lastTrackedStore?: any; - #trackedForDropDrain = false; + #dropDrainReference?: WeakRef; writes: TransactionWrite[] = []; // the set of writes to commit if the conditions are met // the last staged write per store and key, used to chain repeat writes to the same key (linkWrite) declare writesByKey?: Map>; @@ -607,9 +616,10 @@ export class DatabaseTransaction implements Transaction { error.code = 'ERR_TABLE_DROPPING'; throw error; } - if (!this.#trackedForDropDrain && operation.store?.rootStore instanceof RocksDatabase) { - this.#trackedForDropDrain = true; - activeWriteTransactions.add(this); + if (!this.#dropDrainReference && operation.store?.rootStore instanceof RocksDatabase) { + const reference = (this.#dropDrainReference = new WeakRef(this)); + activeWriteTransactions.add(reference); + activeWriteTransactionFinalizer.register(this, reference, reference); } if (operation.key === undefined) return; let writesForStore = (this.writesByKey ??= new Map()).get(operation.store); @@ -631,7 +641,7 @@ export class DatabaseTransaction implements Transaction { } getPendingWriteResolution(): Promise | undefined { - if (!activeWriteTransactions.has(this)) return; + if (!this.#dropDrainReference) return; this.#pendingWriteResolution ??= new Promise((resolve) => { this.#resolvePendingWrites = resolve; }); @@ -676,8 +686,11 @@ export class DatabaseTransaction implements Transaction { } private finishPendingWrites(): void { - activeWriteTransactions.delete(this); - this.#trackedForDropDrain = false; + if (this.#dropDrainReference) { + activeWriteTransactions.delete(this.#dropDrainReference); + activeWriteTransactionFinalizer.unregister(this.#dropDrainReference); + this.#dropDrainReference = undefined; + } this.#resolvePendingWrites?.(); this.#pendingWriteResolution = undefined; this.#resolvePendingWrites = undefined; @@ -1157,7 +1170,10 @@ export class DatabaseTransaction implements Transaction { // migration full-table copy. Both are transient and retryable. Before ERR_TRY_AGAIN was // retried here, the rejection propagated out of the unawaited onCommit() handler as an // unhandled rejection and the write was silently dropped — records lost mid-copy (#308). - if (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') { + if ( + !error[terminalReplayCommitFailure] && + (error.code === 'ERR_BUSY' || error.code === 'ERR_TRY_AGAIN') + ) { // if the transaction failed due to concurrent changes, we need to retry. First record this as an increased risk of contention/retry // for future transactions this.retries++; @@ -1277,7 +1293,11 @@ export class DatabaseTransaction implements Transaction { return txnResolution; }, (error) => { - this.abort(); + try { + this.abort(); + } catch (abortError) { + harperLogger.debug?.('aborting transaction after failed commit', abortError); + } throw error; } ); @@ -1355,11 +1375,14 @@ export class DatabaseTransaction implements Transaction { } // Keep the retained read handle alive until its iterators drain, but do not let a retryable native // code re-enter commit after the staged writes above were discarded. - return Promise.reject( - Object.assign(new Error(error instanceof Error ? error.message : String(error), { cause: error }), { - name: error instanceof Error ? error.name : 'Error', - }) + const terminalError: any = Object.assign( + new Error(error instanceof Error ? error.message : String(error), { cause: error }), + error instanceof Error ? error : undefined, + { name: error instanceof Error ? error.name : 'Error' } ); + if (error instanceof Error) Object.setPrototypeOf(terminalError, Object.getPrototypeOf(error)); + Object.defineProperty(terminalError, terminalReplayCommitFailure, { value: true }); + return Promise.reject(terminalError); } private abortSynchronousCommit(transaction: RocksTransaction, error: unknown): never { try { diff --git a/resources/Table.ts b/resources/Table.ts index 2ebe8aa3af..48e8ea3686 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -67,7 +67,7 @@ import { import { logger } from '../utility/logging/logger.ts'; import { isStaticResourceInstance } from './staticResourceDispatch.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; -import { transaction, contextStorage } from './transaction.ts'; +import { transaction, contextStorage, getExecutingTransaction } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; import { getWorkerIndex, getWorkerCount, getProcessInstanceId } from '../server/threads/manageThreads.js'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; @@ -494,7 +494,7 @@ export function makeTable(options) { .filter((operation) => pendingTableOperations.has(operation)) .map(({ label }) => label); throw new Error( - `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop request is durable: retry drop_table after the blocking operations settle, or restart Harper to complete the drop.` + `dropTable() timed out after ${LOCK_TIMEOUT}ms waiting for ${pending.size} in-flight operation(s) on ${tableName} to settle${directOperationLabels.length ? ` (${directOperationLabels.join(', ')})` : ''}; refusing to drop the column families. The drop request is durable and the table is unavailable; restart Harper to complete the drop.` ); } }; @@ -1494,7 +1494,7 @@ export function makeTable(options) { } const rootStore = primaryStore.rootStore; const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; - const activeTransaction = contextStorage.getStore()?.transaction; + const activeTransaction = getExecutingTransaction(); const currentTableStores = new Set(tableStores()); if ( sharedRocksStore && @@ -1542,7 +1542,7 @@ export function makeTable(options) { }); } catch (error) { const quiescenceError: any = new ServerError( - `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable and this durable drop will complete when Harper restarts. Retry drop_table in this process only after the blocking worker settles. ${error.message}`, + `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable. Restart Harper to complete this durable drop. ${error.message}`, 503 ); quiescenceError.code = error.code; diff --git a/resources/databases.ts b/resources/databases.ts index d86f14888b..2bbfbe7ef5 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2429,34 +2429,37 @@ export async function prepareTableDrop( const preparationKey = tableDropPreparationKey(storePath, tableName, dropGeneration); let matchingTables = incompleteTableDropPreparations.get(preparationKey); if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); - if (preserveTable) matchingTables.add(preserveTable); - for (const databaseName of Object.getOwnPropertyNames(databases)) { - const databaseTables = databases[databaseName]; - const Table = databaseTables?.[tableName]; - if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; - const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); - if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { - throw new ClientError( - `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, - 409 - ); + try { + if (preserveTable) matchingTables.add(preserveTable); + for (const databaseName of Object.getOwnPropertyNames(databases)) { + const databaseTables = databases[databaseName]; + const Table = databaseTables?.[tableName]; + if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; + const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { + throw new ClientError( + `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, + 409 + ); + } + matchingTables.add(Table); } - matchingTables.add(Table); + const preparations = await Promise.allSettled( + [...matchingTables].map(async (Table) => { + try { + await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); + } finally { + matchingTables.delete(Table); + } + }) + ); + const failedPreparation = preparations.find( + (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' + ); + if (failedPreparation) throw failedPreparation.reason; + } finally { + incompleteTableDropPreparations.delete(preparationKey); } - const preparations = await Promise.allSettled( - [...matchingTables].map(async (Table) => { - try { - await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); - } finally { - matchingTables.delete(Table); - } - }) - ); - incompleteTableDropPreparations.delete(preparationKey); - const failedPreparation = preparations.find( - (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' - ); - if (failedPreparation) throw failedPreparation.reason; } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/resources/transaction.ts b/resources/transaction.ts index d1264edac8..139d90af17 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -9,6 +9,11 @@ import { import { AsyncLocalStorage } from 'async_hooks'; export const contextStorage = new AsyncLocalStorage(); +const executingTransactionStorage = new AsyncLocalStorage(); + +export function getExecutingTransaction(): Transaction | undefined { + return executingTransactionStorage.getStore() ?? contextStorage.getStore()?.transaction; +} export function transaction(context: Context, callback: (transaction: Transaction) => T): T; export function transaction(callback: (transaction: Transaction) => T): T; @@ -42,7 +47,7 @@ export function transaction( throw new TypeError('Callback function must be provided to transaction'); } if (context?.transaction?.open === TRANSACTION_STATE.OPEN && typeof callback === 'function') { - return callback(context.transaction); // nothing to be done, already in open transaction + return executingTransactionStorage.run(context.transaction, () => callback(context.transaction)); } const transaction = new DatabaseTransaction(); @@ -53,10 +58,11 @@ export function transaction( transaction.setContext(context); let result; try { + const invokeCallback = () => executingTransactionStorage.run(transaction, () => callback(transaction)); result = (context as any).isExplicit || asyncStorageContext - ? callback(transaction) - : contextStorage.run(context, () => callback(transaction)); + ? invokeCallback() + : contextStorage.run(context, invokeCallback); if ((result as any)?.then) { return (result as any).then(onComplete, onError); } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index bc5f89561b..c239c47e94 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -256,6 +256,27 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('rejects a drop from a nested explicit transaction before tombstoning', async function () { + const tableName = `DropOwnNestedTxn_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + + await transaction(async () => { + const nestedContext = { isExplicit: true }; + await assert.rejects( + () => + transaction(nestedContext, async () => { + await Table.put({ id: 'nested-staged', name: 'pending' }, nestedContext); + await Table.dropTable(); + }), + (error) => error?.code === 'ERR_TABLE_DROP_IN_TRANSACTION' + ); + }); + assert.notStrictEqual(dbisDb.getSync(`${tableName}/`)?.dropping, true); + assert.strictEqual(databases.test?.[tableName], Table); + await Table.dropTable(); + }); + it('preserves shared stores while preparing a database alias', async function () { const tableName = `DropAliasedTable_${process.pid}_${Date.now()}`; const databaseName = `DropAliasDb_${process.pid}_${Date.now()}`; @@ -324,6 +345,33 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(attempts, 1, 'the failed table must not remain registered for later preparations'); }); + it('releases a generation-mismatched preparation before a later retry', async function () { + const tableName = `DropMismatchedPreparation_${process.pid}_${Date.now()}`; + const databaseName = `DropMismatchedPreparationDb_${process.pid}_${Date.now()}`; + const storePath = path.join(testPath, 'drop-mismatched-preparation-database'); + const dropGeneration = 'expected-generation'; + let attempts = 0; + const Table = { + primaryStore: { rootStore: { path: storePath } }, + dbisDB: { + getSync() { + return { dropping: true, dropGeneration: 'different-generation' }; + }, + }, + async _prepareDrop() { + attempts++; + }, + }; + databases[databaseName] = { [tableName]: Table }; + try { + await assert.rejects(prepareTableDrop(storePath, tableName, dropGeneration), /generation does not match/); + } finally { + delete databases[databaseName]; + } + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(attempts, 0, 'the mismatched table must not remain registered for later preparations'); + }); + it('does not wait for a read iterator on another table in the same database', async function () { const droppedTable = defineTable(`DropReadScope_${process.pid}_${Date.now()}`); const otherTable = defineTable(`DropReadScopeOther_${process.pid}_${Date.now()}`); diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index 02f41e2a22..11360cade5 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -187,6 +187,8 @@ describe('commit with open read iterators commits writes immediately on a replay Transaction.prototype.commit = originalCommit; } assert.match(rejection?.message, /forced synchronous replay failure/); + assert.strictEqual(rejection.code, 'ERR_BUSY', 'the terminal failure must retain its public error code'); + assert.strictEqual(rejection.cause?.code, 'ERR_BUSY', 'the terminal failure must retain the native cause'); assert.strictEqual(context.transaction, failedTransaction, 'context release must wait for the retained iterator'); assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); From 31d0ea6babe7dec13249416e292056369e925908 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 21:55:52 -0600 Subject: [PATCH 33/40] perf: avoid drop tracking on joined transactions Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 14 ++++--- resources/Table.ts | 21 ++++++++-- resources/databases.ts | 9 +++- resources/transaction.ts | 15 ++----- server/threads/manageThreads.js | 2 +- .../resources/dropTableQuiescence.test.js | 41 ++++++++++++++++--- .../resources/lingeringWriteCommit.test.js | 9 +++- 7 files changed, 80 insertions(+), 31 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index eb2f8c26d1..c3c30d5d23 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -1375,12 +1375,14 @@ export class DatabaseTransaction implements Transaction { } // Keep the retained read handle alive until its iterators drain, but do not let a retryable native // code re-enter commit after the staged writes above were discarded. - const terminalError: any = Object.assign( - new Error(error instanceof Error ? error.message : String(error), { cause: error }), - error instanceof Error ? error : undefined, - { name: error instanceof Error ? error.name : 'Error' } - ); - if (error instanceof Error) Object.setPrototypeOf(terminalError, Object.getPrototypeOf(error)); + const terminalError: any = new Error(error instanceof Error ? error.message : String(error), { cause: error }); + if (error instanceof Error) { + const originalStack = error.stack; + Object.setPrototypeOf(terminalError, Object.getPrototypeOf(error)); + Object.defineProperties(terminalError, Object.getOwnPropertyDescriptors(error)); + Object.defineProperty(terminalError, 'cause', { configurable: true, value: error }); + Object.defineProperty(terminalError, 'stack', { configurable: true, value: originalStack, writable: true }); + } Object.defineProperty(terminalError, terminalReplayCommitFailure, { value: true }); return Promise.reject(terminalError); } diff --git a/resources/Table.ts b/resources/Table.ts index 48e8ea3686..efaca41d1a 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -67,7 +67,7 @@ import { import { logger } from '../utility/logging/logger.ts'; import { isStaticResourceInstance } from './staticResourceDispatch.ts'; import { Addition, assignTrackedAccessors, updateAndFreeze, hasChanges, GenericTrackedObject } from './tracked.ts'; -import { transaction, contextStorage, getExecutingTransaction } from './transaction.ts'; +import { transaction, contextStorage } from './transaction.ts'; import { MAXIMUM_KEY, writeKey, compareKeys } from 'ordered-binary'; import { getWorkerIndex, getWorkerCount, getProcessInstanceId } from '../server/threads/manageThreads.js'; import { HAS_BLOBS, auditRetention, removeAuditEntry } from './auditStore.ts'; @@ -1494,7 +1494,14 @@ export function makeTable(options) { } const rootStore = primaryStore.rootStore; const sharedRocksStore = databaseName === databasePath && rootStore instanceof RocksDatabase; - const activeTransaction = getExecutingTransaction(); + const processInstanceId = sharedRocksStore ? getProcessInstanceId() : undefined; + if (sharedRocksStore && processInstanceId == null) { + throw new ServerError( + `Cannot safely coordinate dropping ${databaseName}.${tableName} from an unregistered worker`, + 503 + ); + } + const activeTransaction = contextStorage.getStore()?.transaction; const currentTableStores = new Set(tableStores()); if ( sharedRocksStore && @@ -1519,7 +1526,7 @@ export function makeTable(options) { primaryMeta.dropQuiesced = !sharedRocksStore; // A random process-start identity (not the PID, which containers commonly reuse) lets // recovery distinguish a live process that may still hold handles from a clean restart. - if (sharedRocksStore) primaryMeta.dropProcessInstance = getProcessInstanceId(); + if (sharedRocksStore) primaryMeta.dropProcessInstance = processInstanceId; const tombstoneWrite = (dbisDb as any).put(primaryCatalogKey, primaryMeta); if (tombstoneWrite?.then) await tombstoneWrite; } @@ -4218,7 +4225,13 @@ export function makeTable(options) { // in subscription.queue. Without this, the IIFE can fill the queue past // EVENT_HIGH_WATER_MARK and hit waitForDrain before the consumer's listener exists. if (request.listener) subscription!.on('data', request.listener); - const finishInitialScan = beginTableOperation('subscription replay', () => subscription.close()); + let finishInitialScan: () => void; + try { + finishInitialScan = beginTableOperation('subscription replay', () => subscription.close()); + } catch (error) { + subscription.close(); + throw error; + } const result = (async () => { const isCollection = request.isCollection ?? thisId == null; if (isCollection) { diff --git a/resources/databases.ts b/resources/databases.ts index 2bbfbe7ef5..23996aaca1 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2415,8 +2415,10 @@ function canCompleteInterruptedDrop(primaryMeta): boolean { // A completed barrier is safe immediately. An incomplete barrier is safe only after a clean // process start, when none of the handles from the recorded incarnation can still exist. // Tombstones written before the incarnation field existed necessarily came from an older process. + const processInstanceId = manageThreads.getProcessInstanceId(); return ( - primaryMeta?.dropQuiesced === true || primaryMeta?.dropProcessInstance !== manageThreads.getProcessInstanceId() + primaryMeta?.dropQuiesced === true || + (processInstanceId != null && primaryMeta?.dropProcessInstance !== processInstanceId) ); } @@ -2431,16 +2433,18 @@ export async function prepareTableDrop( if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); try { if (preserveTable) matchingTables.add(preserveTable); + let generationMismatch: ClientError | undefined; for (const databaseName of Object.getOwnPropertyNames(databases)) { const databaseTables = databases[databaseName]; const Table = databaseTables?.[tableName]; if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { - throw new ClientError( + generationMismatch ??= new ClientError( `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, 409 ); + continue; } matchingTables.add(Table); } @@ -2457,6 +2461,7 @@ export async function prepareTableDrop( (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' ); if (failedPreparation) throw failedPreparation.reason; + if (generationMismatch) throw generationMismatch; } finally { incompleteTableDropPreparations.delete(preparationKey); } diff --git a/resources/transaction.ts b/resources/transaction.ts index 139d90af17..f03c517631 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -9,11 +9,6 @@ import { import { AsyncLocalStorage } from 'async_hooks'; export const contextStorage = new AsyncLocalStorage(); -const executingTransactionStorage = new AsyncLocalStorage(); - -export function getExecutingTransaction(): Transaction | undefined { - return executingTransactionStorage.getStore() ?? contextStorage.getStore()?.transaction; -} export function transaction(context: Context, callback: (transaction: Transaction) => T): T; export function transaction(callback: (transaction: Transaction) => T): T; @@ -47,7 +42,8 @@ export function transaction( throw new TypeError('Callback function must be provided to transaction'); } if (context?.transaction?.open === TRANSACTION_STATE.OPEN && typeof callback === 'function') { - return executingTransactionStorage.run(context.transaction, () => callback(context.transaction)); + const invokeCallback = () => callback(context.transaction); + return contextStorage.getStore() === context ? invokeCallback() : contextStorage.run(context, invokeCallback); } const transaction = new DatabaseTransaction(); @@ -58,11 +54,8 @@ export function transaction( transaction.setContext(context); let result; try { - const invokeCallback = () => executingTransactionStorage.run(transaction, () => callback(transaction)); - result = - (context as any).isExplicit || asyncStorageContext - ? invokeCallback() - : contextStorage.run(context, invokeCallback); + const invokeCallback = () => callback(transaction); + result = contextStorage.getStore() === context ? invokeCallback() : contextStorage.run(context, invokeCallback); if ((result as any)?.then) { return (result as any).then(onComplete, onError); } diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index 55397a064d..d0cac1e1f7 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -50,7 +50,7 @@ const connectedPorts = []; // these are all known connected worker ports (siblin const PROCESS_INSTANCE_ENV = 'HARPER_INTERNAL_PROCESS_INSTANCE_ID'; const processInstanceId = isMainThread ? randomUUID() - : workerData?.processInstanceId || process.env[PROCESS_INSTANCE_ENV] || randomUUID(); + : workerData?.processInstanceId || process.env[PROCESS_INSTANCE_ENV]; if (isMainThread) process.env[PROCESS_INSTANCE_ENV] = processInstanceId; const MAX_UNEXPECTED_RESTARTS = 50; // Threads get 10s to die before they're forced. In dev (`harper dev`) we widen this: a reload's old diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index c239c47e94..927974042c 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -350,7 +350,8 @@ describe('dropTable worker quiescence', function () { const databaseName = `DropMismatchedPreparationDb_${process.pid}_${Date.now()}`; const storePath = path.join(testPath, 'drop-mismatched-preparation-database'); const dropGeneration = 'expected-generation'; - let attempts = 0; + let liveTableAttempts = 0; + let retainedTableAttempts = 0; const Table = { primaryStore: { rootStore: { path: storePath } }, dbisDB: { @@ -359,17 +360,27 @@ describe('dropTable worker quiescence', function () { }, }, async _prepareDrop() { - attempts++; + liveTableAttempts++; + }, + }; + const RetainedTable = { + primaryStore: { rootStore: { path: storePath } }, + async _prepareDrop() { + retainedTableAttempts++; }, }; databases[databaseName] = { [tableName]: Table }; try { - await assert.rejects(prepareTableDrop(storePath, tableName, dropGeneration), /generation does not match/); + await assert.rejects( + prepareTableDrop(storePath, tableName, dropGeneration, RetainedTable), + /generation does not match/ + ); } finally { delete databases[databaseName]; } await prepareTableDrop(storePath, tableName, dropGeneration); - assert.strictEqual(attempts, 0, 'the mismatched table must not remain registered for later preparations'); + assert.strictEqual(liveTableAttempts, 0, 'the mismatched live table must not be prepared'); + assert.strictEqual(retainedTableAttempts, 1, 'a retained class must be closed before the mismatch NACKs'); }); it('does not wait for a read iterator on another table in the same database', async function () { @@ -433,10 +444,27 @@ describe('dropTable worker quiescence', function () { await Table.put({ id: 'scan', name: 'held' }); const originalSetImmediate = global.setImmediate; + const originalGetRange = Table.primaryStore.getRange; + let scanYieldPending = false; + Table.primaryStore.getRange = function (...args) { + const iterable = originalGetRange.apply(this, args); + return { + *[Symbol.iterator]() { + for (const entry of iterable) { + scanYieldPending = true; + yield entry; + } + }, + }; + }; let releaseScan; global.setImmediate = (callback, ...args) => { - global.setImmediate = originalSetImmediate; - releaseScan = () => originalSetImmediate(callback, ...args); + if (scanYieldPending && !releaseScan) { + scanYieldPending = false; + releaseScan = () => originalSetImmediate(callback, ...args); + return; + } + return originalSetImmediate(callback, ...args); }; let scanPromise; try { @@ -444,6 +472,7 @@ describe('dropTable worker quiescence', function () { await waitFor(() => releaseScan, { message: 'getRecordCount() did not enter its yielded range scan' }); } finally { global.setImmediate = originalSetImmediate; + Table.primaryStore.getRange = originalGetRange; } const originalDropSync = Table.primaryStore.dropSync; diff --git a/unitTests/resources/lingeringWriteCommit.test.js b/unitTests/resources/lingeringWriteCommit.test.js index 11360cade5..4b9053f8a5 100644 --- a/unitTests/resources/lingeringWriteCommit.test.js +++ b/unitTests/resources/lingeringWriteCommit.test.js @@ -167,6 +167,7 @@ describe('commit with open read iterators commits writes immediately on a replay const targetDb = LingerTable.primaryStore.store.db; const context = {}; let failedTransaction; + let synchronousFailure; let iterator; let rejection; try { @@ -178,7 +179,8 @@ describe('commit with open read iterators commits writes immediately on a replay failedTransaction = context.transaction; Transaction.prototype.commit = function (...args) { if (this.store?.db !== targetDb) return originalCommit.apply(this, args); - throw Object.assign(new Error('forced synchronous replay failure'), { code: 'ERR_BUSY' }); + synchronousFailure = Object.assign(new Error('forced synchronous replay failure'), { code: 'ERR_BUSY' }); + throw synchronousFailure; }; }); } catch (error) { @@ -189,6 +191,11 @@ describe('commit with open read iterators commits writes immediately on a replay assert.match(rejection?.message, /forced synchronous replay failure/); assert.strictEqual(rejection.code, 'ERR_BUSY', 'the terminal failure must retain its public error code'); assert.strictEqual(rejection.cause?.code, 'ERR_BUSY', 'the terminal failure must retain the native cause'); + assert.strictEqual( + rejection.stack, + synchronousFailure.stack, + 'the terminal failure must retain its original stack' + ); assert.strictEqual(context.transaction, failedTransaction, 'context release must wait for the retained iterator'); assert.ok(failedTransaction.transaction, 'the iterator must retain its original native read handle'); assert.strictEqual(failedTransaction.writes.length, 0, 'the failed replay must release its tracked writes'); From 45cf9d60eaaf0a918fdc2eb43561908b7576660b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 22:21:13 -0600 Subject: [PATCH 34/40] perf: stop drop-only write overhead Co-Authored-By: GPT-5 Codex --- DESIGN.md | 21 ++++++++++++------- resources/DatabaseTransaction.ts | 5 ----- resources/databases.ts | 9 ++++++++ resources/transaction.ts | 6 ++---- .../resources/dropTableQuiescence.test.js | 3 +++ 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 8b8a19dcbd..2781e422e5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -194,14 +194,19 @@ If the table needs `audit: true`, set it both in the schema (for fresh installs) A table is a set of RocksDB column families (`T/` plus `T/`) and a set of catalog rows in the `__dbis__` store, with no transaction spanning the two. `Table.dropTable()` therefore -persists a `dropping: true` flag on the table's primary catalog entry (`T/`) before any -destructive work, then drops the column families (awaited - a failed drop must surface as the -operation's error, never a swallowed rejection), then removes the catalog rows. If the process -dies or a drop fails partway, the tombstone survives; both the boot-time schema load in -`databases.ts` (`completeInterruptedDrop`) and a same-name `table()` create complete the -interrupted drop instead of resurrecting the table. Without this, surviving catalog rows are -silently re-opened with create-if-missing on the next start, which resurrects "deleted" tables -(with their data, if the column families were never actually removed). +persists a `dropping: true` flag, a unique `dropGeneration`, and the current process incarnation on +the table's primary catalog entry (`T/`) before any destructive work. Every worker then stops new +admission, drains table reads, writes, scans, and index backfills, and closes its column-family +handles through a strict acknowledgement barrier. Only after every worker acknowledges does the +coordinator set `dropQuiesced: true`, drop the column families, and remove the catalog rows. + +The tombstone survives a partial failure. An unquiesced tombstone from the current process is not +safe to complete or recreate because another worker may still hold a handle; the table remains +unavailable and Harper must restart. After a clean restart the process incarnation differs, so +boot-time reconciliation can finish the drop. A same-name `table()` create may finish only a +quiesced drop or one left by an older process. These checks prevent surviving catalog rows from +being re-opened with create-if-missing and resurrecting a deleted table or an undiscoverable +"ghost" column family. ## MCP protocol surface (`components/mcp/`) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index c3c30d5d23..cb2f3fd51e 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -27,9 +27,6 @@ const trackedTxns = new Set(); // chain root — so a chain child can never become its own timeout root (issue #2231). const supervisedWriteRoots = new Set(); const activeWriteTransactions = new Set>(); -const activeWriteTransactionFinalizer = new FinalizationRegistry>((reference) => - activeWriteTransactions.delete(reference) -); const terminalReplayCommitFailure = Symbol('terminalReplayCommitFailure'); const MAX_OUTSTANDING_TXN_DURATION = convertToMS(envMngr.get(CONFIG_PARAMS.STORAGE_MAXTRANSACTIONQUEUETIME)) || 45000; // Allow write transactions to be queued for up to 45 seconds before we start rejecting them const DEBUG_LONG_TXNS = envMngr.get(CONFIG_PARAMS.STORAGE_DEBUGLONGTRANSACTIONS); @@ -619,7 +616,6 @@ export class DatabaseTransaction implements Transaction { if (!this.#dropDrainReference && operation.store?.rootStore instanceof RocksDatabase) { const reference = (this.#dropDrainReference = new WeakRef(this)); activeWriteTransactions.add(reference); - activeWriteTransactionFinalizer.register(this, reference, reference); } if (operation.key === undefined) return; let writesForStore = (this.writesByKey ??= new Map()).get(operation.store); @@ -688,7 +684,6 @@ export class DatabaseTransaction implements Transaction { private finishPendingWrites(): void { if (this.#dropDrainReference) { activeWriteTransactions.delete(this.#dropDrainReference); - activeWriteTransactionFinalizer.unregister(this.#dropDrainReference); this.#dropDrainReference = undefined; } this.#resolvePendingWrites?.(); diff --git a/resources/databases.ts b/resources/databases.ts index 23996aaca1..af4206fedf 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2120,6 +2120,10 @@ async function runIndexing(Table, attributes, indicesToRemove) { let indexed = 0; const attributesLength = attributes.length; await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn + if (Table.primaryStore.dropping) { + await lastResolution; + return; + } if (attributesLength > 0) { let start: any; for (const attribute of attributes) { @@ -2143,6 +2147,10 @@ async function runIndexing(Table, attributes, indicesToRemove) { versions: true, snapshot: false, // don't hold a read transaction this whole time })) { + if (Table.primaryStore.dropping) { + await lastResolution; + return; + } if (!record) continue; // deletion entry // TODO: Do we ever need to interrupt due to a schema change that was not a restart? //if (Table.schemaVersion !== schemaVersion) return; // break out if there are any schema changes and let someone else pick it up @@ -2226,6 +2234,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { hadIndexingErrors = true; logger.error(error); } + if (Table.primaryStore.dropping) return; // Yield one more event turn so any queued when() error callbacks (which fire as // microtasks when their tracked promise settles) have a chance to set hadIndexingErrors // before we decide whether to mark indexing as complete. diff --git a/resources/transaction.ts b/resources/transaction.ts index f03c517631..ca8b710817 100644 --- a/resources/transaction.ts +++ b/resources/transaction.ts @@ -23,19 +23,17 @@ export function transaction( callback?: (transaction: Transaction) => T ): T { let context: Context; - let asyncStorageContext; if (typeof ctx === 'function') { // optional first argument, handle case of no request callback = ctx; - asyncStorageContext = contextStorage.getStore(); - context = asyncStorageContext ?? {}; + context = contextStorage.getStore() ?? {}; } else { // The released placeholder is an absent argument, not a context: normalized before the fallback // chain below so it resolves to the ambient store exactly as the `null` it replaced did, rather // than to a bare `{}` that would drop the caller's user, session and timestamp. const contextArg = isReleasedTransaction(ctx) ? undefined : ctx; // request argument included, but null or undefined, so maybe create a new one - context = contextArg ?? (asyncStorageContext = contextStorage.getStore()) ?? {}; + context = contextArg ?? contextStorage.getStore() ?? {}; } if (typeof callback !== 'function') { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 927974042c..4df1a45957 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -582,8 +582,10 @@ describe('dropTable worker quiescence', function () { let indexingWriteStarted; const indexingWriteStartedPromise = new Promise((resolve) => (indexingWriteStarted = resolve)); let releaseIndexingWrite; + let indexingWrites = 0; const indexingWriteGate = new Promise((resolve) => (releaseIndexingWrite = resolve)); index.put = async function (...args) { + indexingWrites++; indexingWriteStarted(); await indexingWriteGate; return originalPut.apply(this, args); @@ -602,6 +604,7 @@ describe('dropTable worker quiescence', function () { releaseIndexingWrite(); await Promise.all([Table.indexingOperation, dropPromise]); assert.strictEqual(destructivePhaseStarted, true); + assert.ok(indexingWrites < 20, 'drop preparation must cancel the remaining index backfill'); } finally { releaseIndexingWrite(); index.put = originalPut; From 21a4dc22b1c7609fe3cfe0b7368edff694a71b0c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 22:31:54 -0600 Subject: [PATCH 35/40] fix: drain every submitted index write Co-Authored-By: GPT-5 Codex --- resources/databases.ts | 55 +++++++++---------- .../resources/dropTableQuiescence.test.js | 12 ++-- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index af4206fedf..65a114499f 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2110,18 +2110,36 @@ async function runIndexing(Table, attributes, indicesToRemove) { await signalling.signalSchemaChange( new SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName) ); + let hadIndexingErrors = false; + const pendingOperations = new Set>(); + const trackOperation = (operation: T | Promise): T | Promise => { + if (!(operation as Promise)?.then) return operation; + const pendingOperation = Promise.resolve(operation); + pendingOperations.add(pendingOperation); + pendingOperation.then( + () => pendingOperations.delete(pendingOperation), + (error) => { + pendingOperations.delete(pendingOperation); + hadIndexingErrors = true; + logger.error(error); + } + ); + return operation; + }; + const drainSubmittedOperations = async () => { + while (pendingOperations.size > 0) await Promise.allSettled([...pendingOperations]); + }; let lastResolution; for (const index of indicesToRemove) { - lastResolution = index.drop(); + lastResolution = trackOperation(index.drop()); } let interrupted; - let hadIndexingErrors = false; const attributeErrorReported = {}; let indexed = 0; const attributesLength = attributes.length; await new Promise((resolve) => setImmediate(resolve)); // yield event turn, indexing should consistently take at least one event turn if (Table.primaryStore.dropping) { - await lastResolution; + await drainSubmittedOperations(); return; } if (attributesLength > 0) { @@ -2148,7 +2166,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { snapshot: false, // don't hold a read transaction this whole time })) { if (Table.primaryStore.dropping) { - await lastResolution; + await drainSubmittedOperations(); return; } if (!record) continue; // deletion entry @@ -2178,7 +2196,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { const values = getIndexedValues(value, index.indexNulls); if (values) { for (let i = 0, l = values.length; i < l; i++) { - lastResolution = index.put(values[i], key); + lastResolution = trackOperation(index.put(values[i], key)); } } } catch (error) { @@ -2198,11 +2216,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { when( lastResolution, () => outstanding--, - (error) => { - outstanding--; - hadIndexingErrors = true; - logger.error(error); - } + () => outstanding-- ); if (workerData && workerData.restartNumber !== manageThreads.restartNumber) { interrupted = true; @@ -2211,7 +2225,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { // occasionally update our progress so if we crash, we can resume for (const attribute of attributes) { attribute.lastIndexedKey = key; - Table.dbisDB.put(attribute.key, attribute); + trackOperation(Table.dbisDB.put(attribute.key, attribute)); } if (interrupted) return; } @@ -2221,24 +2235,9 @@ async function runIndexing(Table, attributes, indicesToRemove) { else if (didSynchronousIndexing) await new Promise((resolve) => setImmediate(resolve)); // custom indexes (e.g. HNSW) index synchronously and never raise `outstanding`; without this yield a large backfill runs in a single event-loop turn, starving keepalive/replication and queries and never letting the isIndexing flag be observed } } - // Await the last pending put. If it rejects, that is also an indexing error. - // Note: the when() calls above already attach rejection handlers to each record's - // last-put promise; this try-catch specifically handles the case where lastResolution - // itself rejects (i.e. the very last put in the loop failed) which would otherwise - // throw past the hadIndexingErrors check to the outer catch. The broader issue of - // unhandled rejections from non-last puts in multi-value attributes is pre-existing - // and out of scope for this fix. - try { - await lastResolution; - } catch (error) { - hadIndexingErrors = true; - logger.error(error); - } + // A backfill is quiesced only after every write it submitted has settled. + await drainSubmittedOperations(); if (Table.primaryStore.dropping) return; - // Yield one more event turn so any queued when() error callbacks (which fire as - // microtasks when their tracked promise settles) have a chance to set hadIndexingErrors - // before we decide whether to mark indexing as complete. - await new Promise((resolve) => setImmediate(resolve)); if (hadIndexingErrors) { // Some records failed to index. Persist the failure marker in the descriptor so // the next call to table() (including after a restart with a fresh PID) re-triggers diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4df1a45957..8d462ca5da 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -581,13 +581,13 @@ describe('dropTable worker quiescence', function () { const originalPut = index.put; let indexingWriteStarted; const indexingWriteStartedPromise = new Promise((resolve) => (indexingWriteStarted = resolve)); - let releaseIndexingWrite; + let releaseFirstIndexingWrite; let indexingWrites = 0; - const indexingWriteGate = new Promise((resolve) => (releaseIndexingWrite = resolve)); + const firstIndexingWriteGate = new Promise((resolve) => (releaseFirstIndexingWrite = resolve)); index.put = async function (...args) { indexingWrites++; indexingWriteStarted(); - await indexingWriteGate; + if (indexingWrites === 1) await firstIndexingWriteGate; return originalPut.apply(this, args); }; try { @@ -600,13 +600,13 @@ describe('dropTable worker quiescence', function () { }; const dropPromise = Table.dropTable(); for (let turn = 0; turn < 5; turn++) await new Promise(setImmediate); - assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for the index backfill'); - releaseIndexingWrite(); + assert.strictEqual(destructivePhaseStarted, false, 'dropTable() must wait for every submitted index write'); + releaseFirstIndexingWrite(); await Promise.all([Table.indexingOperation, dropPromise]); assert.strictEqual(destructivePhaseStarted, true); assert.ok(indexingWrites < 20, 'drop preparation must cancel the remaining index backfill'); } finally { - releaseIndexingWrite(); + releaseFirstIndexingWrite(); index.put = originalPut; } }); From 9bae87690b0224303a29e9c4e3f0b03ac7c6e4dd Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 23:00:31 -0600 Subject: [PATCH 36/40] fix: enforce drop barrier participation Co-Authored-By: GPT-5 Codex --- resources/databases.ts | 8 +++-- server/threads/itc.js | 9 +++-- server/threads/manageThreads.js | 9 ++--- .../resources/dropTableQuiescence-worker.js | 24 +++++++++++-- .../resources/dropTableQuiescence.test.js | 34 ++++++++++++++++++- .../resources/dropTableUnready-worker.js | 20 +++++++++-- 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index 65a114499f..a9236dee96 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -23,7 +23,7 @@ import { ClientError } from '../utility/errors/hdbError.ts'; import { _assignPackageExport } from '../globals.js'; import { getIndexedValues } from '../utility/lmdb/commonUtility.ts'; import * as signalling from '../utility/signalling.ts'; -import { SchemaEventMsg } from '../server/threads/itc.js'; +import { markItcReadyForStorage, SchemaEventMsg } from '../server/threads/itc.js'; import { workerData } from 'worker_threads'; import harperLogger from '../utility/logging/harper_logger.ts'; const { forComponent } = harperLogger; @@ -284,6 +284,7 @@ export function toRocksCompression(compression: unknown): unknown { } function openRocksDatabase(path: string, options: RocksDatabaseOptions & { dupSort?: boolean }) { + markItcReadyForStorage(); options.disableWAL ??= true; const legacyOptions = options as { compression?: unknown }; // A configured codec applies to every column family, overriding whatever per-table metadata @@ -2227,7 +2228,10 @@ async function runIndexing(Table, attributes, indicesToRemove) { attribute.lastIndexedKey = key; trackOperation(Table.dbisDB.put(attribute.key, attribute)); } - if (interrupted) return; + if (interrupted) { + await drainSubmittedOperations(); + return; + } } if (outstanding > MAX_OUTSTANDING_INDEXING) await lastResolution; else if (outstanding > MIN_OUTSTANDING_INDEXING) diff --git a/server/threads/itc.js b/server/threads/itc.js index 485973643e..69e66ff71e 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -15,6 +15,7 @@ const { module.exports = { sendItcEvent, sendItcEventStrict, + markItcReadyForStorage, validateEvent, SchemaEventMsg, UserEventMsg, @@ -33,7 +34,7 @@ onMessageFromWorkers(async (event, sender) => { } if (event.relayStrictToWorkers && isMainThread) { const relayedEvent = { ...event, relayStrictToWorkers: false, requestId: undefined }; - await broadcastWithStrictAcknowledgement(relayedEvent); + await broadcastWithStrictAcknowledgement(relayedEvent, undefined, event.message.originator); } } catch (error) { handlerError = error; @@ -56,8 +57,10 @@ onMessageFromWorkers(async (event, sender) => { } } }); -if (!isMainThread && workerData?.itcReadyBuffer) { - Atomics.store(new Int32Array(workerData.itcReadyBuffer), 0, 1); +function markItcReadyForStorage() { + if (isMainThread || !workerData?.itcReadyBuffer) return; + const readySignal = new Int32Array(workerData.itcReadyBuffer); + if (Atomics.exchange(readySignal, 0, 1) === 1) return; parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); } diff --git a/server/threads/manageThreads.js b/server/threads/manageThreads.js index d0cac1e1f7..159040a1e8 100644 --- a/server/threads/manageThreads.js +++ b/server/threads/manageThreads.js @@ -721,8 +721,10 @@ function broadcastWithAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) // its handler successfully or exit completely. A timeout, ambiguous MessagePort disconnect, // handler error, or post failure rejects so the caller can leave its durable recovery marker in // place without touching storage. -function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS) { - return broadcastAwaitingAcknowledgements(message, timeout, true, true, connectedPorts, true); +function broadcastWithStrictAcknowledgement(message, timeout = DEFAULT_ACK_TIMEOUT_MS, excludedThreadId) { + const ports = + excludedThreadId == null ? connectedPorts : connectedPorts.filter((port) => port.threadId !== excludedThreadId); + return broadcastAwaitingAcknowledgements(message, timeout, true, true, ports, true); } function sendToThreadWithStrictAcknowledgement(threadId, message, timeout = DEFAULT_ACK_TIMEOUT_MS) { @@ -772,8 +774,7 @@ function broadcastAwaitingAcknowledgements( } else resolve(); }; for (let port of ports) { - // The worker publishes readiness before Table.ts can continue opening stores, so the coordinator - // does not depend on when its event loop handles the matching ITC_READY message. + // Storage opening publishes readiness atomically before creating the first RocksDB handle. const itcReady = port.itcReady || (port.itcReadySignal && Atomics.load(port.itcReadySignal, 0) === 1); if (skipUnready && !itcReady) continue; // Ordinary post-change gossip excludes transient job workers. Strict pre-change barriers diff --git a/unitTests/resources/dropTableQuiescence-worker.js b/unitTests/resources/dropTableQuiescence-worker.js index d452c4becc..294f040e87 100644 --- a/unitTests/resources/dropTableQuiescence-worker.js +++ b/unitTests/resources/dropTableQuiescence-worker.js @@ -3,7 +3,7 @@ require('../testUtils'); const { parentPort } = require('node:worker_threads'); const { setupTestDBPath } = require('../testUtils'); -const { table, closeLoadedDatabases } = require('#src/resources/databases'); +const { table, databases, closeLoadedDatabases } = require('#src/resources/databases'); const { transaction } = require('#src/resources/transaction'); const { onMessageByType, @@ -14,6 +14,8 @@ const { const MESSAGE_TYPE = 'drop-table-quiescence-test'; const CONTROL_TYPE = 'drop-table-quiescence-control'; let TestTable; +let aliasPreparations = 0; +let aliasClosedStores = false; let releaseEmbed; let releaseRead; let releaseTransaction; @@ -36,6 +38,8 @@ function runWorkerFixture() { try { switch (message.command) { case 'initialize': { + aliasPreparations = 0; + aliasClosedStores = false; const attributes = [{ name: 'id', isPrimaryKey: true }, { name: 'name' }]; if (message.withEmbed) { attributes.push({ name: 'vector', type: 'Array', embed: { source: 'name', model: 'unused' } }); @@ -45,6 +49,20 @@ function runWorkerFixture() { database: message.database ?? 'test', attributes, }); + if (message.withAlias) { + const aliasName = `${message.database ?? 'test'}_alias`; + databases[aliasName] = { + [message.table]: { + primaryStore: TestTable.primaryStore, + dbisDB: TestTable.dbisDB, + async _prepareDrop({ closeStores }) { + aliasPreparations++; + aliasClosedStores ||= closeStores; + if (closeStores) TestTable.primaryStore.close(); + }, + }, + }; + } if (message.withEmbed) { const embedGate = new Promise((resolve) => { releaseEmbed = resolve; @@ -154,11 +172,13 @@ function runWorkerFixture() { } try { await TestTable.dropTable(); - report('drop-result', { outcome: 'resolved' }); + report('drop-result', { outcome: 'resolved', aliasPreparations, aliasClosedStores }); } catch (error) { report('drop-result', { outcome: 'rejected', error: error?.stack ?? String(error), + aliasPreparations, + aliasClosedStores, }); } finally { TestTable.primaryStore.dropSync = originalDropSync; diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 8d462ca5da..4dd3d1e49b 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -989,6 +989,36 @@ describe('dropTable worker quiescence', function () { } }); + it('publishes drop-barrier readiness when opening the first RocksDB store', async function () { + const worker = startWorker(UNREADY_WORKER_FIXTURE, { + name: THREAD_TYPES.JOB, + workerIndex: 1, + threadCount: 2, + autoRestart: false, + }); + try { + await new Promise((resolve, reject) => { + worker.once('online', resolve); + worker.once('error', reject); + }); + assert.strictEqual(Atomics.load(worker.itcReadySignal, 0), 0); + const storageOpened = new Promise((resolve) => + worker.on('message', (message) => message.type === 'storage-opened' && resolve()) + ); + worker.postMessage({ type: 'open-storage', table: `DropStorageReady_${process.pid}_${Date.now()}` }); + await storageOpened; + assert.strictEqual(Atomics.load(worker.itcReadySignal, 0), 1); + const storageClosed = new Promise((resolve) => + worker.on('message', (message) => message.type === 'storage-closed' && resolve()) + ); + worker.postMessage({ type: 'close-storage' }); + await storageClosed; + } finally { + worker.wasShutdown = true; + await worker.terminate(); + } + }); + it('NACKs a malformed strict schema event', async function () { const worker = startDropWorker(1, 2); try { @@ -1233,7 +1263,7 @@ describe('dropTable worker quiescence', function () { origin = startDropWorker(2, 3, THREAD_TYPES.JOB); await Promise.all([remote.booted, origin.booted]); remote.send('initialize', { table: tableName }); - origin.send('initialize', { table: tableName }); + origin.send('initialize', { table: tableName, withAlias: true }); await Promise.all([remote.nextEvent('ready'), origin.nextEvent('ready')]); remote.send('begin-transaction', { id: 'remote-staged' }); @@ -1254,6 +1284,8 @@ describe('dropTable worker quiescence', function () { assert.strictEqual(prepared.handlesClosed, true); const dropResult = await dropResultPromise; assert.strictEqual(dropResult.outcome, 'resolved'); + assert.strictEqual(dropResult.aliasPreparations, 1, 'the originating worker must prepare its alias only once'); + assert.strictEqual(dropResult.aliasClosedStores, false, 'the relayed barrier must not close coordinator handles'); assert.deepStrictEqual([...origin.errors, ...remote.errors], []); } finally { await shutdownWorkers(origin, remote); diff --git a/unitTests/resources/dropTableUnready-worker.js b/unitTests/resources/dropTableUnready-worker.js index 2451ae532b..6d46c7917c 100644 --- a/unitTests/resources/dropTableUnready-worker.js +++ b/unitTests/resources/dropTableUnready-worker.js @@ -2,5 +2,21 @@ const { parentPort } = require('node:worker_threads'); -// Deliberately never loads Table.ts or server/threads/itc.js. -parentPort?.on('message', () => {}); +// Deliberately loads no storage or ITC module until the test requests it. +parentPort?.on('message', (message) => { + if (message.type === 'open-storage') { + require('../testUtils'); + const { setupTestDBPath } = require('../testUtils'); + setupTestDBPath(); + const { table } = require('#src/resources/databases'); + table({ + database: 'test', + table: message.table, + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + parentPort.postMessage({ type: 'storage-opened' }); + } else if (message.type === 'close-storage') { + require('#src/resources/databases').closeLoadedDatabases(); + parentPort.postMessage({ type: 'storage-closed' }); + } +}); From 390952dc20e2be392363931c843f7e1d1a9c03d1 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 23:24:49 -0600 Subject: [PATCH 37/40] fix: broadcast drop admission before draining Co-Authored-By: GPT-5 Codex --- DESIGN.md | 9 ++--- resources/Table.ts | 29 ++++++++++------ server/threads/itc.js | 8 +++-- .../resources/dropTableQuiescence.test.js | 34 +++++++++++++++++++ 4 files changed, 63 insertions(+), 17 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 2781e422e5..5cebfed188 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -195,10 +195,11 @@ If the table needs `audit: true`, set it both in the schema (for fresh installs) A table is a set of RocksDB column families (`T/` plus `T/`) and a set of catalog rows in the `__dbis__` store, with no transaction spanning the two. `Table.dropTable()` therefore persists a `dropping: true` flag, a unique `dropGeneration`, and the current process incarnation on -the table's primary catalog entry (`T/`) before any destructive work. Every worker then stops new -admission, drains table reads, writes, scans, and index backfills, and closes its column-family -handles through a strict acknowledgement barrier. Only after every worker acknowledges does the -coordinator set `dropQuiesced: true`, drop the column families, and remove the catalog rows. +the table's primary catalog entry (`T/`) before any destructive work. It then starts the local drain +and strict cross-worker barrier together, so every worker stops admission even if the coordinator's +own drain fails. Each worker drains table reads, writes, scans, and index backfills and closes its +column-family handles. Only after every worker acknowledges does the coordinator set +`dropQuiesced: true`, drop the column families, and remove the catalog rows. The tombstone survives a partial failure. An unquiesced tombstone from the current process is not safe to complete or recreate because another worker may still hold a handle; the table remains diff --git a/resources/Table.ts b/resources/Table.ts index efaca41d1a..d1d4026e51 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -1536,24 +1536,33 @@ export function makeTable(options) { let locallyQuiesced = false; try { if (sharedRocksStore) { - await prepareTableDrop(rootStore.path, tableName, dropGeneration, TableResource); - locallyQuiesced = true; - try { - await signalling.signalTableDropPreparation({ - originator: process.pid, + // Once the tombstone is durable, every worker must stop admission even if this + // coordinator's drain fails. Otherwise peers can acknowledge writes that restart + // recovery would later destroy without ever receiving the barrier. + const [localPreparation, remotePreparation] = await Promise.allSettled([ + prepareTableDrop(rootStore.path, tableName, dropGeneration, TableResource), + signalling.signalTableDropPreparation({ operation: TABLE_DROP_PREPARE_OPERATION, schema: databaseName, table: tableName, path: rootStore.path, dropGeneration, - }); - } catch (error) { + }), + ]); + locallyQuiesced = localPreparation.status === 'fulfilled'; + const failedPreparation = + localPreparation.status === 'rejected' + ? localPreparation.reason + : remotePreparation.status === 'rejected' + ? remotePreparation.reason + : undefined; + if (failedPreparation) { const quiescenceError: any = new ServerError( - `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable. Restart Harper to complete this durable drop. ${error.message}`, + `Unable to quiesce every worker before dropping ${databaseName}.${tableName}; the table remains unavailable. Restart Harper to complete this durable drop. ${failedPreparation.message ?? String(failedPreparation)}`, 503 ); - quiescenceError.code = error.code; - quiescenceError.cause = error; + quiescenceError.code = failedPreparation.code; + quiescenceError.cause = failedPreparation; throw quiescenceError; } } else { diff --git a/server/threads/itc.js b/server/threads/itc.js index 69e66ff71e..642786be2a 100644 --- a/server/threads/itc.js +++ b/server/threads/itc.js @@ -21,6 +21,8 @@ module.exports = { UserEventMsg, }; let serverItcHandlers; +const storageReadySignal = !isMainThread && workerData?.itcReadyBuffer && new Int32Array(workerData.itcReadyBuffer); +let storageReady = false; const STRICT_COORDINATOR_ACK_TIMEOUT_MS = 60000; onMessageFromWorkers(async (event, sender) => { const requestId = event?.requestId; @@ -58,9 +60,9 @@ onMessageFromWorkers(async (event, sender) => { } }); function markItcReadyForStorage() { - if (isMainThread || !workerData?.itcReadyBuffer) return; - const readySignal = new Int32Array(workerData.itcReadyBuffer); - if (Atomics.exchange(readySignal, 0, 1) === 1) return; + if (storageReady || !storageReadySignal) return; + storageReady = true; + Atomics.store(storageReadySignal, 0, 1); parentPort?.postMessage({ type: hdbTerms.ITC_EVENT_TYPES.ITC_READY }); } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4dd3d1e49b..4e4338f798 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -1210,6 +1210,40 @@ describe('dropTable worker quiescence', function () { } }); + it('quiesces peer workers when the coordinator drain fails', async function () { + this.timeout(30000); + const tableName = `DropCoordinatorFailure_${process.pid}_${Date.now()}`; + const Table = defineTable(tableName); + const dbisDb = database({ database: 'test', table: null }).dbisDb; + const originalPrepareDrop = Table._prepareDrop; + let remote; + try { + remote = startDropWorker(1, 2); + await remote.booted; + remote.send('initialize', { table: tableName }); + await remote.nextEvent('ready'); + Table._prepareDrop = async function (options) { + await originalPrepareDrop.call(this, options); + throw new Error('injected coordinator drain failure'); + }; + + const remotePrepared = remote.nextEvent('prepare-finished'); + await assert.rejects(() => Table.dropTable(), /injected coordinator drain failure/); + assert.strictEqual( + (await remotePrepared).handlesClosed, + true, + 'a durable tombstone must stop peer admission even when the coordinator fails' + ); + const tombstone = dbisDb.getSync(`${tableName}/`); + assert.strictEqual(tombstone?.dropping, true); + assert.strictEqual(tombstone?.dropQuiesced, false); + } finally { + Table._prepareDrop = originalPrepareDrop; + if (dbisDb.getSync(`${tableName}/`)?.dropping) await Table.dropTable(); + await shutdownWorkers(remote); + } + }); + it('continues after a worker fully exits during preparation', async function () { this.timeout(30000); // Force-terminating a worker intentionally bypasses closeLoadedDatabases(), so rocksdb-js keeps From c055a90aae12d85c27b914280dabef1aa41d7490 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 23:47:51 -0600 Subject: [PATCH 38/40] fix: release read drains after abort errors Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 6 +++-- .../resources/dropTableQuiescence.test.js | 23 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index cb2f3fd51e..8b92ced5c6 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -535,8 +535,8 @@ export class DatabaseTransaction implements Transaction { // transaction (see the outstanding-iterators branch in commit()), so aborting it here // discards nothing — the replay re-staged the writes AND their audit/txn-log entries // into its own transaction; this handle's never-committed log batch dies with it. +<<<<<<< HEAD const transaction = this.detachOwnedTransaction(); - this.finishPendingReads(); try { transaction?.abort(); } catch (error) { @@ -545,8 +545,10 @@ export class DatabaseTransaction implements Transaction { // loop the handle can still hold write intents, and stalled writers with a clean log is // the worst outcome here. harperLogger.warn?.('Failed to release a transaction’s native handle', error); + } finally { + this.finishPendingReads(); + this.completeDeferredContextRelease(); } - this.completeDeferredContextRelease(); } } diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index 4e4338f798..d2ec0d9e61 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -790,6 +790,29 @@ describe('dropTable worker quiescence', function () { await Table.dropTable(); }); + it('releases the read drain when doneReadTxn native abort throws', async function () { + const { Transaction } = require('@harperfast/rocksdb-js'); + const Table = defineTable(`DropAfterDoneReadAbortThrow_${process.pid}_${Date.now()}`); + const txn = new DatabaseTransaction(); + txn.db = Table.primaryStore; + const nativeTransaction = txn.getReadTxn(); + const readResolution = txn.getPendingReadResolution(); + const originalAbort = Transaction.prototype.abort; + Transaction.prototype.abort = function (...args) { + if (this === nativeTransaction) throw new Error('forced doneReadTxn abort failure'); + return originalAbort.apply(this, args); + }; + try { + assert.throws(() => txn.doneReadTxn(), /forced doneReadTxn abort failure/); + await readResolution; + assert.strictEqual(txn.transaction, null, 'a native abort failure must not retain the wrapper handle'); + } finally { + Transaction.prototype.abort = originalAbort; + nativeTransaction.abort(); + } + await Table.dropTable(); + }); + it('bounds and drains direct deleteHistory removals before dropping the primary store', async function () { const storagePath = path.join(testPath, 'delete-history-removal-databases'); const previousStoragePath = env.get(terms.CONFIG_PARAMS.STORAGE_PATH); From 944ce99f5039d51ec4eab9a7eaff4b16091991d1 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 00:14:08 -0600 Subject: [PATCH 39/40] fix: retain failed drop preparations Co-Authored-By: GPT-5 Codex --- resources/databases.ts | 60 +++++++++---------- .../resources/dropTableQuiescence.test.js | 9 +-- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/resources/databases.ts b/resources/databases.ts index a9236dee96..9da007e884 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -2443,40 +2443,36 @@ export async function prepareTableDrop( const preparationKey = tableDropPreparationKey(storePath, tableName, dropGeneration); let matchingTables = incompleteTableDropPreparations.get(preparationKey); if (!matchingTables) incompleteTableDropPreparations.set(preparationKey, (matchingTables = new Set())); - try { - if (preserveTable) matchingTables.add(preserveTable); - let generationMismatch: ClientError | undefined; - for (const databaseName of Object.getOwnPropertyNames(databases)) { - const databaseTables = databases[databaseName]; - const Table = databaseTables?.[tableName]; - if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; - const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); - if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { - generationMismatch ??= new ClientError( - `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, - 409 - ); - continue; - } - matchingTables.add(Table); + if (preserveTable) matchingTables.add(preserveTable); + let generationMismatch: ClientError | undefined; + for (const databaseName of Object.getOwnPropertyNames(databases)) { + const databaseTables = databases[databaseName]; + const Table = databaseTables?.[tableName]; + if (!Table || Table.primaryStore?.rootStore?.path !== storePath) continue; + const primaryMeta = Table.dbisDB?.getSync?.(`${tableName}/`); + if (!primaryMeta?.dropping || primaryMeta.dropGeneration !== dropGeneration) { + generationMismatch ??= new ClientError( + `Drop generation does not match on this worker for ${databaseName}.${tableName}; refusing to acknowledge preparation`, + 409 + ); + continue; } - const preparations = await Promise.allSettled( - [...matchingTables].map(async (Table) => { - try { - await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); - } finally { - matchingTables.delete(Table); - } - }) - ); - const failedPreparation = preparations.find( - (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' - ); - if (failedPreparation) throw failedPreparation.reason; - if (generationMismatch) throw generationMismatch; - } finally { - incompleteTableDropPreparations.delete(preparationKey); + matchingTables.add(Table); } + // A failed hidden class is no longer discoverable through databases; retain it for a later barrier. + const preparations = await Promise.allSettled( + [...matchingTables].map(async (Table) => { + await Table._prepareDrop({ closeStores: Table.primaryStore !== preserveTable?.primaryStore }); + matchingTables.delete(Table); + }) + ); + const failedPreparation = preparations.find( + (preparation): preparation is PromiseRejectedResult => preparation.status === 'rejected' + ); + if (failedPreparation) throw failedPreparation.reason; + if (!matchingTables.size && incompleteTableDropPreparations.get(preparationKey) === matchingTables) + incompleteTableDropPreparations.delete(preparationKey); + if (generationMismatch) throw generationMismatch; } export function dropTableMeta({ table: tableName, database: databaseName }) { diff --git a/unitTests/resources/dropTableQuiescence.test.js b/unitTests/resources/dropTableQuiescence.test.js index d2ec0d9e61..f8b5c1493b 100644 --- a/unitTests/resources/dropTableQuiescence.test.js +++ b/unitTests/resources/dropTableQuiescence.test.js @@ -317,7 +317,7 @@ describe('dropTable worker quiescence', function () { } }); - it('releases a failed preparation before a later retry', async function () { + it('retains a failed hidden preparation for a safe retry', async function () { const tableName = `DropFailedPreparation_${process.pid}_${Date.now()}`; const databaseName = `DropFailedPreparationDb_${process.pid}_${Date.now()}`; const storePath = path.join(testPath, 'drop-failed-preparation-database'); @@ -331,8 +331,7 @@ describe('dropTable worker quiescence', function () { }, }, async _prepareDrop() { - attempts++; - throw new Error('forced preparation failure'); + if (++attempts === 1) throw new Error('forced preparation failure'); }, }; databases[databaseName] = { [tableName]: Table }; @@ -342,7 +341,9 @@ describe('dropTable worker quiescence', function () { delete databases[databaseName]; } await prepareTableDrop(storePath, tableName, dropGeneration); - assert.strictEqual(attempts, 1, 'the failed table must not remain registered for later preparations'); + assert.strictEqual(attempts, 2, 'the failed hidden table must be retried by the next preparation barrier'); + await prepareTableDrop(storePath, tableName, dropGeneration); + assert.strictEqual(attempts, 2, 'a successful retry must release the retained table'); }); it('releases a generation-mismatched preparation before a later retry', async function () { From b558fb1ab4fee1f2f27710b9a403f92f229adb56 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 00:31:49 -0600 Subject: [PATCH 40/40] fix: preserve LMDB scans during table drops Co-Authored-By: GPT-5 Codex --- resources/DatabaseTransaction.ts | 1 - resources/Table.ts | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index 8b92ced5c6..9d40dc7fb6 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -535,7 +535,6 @@ export class DatabaseTransaction implements Transaction { // transaction (see the outstanding-iterators branch in commit()), so aborting it here // discards nothing — the replay re-staged the writes AND their audit/txn-log entries // into its own transaction; this handle's never-committed log batch dies with it. -<<<<<<< HEAD const transaction = this.detachOwnedTransaction(); try { transaction?.abort(); diff --git a/resources/Table.ts b/resources/Table.ts index d1d4026e51..dc768ebe23 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -4978,7 +4978,7 @@ export function makeTable(options) { if (value != null) recordCount++; entriesScanned++; await rest(); - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); if (!exactCount && !completeForExact && performance.now() - start > TIME_LIMIT) { if (!counted) { counted = true; @@ -5018,7 +5018,7 @@ export function makeTable(options) { if (value != null) recordCount++; reverseScanned++; await rest(); - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); if (reverseScanned >= limit) break; } // Use the actual entries sampled, not limit*2: the reverse scan can yield fewer than `limit` @@ -5345,7 +5345,7 @@ export function makeTable(options) { end: endTime, })) { await rest(); // yield to other async operations - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; await trackRemoval(removeAuditEntry(auditStore, auditRecord)); entriesDeleted++; @@ -5356,7 +5356,7 @@ export function makeTable(options) { for (const entry of primaryStore.getRange({ start: 0, versions: true })) { const { value, localTime } = entry; await rest(); // yield to other async operations - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); if (value === null && localTime < endTime) { await trackRemoval(removeEntry(primaryStore, entry)); } @@ -5396,7 +5396,7 @@ export function makeTable(options) { for (let next = iterator.next(); !next.done; next = iterator.next()) { const auditRecord = next.value; await rest(); // yield to other async operations - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); if (auditRecord.tableId !== tableId) continue; yield { id: auditRecord.recordId, @@ -5407,7 +5407,7 @@ export function makeTable(options) { user: auditRecord.user, operation: auditRecord.originatingOperation, }; - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); } } finally { try { @@ -5430,7 +5430,7 @@ export function makeTable(options) { const auditWindow = 100; do { await rest(); // yield to other async operations - if (droppingTable) throw tableDroppingError(); + if (isRocksDB && droppingTable) throw tableDroppingError(); let insertionPoint = history.length; let highestPreviousVersion = 0; const start = nextVersion - auditWindow;