From e1d72aecc3b9cc7b1a7c546cf2d588f2b4b54654 Mon Sep 17 00:00:00 2001 From: Logan Lindquist Land Date: Sun, 23 Aug 2026 21:48:33 -0500 Subject: [PATCH] perf: cache Turso query statements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse prepared statements per adapter so Turso callers stop paying\nprepare/close churn on every query while keeping native resources\nbounded and concurrency-safe.\n\n• Serialize access per statement because concurrent rebinds can corrupt\n results on the native driver\n• Evict and drain statements safely, and expose adapter disposal without\n closing the caller-owned database\n• Cover the real driver path, caller-controlled table names, and release\n packaging exports --- docs/TURSO.md | 58 +++++++--- scripts/check-dist-types.mjs | 13 ++- src/turso.ts | 206 +++++++++++++++++++++++++++++++---- tests/database.test.ts | 104 +++++++++++++++++- tests/turso-database.test.ts | 91 ++++++++++++++-- 5 files changed, 419 insertions(+), 53 deletions(-) diff --git a/docs/TURSO.md b/docs/TURSO.md index 9dee265..31f14a8 100644 --- a/docs/TURSO.md +++ b/docs/TURSO.md @@ -65,6 +65,11 @@ const results = await search({ limit: 5, embeddingOptions, }); + +// When this adapter's lifetime ends, drain and close its cached statements +// before closing the caller-owned database handle. +await client.dispose(); +await database.close(); ``` Use `":memory:"` instead of a file path for an ephemeral database. @@ -74,6 +79,29 @@ Use `":memory:"` instead of a file path for an ephemeral database. result shapes — `SearchOptions`, `SearchResult`, `IndexerOptions`, `IndexResult` — are identical on both backends. +### Adapter lifetime and disposal + +One adapter caches up to **32 query statements**, keyed by SQL and evicted in +least-recently-used order. The bound matters because public `tableName` options +are embedded in SQL: an unbounded cache would let a long-lived process retain a +new native statement for every valid table name it sees. Statements that are +currently running or queued are closed after they finish rather than being +evicted out from under a call. + +Call `await client.dispose()` after all work using that adapter has settled. +Disposal drains queued query calls and closes every cached statement, but does +**not** close the database handle you supplied. It is terminal: later calls on +that adapter reject. Close the database separately, after disposal. If you omit +disposal, the reusable cache is still bounded at 32 entries and lives until the +underlying handle or process exits; statements evicted while in flight live +only until their queued calls finish. + +The adapter serializes concurrent calls that share one cached statement. This +is required for correctness, not just memory use: the native statement mutates +its current bindings, so overlapping `all()` calls with different arguments can +otherwise return another caller's rows. Different cached SQL statements remain +independent. + ## What is different on Turso ### There is no ANN vector index, so search is a full scan @@ -192,13 +220,14 @@ point is still type-checked by `deno task check` so the claim above stays true. unchanged, and the main entry point exports no new symbol and references no Turso type. -`tursoAdapter()` returns a `DatabaseAdapter`, and that type is exported from -`libsql-search/turso` so you can name it: +`tursoAdapter()` returns a `TursoAdapter`, which extends `DatabaseAdapter` with +the disposal hook. Both types are exported from `libsql-search/turso` so you can +name them: ```ts -import { tursoAdapter, type DatabaseAdapter } from "libsql-search/turso"; +import { tursoAdapter, type TursoAdapter } from "libsql-search/turso"; -let client: DatabaseAdapter; +let client: TursoAdapter; ``` It is exported from the subpath only. The main entry point does not export it, @@ -217,15 +246,18 @@ to prove the two stay interchangeable, and runs as part of Three things in `src/turso.ts` look like noise and are not. Each has a regression test; none of them fails loudly at runtime if removed. -**Prepared statements must be closed.** A statement holds native memory that the -garbage collector cannot reclaim, because it is not JavaScript heap. Every -`prepare()` is released with `close()` in a `finally`. Measured through the -built bundle, 60 000 `executeQuery()` calls on one handle grow RSS by ~570 MB -without the close and ~150 MB with it. `search()` issues exactly one query, so -an SSR site calling it per request is the case that turns this from untidy into -an OOM. Inside `executeAtomicWrite()` the cached statements are released only -*after* `COMMIT` or `ROLLBACK` — a statement stays bound to the transaction -while it is open. +**Prepared statements are bounded, serialized, and explicitly disposed.** A +statement holds native memory that the garbage collector cannot reclaim, +because it is not JavaScript heap. The query path therefore reuses a 32-entry +LRU instead of preparing on every request, serializes rebinds per entry, and +closes evictions only after their queued calls finish. `TursoAdapter.dispose()` +drains and closes the remaining cache. Measured through the built bundle, +60 000 `executeQuery()` calls grew RSS by ~570 MB when statements were never +closed, ~150 MB when each call prepared and closed, and ~7 MB when one statement +was reused. `search()` issues exactly one query, so the SSR-per-request path is +where reuse matters most. Inside `executeAtomicWrite()` the transaction-local +statements are still released only *after* `COMMIT` or `ROLLBACK` — a statement +stays bound to the transaction while it is open. **`BEGIN IMMEDIATE` sits outside the `try`.** If it were inside, a `BEGIN` that fails because another rebuild already holds the write lock would fall into the diff --git a/scripts/check-dist-types.mjs b/scripts/check-dist-types.mjs index 7cc3490..b1771e7 100644 --- a/scripts/check-dist-types.mjs +++ b/scripts/check-dist-types.mjs @@ -151,7 +151,12 @@ async function assertMainEntryUnchanged() { * below proves rather than assumes. */ const EXPECTED_TURSO_VALUE_EXPORTS = ['tursoAdapter']; -const EXPECTED_TURSO_TYPE_EXPORTS = ['DatabaseAdapter', 'TursoDatabase', 'TursoStatement']; +const EXPECTED_TURSO_TYPE_EXPORTS = [ + 'DatabaseAdapter', + 'TursoAdapter', + 'TursoDatabase', + 'TursoStatement', +]; async function assertTursoEntryExports() { const source = await readFile(join(distDirectory, 'turso.d.ts'), 'utf8'); @@ -216,11 +221,13 @@ declare const handle: { exec(sql: string): unknown; prepare(sql: string): { run( // The assignments are the assertion: an adapter produced by the subpath entry // must satisfy the client type declared by the main entry, across two // independently bundled declarations. -const searchClient: SearchOptions['client'] = tursoAdapter(handle); -const indexClient: IndexerOptions['client'] = tursoAdapter(handle); +const adapter = tursoAdapter(handle); +const searchClient: SearchOptions['client'] = adapter; +const indexClient: IndexerOptions['client'] = adapter; void searchClient; void indexClient; +void adapter.dispose(); `; await writeFile(consumerFile, consumerSource); diff --git a/src/turso.ts b/src/turso.ts index ef1e07e..a91e1c3 100644 --- a/src/turso.ts +++ b/src/turso.ts @@ -14,11 +14,21 @@ * import { tursoAdapter } from 'libsql-search/turso'; * import { createTable, indexContent, search } from 'libsql-search'; * - * const client = tursoAdapter(await connect('./local.db')); + * const database = await connect('./local.db'); + * const client = tursoAdapter(database); + * const embeddingOptions = { + * provider: 'openai-compatible' as const, + * baseUrl: 'https://embeddings.example.com/v1', + * model: 'bge-large-en-v1.5', + * dimensions: 1024 + * }; * - * await createTable(client); - * await indexContent({ client, contentPath: './content' }); - * const results = await search({ client, query: 'vector search' }); + * await createTable(client, 'articles', 1024); + * await indexContent({ client, contentPath: './content', embeddingOptions }); + * const results = await search({ client, query: 'vector search', embeddingOptions }); + * + * await client.dispose(); + * await database.close(); * ``` * * @module libsql-search/turso @@ -55,8 +65,9 @@ export interface TursoStatement { * * Not closing leaks roughly 10 KB of native memory per prepare on * `@tursodatabase/database`, which the garbage collector does not reclaim - * because it is not JavaScript heap. A server calling `search()` per request - * grows without bound until the process is killed. + * because it is not JavaScript heap. The adapter therefore closes + * transaction-local statements immediately and query statements on safe LRU + * eviction or disposal. */ close?(): unknown; } @@ -74,6 +85,38 @@ export interface TursoDatabase { prepare(sql: string): TursoStatement; } +/** + * A Turso-backed adapter with an explicit prepared-statement disposal hook. + * + * `dispose()` is terminal: it waits for queued query calls, closes every + * cached query statement, and rejects later adapter operations. It does not + * close the caller-owned {@link TursoDatabase} handle. + */ +export interface TursoAdapter extends DatabaseAdapter { + dispose(): Promise; +} + +/** + * Maximum number of idle/reusable query statements retained by one adapter. + * + * SQL includes caller-controlled table names, so an unbounded map would turn + * statement reuse into a slower native-memory leak. Thirty-two entries cover + * the library's fixed query shapes across several tables while keeping that + * lifetime cost small and deterministic. + */ +const MAX_QUERY_STATEMENT_CACHE_SIZE = 32; + +interface QueryStatementCacheEntry { + readonly sql: string; + readonly statement: TursoStatement; + /** Resolves when every call already queued on this statement has finished. */ + tail: Promise; + /** Includes the running call and calls waiting for their turn. */ + pending: number; + retired: boolean; + closePromise?: Promise; +} + /** * Wrap a `@tursodatabase/database` handle so this library's functions can use * it. @@ -94,10 +137,93 @@ export interface TursoDatabase { * This preserves the guarantee that a failed rebuild leaves the previous * index intact. */ -export function tursoAdapter(database: TursoDatabase): DatabaseAdapter { +export function tursoAdapter(database: TursoDatabase): TursoAdapter { assertTursoDatabase(database); - return { + const queryStatementsBySql = new Map(); + const liveQueryStatements = new Set(); + let disposed = false; + let disposePromise: Promise | undefined; + + const assertUsable = (): void => { + if (disposed) { + throw new Error('This Turso adapter has been disposed'); + } + }; + + const closeQueryStatement = ( + entry: QueryStatementCacheEntry + ): Promise => { + entry.closePromise ??= closeStatement(entry.statement).finally(() => { + liveQueryStatements.delete(entry); + }); + + return entry.closePromise; + }; + + const retireQueryStatement = (entry: QueryStatementCacheEntry): void => { + if (queryStatementsBySql.get(entry.sql) === entry) { + queryStatementsBySql.delete(entry.sql); + } + + entry.retired = true; + if (entry.pending === 0) { + // closeStatement() absorbs both synchronous throws and rejected close + // promises, so deliberately detaching this cannot create an unhandled + // rejection. + void closeQueryStatement(entry); + } + }; + + const acquireQueryStatement = (sql: string): { + entry: QueryStatementCacheEntry; + turn: Promise; + release: () => void; + } => { + assertUsable(); + + let entry = queryStatementsBySql.get(sql); + if (entry === undefined) { + entry = { + sql, + statement: database.prepare(sql), + tail: Promise.resolve(), + pending: 0, + retired: false + }; + queryStatementsBySql.set(sql, entry); + liveQueryStatements.add(entry); + + if (queryStatementsBySql.size > MAX_QUERY_STATEMENT_CACHE_SIZE) { + const oldest = queryStatementsBySql.values().next().value as + | QueryStatementCacheEntry + | undefined; + if (oldest !== undefined) { + retireQueryStatement(oldest); + } + } + } else { + // Map iteration order is the LRU order. Refresh a hit to the newest end. + queryStatementsBySql.delete(sql); + queryStatementsBySql.set(sql, entry); + } + + entry.pending += 1; + + // The native statement mutates its bound parameters. Concurrent all() + // calls on one statement race and can return another caller's rows, so + // every cache entry owns a tiny promise queue. + const turn = entry.tail; + let release!: () => void; + const completion = new Promise(resolve => { + release = resolve; + }); + entry.tail = turn.then(() => completion); + + return { entry, turn, release }; + }; + + const adapter: TursoAdapter = { libsqlSearchAdapter: true, backend: 'turso', @@ -109,6 +235,7 @@ export function tursoAdapter(database: TursoDatabase): DatabaseAdapter { supportsVectorIndex: false, async executeDdl(sql: string): Promise { + assertUsable(); await database.exec(sql); }, @@ -118,20 +245,31 @@ export function tursoAdapter(database: TursoDatabase): DatabaseAdapter { | Readonly> | ReadonlyArray ): Promise>> { - const statement = database.prepare(sql); + const { entry, turn, release } = acquireQueryStatement(sql); + await turn; try { // `all()` binds nothing when called with no argument. Passing an // explicit `undefined` would be read as a single positional bind of // NULL. - const rows = await (args === undefined ? statement.all() : statement.all(args)); + const rows = await ( + args === undefined ? entry.statement.all() : entry.statement.all(args) + ); return rows as Array>; + } catch (error) { + // Do not keep a statement whose execution failed in the reusable LRU. + // Calls already queued on it may finish, then the final release closes + // it; a later call prepares a clean replacement. + retireQueryStatement(entry); + throw error; } finally { - // Every query prepares its own statement, so skipping this leaks on - // every call. An SSR site calling search() per request is the case that - // makes it fatal rather than untidy. - closeStatement(statement); + entry.pending -= 1; + release(); + + if (entry.retired && entry.pending === 0) { + await closeQueryStatement(entry); + } } }, @@ -154,6 +292,7 @@ export function tursoAdapter(database: TursoDatabase): DatabaseAdapter { args?: ReadonlyArray; }> ): Promise { + assertUsable(); const preparedBySql = new Map(); const prepareOnce = (sql: string): TursoStatement => { @@ -199,29 +338,56 @@ export function tursoAdapter(database: TursoDatabase): DatabaseAdapter { // stays bound to the transaction while it is open, so closing earlier // would release it out from under the write in progress. for (const prepared of preparedBySql.values()) { - closeStatement(prepared); + await closeStatement(prepared); } preparedBySql.clear(); } + }, + + dispose(): Promise { + disposePromise ??= (async () => { + disposed = true; + + // Copy first because retiring an entry removes it from the LRU map. + for (const entry of [...queryStatementsBySql.values()]) { + retireQueryStatement(entry); + } + + // Includes already-evicted statements that still have queued calls. + // Their per-entry tails resolve only after the last call releases its + // turn, so no native statement is closed while it is still in use. + await Promise.all( + [...liveQueryStatements].map(async entry => { + await entry.tail; + await closeQueryStatement(entry); + }) + ); + })(); + + return disposePromise; } }; + + return adapter; } /** * Release a prepared statement, ignoring any failure. * - * Called from `finally` blocks, so throwing here would replace the error that - * is actually worth reporting. A close failure is not actionable by the caller, - * and closing is idempotent on this backend. + * Used from transaction `finally` blocks, query-cache eviction, and adapter + * disposal. Throwing here could replace the error that is actually worth + * reporting or turn best-effort cache cleanup into an unhandled rejection. A + * close failure is not actionable by the caller, and closing is idempotent on + * this backend. * * Deliberately quieter than `warnOnFailedRollback` below: a failed rollback * happens at most once per rebuild, while this runs once per query — on the * exact hot path the close exists to protect. Warning here would flood it. */ -function closeStatement(statement: TursoStatement): void { +async function closeStatement(statement: TursoStatement): Promise { try { - statement.close?.(); + await statement.close?.(); } catch { // ignored on purpose } diff --git a/tests/database.test.ts b/tests/database.test.ts index 249b78b..080102f 100644 --- a/tests/database.test.ts +++ b/tests/database.test.ts @@ -242,6 +242,7 @@ describe('database boundary', () => { */ function createRecordingHandle(options: { omitClose?: boolean } = {}) { const events: string[] = []; + let prepares = 0; let closes = 0; const handle = { @@ -249,6 +250,7 @@ describe('database boundary', () => { events.push(`exec:${sql}`); }, prepare: (sql: string) => { + prepares += 1; events.push(`prepare:${sql}`); const statement: Record = { @@ -273,15 +275,30 @@ describe('database boundary', () => { } }; - return { handle, events, closeCount: () => closes }; + return { + handle, + events, + prepareCount: () => prepares, + closeCount: () => closes + }; } - it('should close the prepared statement after a query', async () => { - const { handle, events } = createRecordingHandle(); + it('should reuse a query statement until the adapter is disposed', async () => { + const { handle, events, prepareCount, closeCount } = createRecordingHandle(); + const adapter = tursoAdapter(handle); + + await adapter.executeQuery('SELECT 1'); + await adapter.executeQuery('SELECT 1'); - await tursoAdapter(handle).executeQuery('SELECT 1'); + expect(prepareCount()).toBe(1); + expect(closeCount()).toBe(0); + expect(events).toEqual(['prepare:SELECT 1', 'all', 'all']); - expect(events).toEqual(['prepare:SELECT 1', 'all', 'close']); + await adapter.dispose(); + + expect(closeCount()).toBe(1); + expect(events.at(-1)).toBe('close'); + await expect(adapter.executeQuery('SELECT 1')).rejects.toThrow('disposed'); }); it('should close the prepared statement even when the query throws', async () => { @@ -314,8 +331,83 @@ describe('database boundary', () => { // Optional on the interface, so the repo's own fake handles and any // driver that manages statement lifetime itself keep working. const { handle } = createRecordingHandle({ omitClose: true }); + const adapter = tursoAdapter(handle); + + await expect(adapter.executeQuery('SELECT 1')).resolves.toEqual([]); + await expect(adapter.dispose()).resolves.toBeUndefined(); + }); + + it('should bound caller-controlled table SQL to 32 cached statements', async () => { + const { handle, prepareCount, closeCount } = createRecordingHandle(); + const adapter = tursoAdapter(handle); + + for (let i = 0; i < 64; i++) { + await getFolders(adapter, `articles_${String(i)}`); + } + + expect(prepareCount()).toBe(64); + expect(closeCount()).toBe(32); + + // The newest entry is still cached and reusable. + await getFolders(adapter, 'articles_63'); + expect(prepareCount()).toBe(64); + + await adapter.dispose(); + expect(closeCount()).toBe(64); + }); + + it('should defer eviction and disposal until an in-flight query finishes', async () => { + let releaseSlow!: () => void; + const slowResult = new Promise>>(resolve => { + releaseSlow = () => resolve([]); + }); + let signalStarted!: () => void; + const started = new Promise(resolve => { + signalStarted = resolve; + }); + const closed: string[] = []; + const handle = { + exec: async () => {}, + prepare: (sql: string) => ({ + run: async () => {}, + all: async () => { + if (sql === 'SELECT slow') { + signalStarted(); + return slowResult; + } + return []; + }, + close: () => { + closed.push(sql); + } + }) + }; + const adapter = tursoAdapter(handle); + + const slowQuery = adapter.executeQuery('SELECT slow'); + await started; + + // The 33rd distinct SQL evicts the active oldest entry from the 32-slot + // LRU, but must not close it while all() is still using it. + for (let i = 0; i < 32; i++) { + await adapter.executeQuery(`SELECT ${String(i)}`); + } + expect(closed).not.toContain('SELECT slow'); + + let disposed = false; + const disposal = adapter.dispose().then(() => { + disposed = true; + }); + await Promise.resolve(); + expect(disposed).toBe(false); + + releaseSlow(); + await slowQuery; + await disposal; - await expect(tursoAdapter(handle).executeQuery('SELECT 1')).resolves.toEqual([]); + expect(disposed).toBe(true); + expect(closed.filter(sql => sql === 'SELECT slow')).toHaveLength(1); + expect(closed).toHaveLength(33); }); it('should close cached statements only after COMMIT', async () => { diff --git a/tests/turso-database.test.ts b/tests/turso-database.test.ts index d90ada6..8ba575d 100644 --- a/tests/turso-database.test.ts +++ b/tests/turso-database.test.ts @@ -500,12 +500,17 @@ describeTurso('turso database backend', () => { * calls close(). This proves the real driver's statements actually accept * it on the paths users take, including inside an open transaction. */ - function createCloseTrackingHandle(): { handle: TursoDatabase; closed: number } { - const tracker = { closed: 0 }; + function createCloseTrackingHandle(): { + handle: TursoDatabase; + prepared: number; + closed: number; + } { + const tracker = { prepared: 0, closed: 0 }; const handle: TursoDatabase = { exec: (sql: string) => database.exec(sql), prepare: (sql: string): TursoStatement => { + tracker.prepared += 1; const statement = database.prepare(sql) as TursoStatement & { close?(): unknown }; return { @@ -521,38 +526,48 @@ describeTurso('turso database backend', () => { return { handle, + get prepared() { + return tracker.prepared; + }, get closed() { return tracker.closed; } }; } - it('should close the statement behind a real search()', async () => { + it('should retain the statement behind a real search until disposal', async () => { await createTable(client); await insertTestArticle({ slug: 'a', title: 'A', content: 'TypeScript' }); const tracking = createCloseTrackingHandle(); + const tracked = tursoAdapter(tracking.handle); await search({ - client: tursoAdapter(tracking.handle), + client: tracked, query: 'TypeScript', embeddingOptions: TEST_EMBEDDING_OPTIONS }); - // search() issues exactly one query, so one prepare and one close. An SSR - // site calls this per request; leaking here grows the process without - // bound. + expect(tracking.prepared).toBe(1); + expect(tracking.closed).toBe(0); + + await tracked.dispose(); expect(tracking.closed).toBe(1); }, 30000); - it('should close the statements behind a real retrieval helper', async () => { + it('should retain the statement behind a real retrieval helper until disposal', async () => { await createTable(client); await insertTestArticle({ slug: 'a', title: 'A', content: 'TypeScript' }); const tracking = createCloseTrackingHandle(); - const folders = await getFolders(tursoAdapter(tracking.handle)); + const tracked = tursoAdapter(tracking.handle); + const folders = await getFolders(tracked); expect(folders).toEqual(['root']); + expect(tracking.prepared).toBe(1); + expect(tracking.closed).toBe(0); + + await tracked.dispose(); expect(tracking.closed).toBe(1); }, 30000); @@ -562,9 +577,10 @@ describeTurso('turso database backend', () => { await writeFile(join(testDir, 'beta.md'), '---\ntitle: Beta\n---\nContent'); const tracking = createCloseTrackingHandle(); + const tracked = tursoAdapter(tracking.handle); await indexContent({ - client: tursoAdapter(tracking.handle), + client: tracked, contentPath: testDir, embeddingOptions: TEST_EMBEDDING_OPTIONS }); @@ -572,8 +588,12 @@ describeTurso('turso database backend', () => { // One DELETE plus one cached INSERT, both closed after COMMIT. Closing a // statement bound to a transaction that already committed is safe; the // committed rows below prove the close did not disturb the write. + expect(tracking.prepared).toBe(2); expect(tracking.closed).toBe(2); expect(await indexedTitles()).toEqual(['Alpha', 'Beta']); + + await tracked.dispose(); + expect(tracking.closed).toBe(2); }, 30000); it('should survive repeated queries on one handle', async () => { @@ -592,7 +612,56 @@ describeTurso('turso database backend', () => { expect(folders).toEqual(['root']); } - expect(tracking.closed).toBe(250); + // Call count grows while the native statement count stays constant. + // Reverting to prepare-per-call makes this assertion fail at 250. + expect(tracking.prepared).toBe(1); + expect(tracking.closed).toBe(0); + + await tracked.dispose(); + expect(tracking.closed).toBe(1); + }, 30000); + + it('should serialize concurrent rebinds on one cached real statement', async () => { + const tracking = createCloseTrackingHandle(); + const tracked = tursoAdapter(tracking.handle); + const expected = Array.from({ length: 250 }, (_, index) => index); + + const rows = await Promise.all( + expected.map(value => tracked.executeQuery('SELECT ? AS value', [value])) + ); + + expect(rows.map(result => result[0]?.value)).toEqual(expected); + expect(tracking.prepared).toBe(1); + expect(tracking.closed).toBe(0); + + await tracked.dispose(); + expect(tracking.closed).toBe(1); + }, 30000); + + it('should reuse a cached real query after an intervening rebuild transaction', async () => { + await createTable(client); + const tracking = createCloseTrackingHandle(); + const tracked = tursoAdapter(tracking.handle); + + await expect(getFolders(tracked)).resolves.toEqual([]); + expect(tracking.prepared).toBe(1); + + await writeFile(join(testDir, 'alpha.md'), '---\ntitle: Alpha\n---\nContent'); + await indexContent({ + client: tracked, + contentPath: testDir, + embeddingOptions: TEST_EMBEDDING_OPTIONS + }); + + // Two transaction-local statements were prepared and closed. The + // original getFolders statement remains cached and sees committed rows. + expect(tracking.prepared).toBe(3); + expect(tracking.closed).toBe(2); + await expect(getFolders(tracked)).resolves.toEqual(['root']); + expect(tracking.prepared).toBe(3); + + await tracked.dispose(); + expect(tracking.closed).toBe(3); }, 30000); });