From 93c00c5ec97a8c451fd72713936fe263f3d22cc4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 14 Aug 2026 16:18:00 +0200 Subject: [PATCH 1/9] Bump onnxruntime-web to a build that accepts Blob-backed external data onnxruntime PR #29477, "[web] Support Blob-backed external data for on-demand loading in JSPI builds", merged on 2026-07-14. The pin here predates it (2026-04-16), so the runtime rejects a Blob handed to it as external data and the change in the next commit has nothing to hand it to. 1.29.0-dev.20260811-e415ef9afd is dated after that merge and is the build the change was developed and measured against. --- packages/transformers/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/transformers/package.json b/packages/transformers/package.json index 8ea694b11..f7f092e68 100644 --- a/packages/transformers/package.json +++ b/packages/transformers/package.json @@ -58,7 +58,7 @@ "@huggingface/jinja": "^0.5.6", "@huggingface/tokenizers": "^0.1.3", "onnxruntime-node": "1.24.3", - "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "onnxruntime-web": "1.29.0-dev.20260811-e415ef9afd", "sharp": "^0.34.5" }, "devDependencies": { From cda4d24a1aeccb6bc9325b74d5d72fced0fb0f0f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 14 Aug 2026 16:18:00 +0200 Subject: [PATCH 2/9] Hand onnxruntime-web the cached Blob instead of a copy of the weights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading a model from Cache Storage currently materialises every external-data file into the JS heap before onnxruntime-web ever sees it: `loadResourceFile` reads the `Response` into a `Uint8Array`, and `getModelDataFiles` passes that buffer on. For a multi-gigabyte model that is a full second copy of the weights, live at the same moment the runtime is allocating its own. A Cache Storage `Response.blob()` is a file reference rather than a copy, and onnxruntime-web accepts one for external data as of the runtime bumped in the previous commit. So on a cache hit the bytes can go straight from disk to the runtime and never enter the heap at all. Measured on a 2.887 GB model, same runtime and same session: peak resident goes from 6,056 MB to 4,221 MB. The saving is the size of the weights, so it grows with the model. Two deliberate limits. It only applies on a cache HIT. Reading the stream to report download progress would defeat the purpose — the chunks would be resident twice, once as buffers and once in Blob storage — and while several gigabytes are arriving, progress is worth more than peak memory. Every load after the first takes the new path, which is where the peak actually matters, and is unchanged for Node, which keeps returning a path. And it is opt-in, threaded from the one caller that knows it is looking at external data. `loadResourceFile` is generic — it also serves config.json and tokenizer.json, whose callers parse the result as text — so returning a Blob unconditionally would break them. --- packages/transformers/src/utils/hub.js | 41 ++++++++++++++++--- .../transformers/src/utils/model-loader.js | 9 ++-- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index 09f42e168..0daf06bf7 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -245,9 +245,11 @@ export async function storeCachedResource(path_or_repo_id, filename, cache, cach * @param {PretrainedOptions} [options] An object containing optional parameters. * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. * @param {import('./cache.js').CacheInterface | null} [cache] The cache instance to use. + * @param {boolean} [as_blob=false] Whether to return a `Blob` when the file is served from the cache, + * rather than reading it into a `Uint8Array`. Only honoured on a cache hit — see the note at the use site. * * @throws Will throw an error if the file is not found and `fatal` is true. - * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. + * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache. */ export async function loadResourceFile( path_or_repo_id, @@ -256,6 +258,7 @@ export async function loadResourceFile( options = {}, return_path = false, cache = null, + as_blob = false, ) { const { requestURL, localPath, remoteURL, proposedCacheKey, validModelId } = buildResourcePaths( path_or_repo_id, @@ -366,7 +369,26 @@ export async function loadResourceFile( let buffer; if (typeof response !== 'string') { - if (!options.progress_callback) { + if (as_blob && cacheHit) { + // The whole point of `as_blob`, and it is one call: a Cache Storage `Response` hands back a + // Blob that is a FILE REFERENCE rather than a copy, so the bytes never enter the JS heap. + // + // Gated on `cacheHit`, which is a deliberate trade rather than caution. Reading the stream to + // report progress would defeat the whole thing — the chunks would be resident twice, once as + // buffers and once in Blob storage — and on a cold download progress is worth more than peak + // memory, because the user is watching several gigabytes arrive. Every load AFTER the first + // takes this branch, is instantaneous, and is where the peak actually matters. + buffer = /** @type {any} */ (await response.blob()); + + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + progress: 100, + loaded: buffer.size, + total: buffer.size, + }); + } else if (!options.progress_callback) { // If no progress callback is specified, we can use the `.arrayBuffer()` // method to read the response. buffer = new Uint8Array(await response.arrayBuffer()); @@ -494,11 +516,20 @@ const INFLIGHT_LOADS = new Map(); * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. * @param {PretrainedOptions} [options] An object containing optional parameters. * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. + * @param {boolean} [as_blob=false] Whether to accept a `Blob` for a file served from the cache, avoiding a + * copy of its bytes on the JS heap. Intended for external data, which is handed straight to the runtime. * * @throws Will throw an error if the file is not found and `fatal` is true. - * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. + * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache. */ -export async function getModelFile(path_or_repo_id, filename, fatal = true, options = {}, return_path = false) { +export async function getModelFile( + path_or_repo_id, + filename, + fatal = true, + options = {}, + return_path = false, + as_blob = false, +) { if (!env.allowLocalModels) { // User has disabled local models, so we just make sure other settings are correct. @@ -529,7 +560,7 @@ export async function getModelFile(path_or_repo_id, filename, fatal = true, opti file: filename, }); pending = getCache(options.cache_dir).then((cache) => - loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache), + loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache, as_blob), ); if (loads === INFLIGHT_LOADS) { pending = pending.finally(() => INFLIGHT_LOADS.delete(key)); diff --git a/packages/transformers/src/utils/model-loader.js b/packages/transformers/src/utils/model-loader.js index 599aefe9d..5c137342b 100644 --- a/packages/transformers/src/utils/model-loader.js +++ b/packages/transformers/src/utils/model-loader.js @@ -57,7 +57,7 @@ export async function getCoreModelFile(pretrained_model_name_or_path, fileName, * @param {import('./hub.js').PretrainedModelOptions} options Additional options for loading the model. * @param {import('./hub.js').ExternalData|Record|undefined} use_external_data_format External data format configuration. * @param {any} [session_options] Optional session options that may contain externalData configuration. - * @returns {Promise>} A Promise that resolves to an array of external data files. + * @returns {Promise>} A Promise that resolves to an array of external data files. */ export async function getModelDataFiles( pretrained_model_name_or_path, @@ -70,7 +70,7 @@ export async function getModelDataFiles( const baseName = `${fileName}${suffix}.onnx`; const return_path = apis.IS_NODE_ENV; - /** @type {Promise[]} */ + /** @type {Promise[]} */ let externalDataPromises = []; const num_chunks = resolveExternalDataFormat(use_external_data_format, baseName, fileName); @@ -91,8 +91,11 @@ export async function getModelDataFiles( true, options, return_path, + // In the browser, hand onnxruntime-web a Blob rather than a materialised buffer. + // Node keeps returning a path, which is cheaper still. + !return_path, ); - resolve(data instanceof Uint8Array ? { path, data } : path); + resolve(data instanceof Uint8Array || data instanceof Blob ? { path, data } : path); }), ); } From 12cebf1cd3ab1a3400a4e9c9b6e5b70cd1bfe675 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 14 Aug 2026 17:17:34 +0200 Subject: [PATCH 3/9] Stream a cold external-data download into the cache instead of buffering it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit made a WARM load cheap: a cached file comes back as a Blob and its bytes never reach the JS heap. A cold one was untouched, and a cold one is the case that fails. `getModelDataFiles` starts every external-data chunk concurrently, and each is read into a `Uint8Array` so download progress can be reported — so a first load peaks at the SUM of the chunks rather than the largest. Measured on a 17 GB model from an empty cache: `Array buffer allocation failed` at 16.2 of 17.0 GB, 13 GB cached, load failed. It succeeds on a later attempt only because each file is cached as it completes, so the retry has less left to buffer. The body can go straight into Cache Storage instead. `cache.put` takes a `Response` and writes it to disk without materialising it, and reading it back with `cache.match` gives the same Blob a warm load gets — so the bytes travel network -> disk -> runtime and never sit on the heap. Progress survives, which is the only reason the buffer existed: a pass-through `TransformStream` counts bytes as they go past. It holds one chunk, not the file, and backpressure keeps it that way. If the cache refuses the write — QuotaExceededError being the expected one — it falls back to the buffered path. That has to re-fetch rather than reuse the response, whose body the failed attempt consumed. --- packages/transformers/src/utils/hub.js | 53 ++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index 0daf06bf7..bf41dd00e 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -388,6 +388,59 @@ export async function loadResourceFile( loaded: buffer.size, total: buffer.size, }); + } else if (as_blob && toCacheResponse && response.body) { + // COLD, and headed for the cache anyway. Stream the body straight into Cache Storage and then + // read it back as a Blob, so the bytes go network -> disk -> runtime and never sit on the JS + // heap at all. + // + // This is the case that actually fails. `getModelDataFiles` starts every external-data chunk + // concurrently, and reading each one into a `Uint8Array` to report progress means a cold load + // peaks at the SUM of the chunks: a 17 GB model raises `Array buffer allocation failed` at + // ~16 GB and only completes on a later attempt, once enough files are cached to take the + // branch above. + // + // Progress survives, which is the reason the buffer existed. A pass-through `TransformStream` + // counts bytes as they go by; it holds one chunk, not the file, and backpressure keeps it + // that way. + let loaded = 0; + const total = parseInt(response.headers.get('content-length'), 10) || 0; + const counting = new TransformStream({ + transform(chunk, controller) { + loaded += chunk.byteLength; + dispatchCallback(options.progress_callback, { + status: 'progress', + name: path_or_repo_id, + file: filename, + progress: total ? (loaded / total) * 100 : 0, + loaded, + total, + }); + controller.enqueue(chunk); + }, + }); + + // `content-length` explicitly, because the Cache API may strip it — same reason the buffered + // store below sets it. + const headers = new Headers(response.headers); + if (total) headers.set('content-length', String(total)); + + try { + await cache.put(cacheKey, new Response(response.body.pipeThrough(counting), { headers })); + const stored = await cache.match(cacheKey); + if (!stored) throw new Error('cache.match missed the entry just written'); + buffer = /** @type {any} */ (await stored.blob()); + // Already stored, so the block at the end of this function must not store it again. + toCacheResponse = false; + } catch (err) { + // The buffered path keeps working when the cache refuses the write (QuotaExceededError is + // the expected one). It cannot reuse `response` — the body is consumed — so it re-fetches. + logger.warn(`Unable to stream response into the cache, falling back to a buffer: ${err}.`); + // `getFile`, not bare `fetch`: it routes through `env.fetch` and applies + // `getFetchHeaders`, so a gated repo keeps its Authorization header on the way back. + const retry = await getFile(remoteURL); + if (retry.status !== 200) return handleError(retry.status, remoteURL, fatal); + buffer = new Uint8Array(await retry.arrayBuffer()); + } } else if (!options.progress_callback) { // If no progress callback is specified, we can use the `.arrayBuffer()` // method to read the response. From 96281e5ead5ef6a69e2c0d6904f33fc79a404553 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 27 Aug 2026 23:36:25 +0200 Subject: [PATCH 4/9] Update the lockfile for the onnxruntime-web bump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm install --frozen-lockfile` fails on Node 18, 20 and 22 — eight seconds in, before anything is built or tested — because the ORT bump changed `packages/transformers/package.json` and left `pnpm-lock.yaml` pinning 1.26.0-dev.20260416-b7804b056c. Regenerated with `pnpm install --lockfile-only`, so the change is the resolution and nothing else. --- pnpm-lock.yaml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bea34792e..cd1eca67c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,8 +27,8 @@ importers: specifier: 1.24.3 version: 1.24.3 onnxruntime-web: - specifier: 1.26.0-dev.20260416-b7804b056c - version: 1.26.0-dev.20260416-b7804b056c + specifier: 1.29.0-dev.20260811-e415ef9afd + version: 1.29.0-dev.20260811-e415ef9afd sharp: specifier: ^0.34.5 version: 0.34.5 @@ -760,6 +760,7 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} @@ -1253,11 +1254,12 @@ packages: glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-agent@3.0.0: resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} @@ -1685,18 +1687,18 @@ packages: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} - onnxruntime-common@1.24.0-dev.20251116-b39e144322: - resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} - onnxruntime-common@1.24.3: resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} + onnxruntime-common@1.29.0-dev.20260723-1b1e1db7bc: + resolution: {integrity: sha512-MtH74K5iL3PZZrhVwElASZKy23yBlBwDi/J1Z+KrcwiVrOUDQ/rB3OcL944/t/40b9kHHFQQ/67FOetzQChvhQ==} + onnxruntime-node@1.24.3: resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} os: [win32, darwin, linux] - onnxruntime-web@1.26.0-dev.20260416-b7804b056c: - resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} + onnxruntime-web@1.29.0-dev.20260811-e415ef9afd: + resolution: {integrity: sha512-HYm3DQkPEGeCI83Zl1thEdSIthaH00Ew1VR+SrEuQU61KDa9d4UwmnXpFvdDBmL31LE5i/6TnNgPaGecBd9sWA==} p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} @@ -3823,22 +3825,22 @@ snapshots: dependencies: mimic-fn: 2.1.0 - onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} - onnxruntime-common@1.24.3: {} + onnxruntime-common@1.29.0-dev.20260723-1b1e1db7bc: {} + onnxruntime-node@1.24.3: dependencies: adm-zip: 0.5.16 global-agent: 3.0.0 onnxruntime-common: 1.24.3 - onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + onnxruntime-web@1.29.0-dev.20260811-e415ef9afd: dependencies: flatbuffers: 25.9.23 guid-typescript: 1.0.9 long: 5.3.2 - onnxruntime-common: 1.24.0-dev.20251116-b39e144322 + onnxruntime-common: 1.29.0-dev.20260723-1b1e1db7bc platform: 1.3.6 protobufjs: 7.5.4 From 60e92d5b99cbd6ffcaa790dc4542843176310a7e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 27 Aug 2026 23:40:32 +0200 Subject: [PATCH 5/9] Include `as_blob` in the in-flight dedup key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `as_blob` changes the RESOLVED TYPE of `getModelFile`, not just how the bytes are fetched, so two callers asking for the same file with different values are not asking the same question. Left out of the key, whichever one arrives first decides for both and the other receives a `Blob` where it expects a `Uint8Array`, or the reverse — silently, and only under concurrency. `return_path` is in the key already for exactly this reason. Adds focused warm/cold tests against an in-memory cache and a stubbed `env.fetch`, so they assert which branch ran rather than wall-clock behaviour and touch no network: · cold — streams into the cache, resolves a Blob, stores exactly once (not a second time by the block at the end of loadResourceFile), and still reports progress up to the total, which is the reason the buffer existed; · warm — resolves a Blob from the cache without re-fetching, and reports the single completed event; · a cache that refuses the write falls back to a buffer and re-fetches, rather than failing the load; · `as_blob` unset still resolves a Uint8Array; · both orderings of a concurrent Blob/bytes pair get the type they asked for. The last two fail without the key change and pass with it — checked by reverting it. --- packages/transformers/src/utils/hub.js | 7 +- .../tests/utils/hub_external_data.test.js | 138 ++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 packages/transformers/tests/utils/hub_external_data.test.js diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index bf41dd00e..718c580a8 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -602,7 +602,12 @@ export async function getModelFile( // sibling calls within one pipeline() share a single download (and a // single `initiate` event). Otherwise fall back to the global in-flight // map for concurrent dedup, with entries cleared on settle. - const key = makePretrainedOptionsKey(path_or_repo_id, options, filename, fatal, return_path); + // + // `as_blob` is part of the key because it changes the RESOLVED TYPE, not just how the bytes are fetched. + // Two callers asking for the same file with different values are not asking the same question: without + // it, whichever arrives first decides, and the other gets a `Blob` where it expects a `Uint8Array` or the + // reverse. `return_path` is in the key for exactly this reason already. + const key = makePretrainedOptionsKey(path_or_repo_id, options, filename, fatal, return_path, as_blob); const { progress_callback } = options; const loads = progress_callback instanceof DefaultProgressCallback ? progress_callback.loads : INFLIGHT_LOADS; let pending = loads.get(key); diff --git a/packages/transformers/tests/utils/hub_external_data.test.js b/packages/transformers/tests/utils/hub_external_data.test.js new file mode 100644 index 000000000..d4d14d30c --- /dev/null +++ b/packages/transformers/tests/utils/hub_external_data.test.js @@ -0,0 +1,138 @@ +import { env } from "../../src/env.js"; +import { getModelFile } from "../../src/utils/hub.js"; + +/** + * The two things `as_blob` has to get right are a COLD file and a WARM one, and they take different branches: + * cold streams the body into the cache and reads it back, warm reads the cached `Response` directly. Both are + * exercised here against an in-memory cache and a stubbed `env.fetch`, so nothing touches the network and the + * assertions can be about which branch ran rather than about wall-clock behaviour. + */ +class MemoryCache { + constructor() { + /** @type {Map} */ + this.entries = new Map(); + this.puts = 0; + /** @type {Error|null} Set to make `put` reject, standing in for QuotaExceededError. */ + this.putError = null; + } + + async match(key) { + const hit = this.entries.get(key); + return hit ? hit.clone() : undefined; + } + + async put(key, response) { + this.puts++; + if (this.putError) throw this.putError; + // Read the body here rather than storing the streaming `Response`: a real Cache Storage entry is settled + // by the time `match` can see it, and holding the live stream would let a test pass on a tee that a real + // cache would never hand back. + this.entries.set(key, new Response(await response.arrayBuffer(), { headers: response.headers })); + } +} + +const BYTES = new Uint8Array(4096).fill(7); + +describe("External data as a Blob", () => { + const saved = {}; + let cache; + let fetches; + + beforeEach(() => { + for (const k of ["useCustomCache", "customCache", "useBrowserCache", "useFSCache", "allowLocalModels", "fetch"]) { + saved[k] = env[k]; + } + cache = new MemoryCache(); + fetches = 0; + env.useCustomCache = true; + env.customCache = cache; + env.useBrowserCache = false; + env.useFSCache = false; + env.allowLocalModels = false; + env.fetch = async () => { + ++fetches; + return new Response(BYTES, { + status: 200, + headers: { "content-length": String(BYTES.length), "content-type": "application/octet-stream" }, + }); + }; + }); + + afterEach(() => { + for (const [k, v] of Object.entries(saved)) env[k] = v; + }); + + const load = (as_blob, options = {}, filename = "onnx/model.onnx_data") => + getModelFile("hf-internal-testing/blob-external-data", filename, true, options, false, as_blob); + + it("streams a cold file into the cache and resolves a Blob, storing it once", async () => { + const progress = []; + const out = await load(true, { progress_callback: (e) => e.status === "progress" && progress.push(e) }); + + expect(out).toBeInstanceOf(Blob); + expect(out.size).toBe(BYTES.length); + expect(cache.puts).toBe(1); // and NOT stored a second time by the block at the end of loadResourceFile + expect(fetches).toBe(1); + + // Progress survives the streaming path — it is the reason the buffer existed, so losing it silently would + // be the regression that matters most to a user watching gigabytes arrive. + expect(progress.length).toBeGreaterThan(0); + expect(progress.at(-1).loaded).toBe(BYTES.length); + expect(progress.at(-1).total).toBe(BYTES.length); + }); + + it("reads a warm file straight out of the cache without fetching again", async () => { + await load(true); + expect(fetches).toBe(1); + + const progress = []; + const out = await load(true, { progress_callback: (e) => e.status === "progress" && progress.push(e) }); + + expect(out).toBeInstanceOf(Blob); + expect(out.size).toBe(BYTES.length); + expect(fetches).toBe(1); // served from the cache + expect(cache.puts).toBe(1); // and not rewritten + // The warm branch reports one completed event rather than a stream of them. + expect(progress.at(-1)).toMatchObject({ progress: 100, loaded: BYTES.length, total: BYTES.length }); + }); + + it("falls back to a buffer when the cache refuses the write", async () => { + cache.putError = Object.assign(new Error("quota"), { name: "QuotaExceededError" }); + + const out = await load(true); + + // A cache that cannot accept the file must not fail the load — it costs the memory saving, nothing else. + expect(out).toBeInstanceOf(Uint8Array); + expect(out.length).toBe(BYTES.length); + // Re-fetched, because the streaming attempt consumed the first body. + expect(fetches).toBe(2); + }); + + it("still resolves a Uint8Array when as_blob is not asked for", async () => { + const out = await load(false); + expect(out).toBeInstanceOf(Uint8Array); + expect(out.length).toBe(BYTES.length); + }); + + // ⛔ THE REGRESSION THIS EXISTS FOR. `as_blob` changes the RESOLVED TYPE, so it has to be part of the + // in-flight dedup key. Without it whichever caller arrives first decides for both, and the other gets a + // `Blob` where it expects a `Uint8Array` or the reverse — silently, and only under concurrency. + it("does not hand a Blob to a concurrent caller that asked for bytes", async () => { + await load(true); // warm, so both calls below take the same branch and race deterministically + + const [asBlob, asBytes] = await Promise.all([load(true), load(false)]); + + expect(asBlob).toBeInstanceOf(Blob); + expect(asBytes).toBeInstanceOf(Uint8Array); + }); + + it("does not hand bytes to a concurrent caller that asked for a Blob", async () => { + await load(true); + + // The reverse order, because a key bug is only visible in whichever direction loses the race. + const [asBytes, asBlob] = await Promise.all([load(false), load(true)]); + + expect(asBytes).toBeInstanceOf(Uint8Array); + expect(asBlob).toBeInstanceOf(Blob); + }); +}); From a731c00bf5362698e50b6e516638e7e1f39c8c83 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 27 Aug 2026 23:41:19 +0200 Subject: [PATCH 6/9] Say which half of the memory saving is unconditional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review is right that the claim and the default path do not match, and the reason is that this PR has TWO savings which the comments ran together: · The DOWNLOAD no longer buffers. Streaming the body into Cache Storage and reading it back means the file never sits on the JS heap on its way in. This happens before onnxruntime-web sees anything and holds whichever build is in use. It is the half that stops a cold load of a model with several multi-gigabyte chunks dying in getModelDataFiles, where every chunk starts concurrently and the peak is their sum. · The SESSION may or may not copy. Handing the runtime a Blob only avoids materialising it if the runtime can read it that way: the JSPI build can, the default asyncify build cannot. So "the bytes never sit on the JS heap at all" was true of the download and overstated end to end. The comments now say which is which, and say plainly that on the default build this buys the download and not the session. No behaviour change — comments only. --- packages/transformers/src/utils/hub.js | 16 +++++++++++++--- packages/transformers/src/utils/model-loader.js | 5 +++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index 718c580a8..ebd59e039 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -371,7 +371,9 @@ export async function loadResourceFile( if (typeof response !== 'string') { if (as_blob && cacheHit) { // The whole point of `as_blob`, and it is one call: a Cache Storage `Response` hands back a - // Blob that is a FILE REFERENCE rather than a copy, so the bytes never enter the JS heap. + // Blob that is a FILE REFERENCE rather than a copy, so reading the file does not put it on + // the JS heap. What the RUNTIME then does with that Blob is a separate question — see the + // note on the cold branch below. // // Gated on `cacheHit`, which is a deliberate trade rather than caution. Reading the stream to // report progress would defeat the whole thing — the chunks would be resident twice, once as @@ -390,8 +392,16 @@ export async function loadResourceFile( }); } else if (as_blob && toCacheResponse && response.body) { // COLD, and headed for the cache anyway. Stream the body straight into Cache Storage and then - // read it back as a Blob, so the bytes go network -> disk -> runtime and never sit on the JS - // heap at all. + // read it back as a Blob, so the DOWNLOAD goes network -> disk without the file ever sitting + // on the JS heap. + // + // ⚠️ TWO SEPARATE SAVINGS, AND ONLY ONE OF THEM IS UNCONDITIONAL. This branch is about the + // download, happens before onnxruntime-web sees anything, and holds whichever build is in + // use — it is what stops a cold multi-gigabyte load dying in `getModelDataFiles` below. What + // the runtime does with the Blob afterwards is the other saving and is NOT unconditional: the + // JSPI build reads external data from a Blob without materialising it, while the default + // asyncify build copies it in. So on the default build this buys the download and not the + // session, and the end-to-end peak is unchanged at session creation. // // This is the case that actually fails. `getModelDataFiles` starts every external-data chunk // concurrently, and reading each one into a `Uint8Array` to report progress means a cold load diff --git a/packages/transformers/src/utils/model-loader.js b/packages/transformers/src/utils/model-loader.js index 5c137342b..c0a74b590 100644 --- a/packages/transformers/src/utils/model-loader.js +++ b/packages/transformers/src/utils/model-loader.js @@ -93,6 +93,11 @@ export async function getModelDataFiles( return_path, // In the browser, hand onnxruntime-web a Blob rather than a materialised buffer. // Node keeps returning a path, which is cheaper still. + // + // Accepting a Blob is safe on either web build; only the JSPI one avoids copying it + // into the heap at session creation. The unconditional win is upstream of here — the + // download no longer buffers each chunk, which is what a cold load of a model with + // several multi-gigabyte chunks runs out of memory doing. !return_path, ); resolve(data instanceof Uint8Array || data instanceof Blob ? { path, data } : path); From 5c5578b91606e0f4c744207156121bf5f8c3eef9 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 27 Aug 2026 23:42:36 +0200 Subject: [PATCH 7/9] Format the new test with the repo's prettier --- packages/transformers/tests/utils/hub_external_data.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/transformers/tests/utils/hub_external_data.test.js b/packages/transformers/tests/utils/hub_external_data.test.js index d4d14d30c..59849bb1a 100644 --- a/packages/transformers/tests/utils/hub_external_data.test.js +++ b/packages/transformers/tests/utils/hub_external_data.test.js @@ -62,8 +62,7 @@ describe("External data as a Blob", () => { for (const [k, v] of Object.entries(saved)) env[k] = v; }); - const load = (as_blob, options = {}, filename = "onnx/model.onnx_data") => - getModelFile("hf-internal-testing/blob-external-data", filename, true, options, false, as_blob); + const load = (as_blob, options = {}, filename = "onnx/model.onnx_data") => getModelFile("hf-internal-testing/blob-external-data", filename, true, options, false, as_blob); it("streams a cold file into the cache and resolves a Blob, storing it once", async () => { const progress = []; From a04faae22476e1fe1d4ee35c148f7bf4dbcee2b0 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 27 Aug 2026 23:45:58 +0200 Subject: [PATCH 8/9] Make `tsc --build` pass with the Blob type widened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm build` runs `tsc --build`, and it has been failing on this branch since the Blob was introduced — five errors, unnoticed because CI never got past `pnpm install`. Fixing the lockfile in the first commit would have moved the red from install to build rather than clearing it. · `buffer` was annotated `Uint8Array` and now legitimately holds a Blob on both `as_blob` branches. Widened rather than cast away at each use, so a future branch that forgets which it is fails at build. · `cache.match` is typed for every backend including the Node file cache; only a `Response` has `.blob()`, and only a `Response` reaches that line. Cast instead of the blanket `any` it had. · `INFLIGHT_LOADS` holds Blob-resolving promises now. · `getCoreModelFile` never passes `as_blob`, so a Blob cannot come back; narrowed there rather than widening every caller of a function that only ever wants bytes or a path. · The store-at-the-end block cannot see a Blob either — a warm read is a cache hit, which never sets `toCacheResponse`, and a cold stream writes the entry itself and then clears the flag. The narrowing says so, so a third branch that produced a Blob without storing it would land there loudly. tsc --build is clean, the bundles build, and tests/utils is 296 passing. --- packages/transformers/src/utils/hub.js | 19 ++++++++++++++----- .../transformers/src/utils/model-loader.js | 6 +++++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index ebd59e039..572f1881d 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -365,7 +365,9 @@ export async function loadResourceFile( // loads from disk directly). A completion progress event is emitted // after the caching block below to ensure progress_total reaches 100%. } else { - /** @type {Uint8Array} */ + // A `Blob` on the `as_blob` branches, bytes everywhere else. Widened rather than cast away at each + // use, so a future branch that forgets which it is fails here instead of at run time. + /** @type {Uint8Array|Blob} */ let buffer; if (typeof response !== 'string') { @@ -380,7 +382,7 @@ export async function loadResourceFile( // buffers and once in Blob storage — and on a cold download progress is worth more than peak // memory, because the user is watching several gigabytes arrive. Every load AFTER the first // takes this branch, is instantaneous, and is where the peak actually matters. - buffer = /** @type {any} */ (await response.blob()); + buffer = await response.blob(); dispatchCallback(options.progress_callback, { status: 'progress', @@ -438,7 +440,9 @@ export async function loadResourceFile( await cache.put(cacheKey, new Response(response.body.pipeThrough(counting), { headers })); const stored = await cache.match(cacheKey); if (!stored) throw new Error('cache.match missed the entry just written'); - buffer = /** @type {any} */ (await stored.blob()); + // `cache.match` is typed for every backend, including the Node file cache; what was just + // written here is a `Response`, and only a `Response` has `.blob()`. + buffer = await /** @type {Response} */ (stored).blob(); // Already stored, so the block at the end of this function must not store it again. toCacheResponse = false; } catch (err) { @@ -516,7 +520,12 @@ export async function loadResourceFile( cacheKey && typeof response !== 'string' ) { - await storeCachedResource(path_or_repo_id, filename, cache, cacheKey, response, result, options); + // ⛔ NEVER A BLOB HERE, and the two `as_blob` branches are why. A warm read is a cache HIT, which + // never sets `toCacheResponse`; a cold stream writes the entry itself and then clears the flag. So + // anything reaching this line was buffered, and the narrowing is a statement of that rather than a + // convenience — if a third branch ever produces a Blob without storing it, this is where it lands. + const buffered = /** @type {Uint8Array} */ (result); + await storeCachedResource(path_or_repo_id, filename, cache, cacheKey, response, buffered, options); } // In Node.js with return_path, the buffer read is skipped so no progress @@ -565,7 +574,7 @@ export async function loadResourceFile( throw new Error('Unable to get model file path or buffer.'); } -/** @type {Map>} Pending file loads keyed by resource identity. */ +/** @type {Map>} Pending file loads keyed by resource identity. */ const INFLIGHT_LOADS = new Map(); /** diff --git a/packages/transformers/src/utils/model-loader.js b/packages/transformers/src/utils/model-loader.js index c0a74b590..d9f0c858a 100644 --- a/packages/transformers/src/utils/model-loader.js +++ b/packages/transformers/src/utils/model-loader.js @@ -45,7 +45,11 @@ export async function getCoreModelFile(pretrained_model_name_or_path, fileName, const baseName = `${fileName}${suffix}.onnx`; const fullPath = `${options.subfolder ?? ''}/${baseName}`; - return await getModelFile(pretrained_model_name_or_path, fullPath, true, options, apis.IS_NODE_ENV); + // Not `as_blob`, so a `Blob` cannot come back — narrowed here rather than widening every caller of a + // function that only ever asks for bytes or a path. + return /** @type {string|Uint8Array} */ ( + await getModelFile(pretrained_model_name_or_path, fullPath, true, options, apis.IS_NODE_ENV) + ); } /** From face029db94d67de5428fefa27c346b84f8f046c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 28 Aug 2026 11:27:12 +0200 Subject: [PATCH 9/9] Document both paths `as_blob` is honoured on, and test the download half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review points that the last round left open. `as_blob`'s documentation still said cache-hit-only on both `loadResourceFile` and `getModelFile`, which stopped being true when the cold path started streaming into the cache and returning the new Blob in the same call. Both now name the two paths it IS honoured on, and say that a Uint8Array comes back otherwise — a cold file with no cache, and the fallback when the cache refuses the write — so callers know they have to handle both. And the asyncify/JSPI split is now tested, not just asserted in a comment. Whether onnxruntime-web copies the Blob at session creation depends on its build and is not observable from this layer; what is observable, and is the saving this file is responsible for, is that the DOWNLOAD never materialises the body. The new case spies on the fetched response and asserts `arrayBuffer()` is never called on the streaming path — paired with the buffered path, where the same spy must SEE the call, so it cannot pass by being wired to the wrong object. --- packages/transformers/src/utils/hub.js | 16 ++++++---- .../tests/utils/hub_external_data.test.js | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/transformers/src/utils/hub.js b/packages/transformers/src/utils/hub.js index 572f1881d..3f8ba5109 100755 --- a/packages/transformers/src/utils/hub.js +++ b/packages/transformers/src/utils/hub.js @@ -245,11 +245,14 @@ export async function storeCachedResource(path_or_repo_id, filename, cache, cach * @param {PretrainedOptions} [options] An object containing optional parameters. * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. * @param {import('./cache.js').CacheInterface | null} [cache] The cache instance to use. - * @param {boolean} [as_blob=false] Whether to return a `Blob` when the file is served from the cache, - * rather than reading it into a `Uint8Array`. Only honoured on a cache hit — see the note at the use site. + * @param {boolean} [as_blob=false] Whether to return a `Blob` rather than reading the file into a + * `Uint8Array`. Honoured on two paths: a cache HIT, which reads the cached `Response` directly, and a COLD + * file that is headed for the cache anyway, which is streamed into it and read straight back. Ignored — and a + * `Uint8Array` returned — when neither applies, which covers a cold file with no cache available and the + * fallback taken when the cache refuses the write. * * @throws Will throw an error if the file is not found and `fatal` is true. - * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache. + * @returns {Promise} A Promise that resolves with the file path as a string if `return_path` is true; otherwise a `Blob` when `as_blob` was honoured, and the file content as a `Uint8Array` in every other case. */ export async function loadResourceFile( path_or_repo_id, @@ -588,11 +591,12 @@ const INFLIGHT_LOADS = new Map(); * @param {boolean} [fatal=true] Whether to throw an error if the file is not found. * @param {PretrainedOptions} [options] An object containing optional parameters. * @param {boolean} [return_path=false] Whether to return the path of the file instead of the file content. - * @param {boolean} [as_blob=false] Whether to accept a `Blob` for a file served from the cache, avoiding a - * copy of its bytes on the JS heap. Intended for external data, which is handed straight to the runtime. + * @param {boolean} [as_blob=false] Whether to accept a `Blob` instead of a copy of the file's bytes on the JS + * heap. Intended for external data, which is handed straight to the runtime. Honoured for a cached file and + * for a cold one being written to the cache; a `Uint8Array` comes back otherwise, so callers must handle both. * * @throws Will throw an error if the file is not found and `fatal` is true. - * @returns {Promise} A Promise that resolves with the file content as a Uint8Array if `return_path` is false, or the file path as a string if `return_path` is true. Resolves with a `Blob` when `as_blob` is set and the file came from the cache. + * @returns {Promise} A Promise that resolves with the file path as a string if `return_path` is true; otherwise a `Blob` when `as_blob` was honoured, and the file content as a `Uint8Array` in every other case. */ export async function getModelFile( path_or_repo_id, diff --git a/packages/transformers/tests/utils/hub_external_data.test.js b/packages/transformers/tests/utils/hub_external_data.test.js index 59849bb1a..99c5e141a 100644 --- a/packages/transformers/tests/utils/hub_external_data.test.js +++ b/packages/transformers/tests/utils/hub_external_data.test.js @@ -113,6 +113,35 @@ describe("External data as a Blob", () => { expect(out.length).toBe(BYTES.length); }); + // ⛔ THE HALF OF THE MEMORY CLAIM THIS LAYER CAN ACTUALLY TEST. Whether onnxruntime-web copies the Blob at + // session creation depends on its build — JSPI does not, the default asyncify one does — and that is not + // observable from here. What IS observable, and is the saving this file is responsible for, is that the + // DOWNLOAD never materialises the body: no `arrayBuffer()` on the fetched response, on any build. + // + // Paired with the negative case on purpose. Asserting "never called" alone would pass just as happily if + // the spy were wired to the wrong object, so the same spy has to SEE the call on the buffered path. + it("streams a cold file without materialising the response, and buffers it when not asked for a Blob", async () => { + let materialised = 0; + const inner = env.fetch; + env.fetch = async (...args) => { + const response = await inner(...args); + const original = response.arrayBuffer.bind(response); + response.arrayBuffer = async () => { + ++materialised; + return original(); + }; + return response; + }; + + expect(await load(true)).toBeInstanceOf(Blob); + expect(materialised).toBe(0); + + // Same spy, same fetch, `as_blob` off: the buffered path reads the body onto the heap, which is what the + // streaming branch is avoiding — and proves the assertion above is watching the right object. + expect(await load(false, {}, "onnx/other.onnx_data")).toBeInstanceOf(Uint8Array); + expect(materialised).toBe(1); + }); + // ⛔ THE REGRESSION THIS EXISTS FOR. `as_blob` changes the RESOLVED TYPE, so it has to be part of the // in-flight dedup key. Without it whichever caller arrives first decides for both, and the other gets a // `Blob` where it expects a `Uint8Array` or the reverse — silently, and only under concurrency.