From 78594f36057e67f7bd5ae08e9f6b95d4c804af2e Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 19 Aug 2026 15:49:02 +0530 Subject: [PATCH] fix: retry caption requests across processor slots --- platform/src/lib/youtube-processor-client.ts | 28 ++++- .../test/youtube-processor-client.test.ts | 111 ++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/platform/src/lib/youtube-processor-client.ts b/platform/src/lib/youtube-processor-client.ts index 9d7c175..9514644 100644 --- a/platform/src/lib/youtube-processor-client.ts +++ b/platform/src/lib/youtube-processor-client.ts @@ -148,6 +148,22 @@ function failureFrom(payload: unknown): ProcessorFailure['error'] | undefined { return payload.error; } +function shouldFallbackResult(operation: YouTubeOperation, result: unknown): boolean { + if (operation.kind !== 'caption-tracks' || !isRecord(result)) return false; + const metadata = result.meta; + return Array.isArray(result.tracks) + && result.tracks.length === 0 + && isRecord(metadata) + && metadata.partial === true; +} + +function shouldFallbackError(operation: YouTubeOperation, error: YouTubeProcessorError): boolean { + // A transcript NOT_FOUND can mean that one YouTube response omitted its + // caption catalog. Try an independent processor slot before treating it as + // a genuine captionless video. + return operation.kind === 'transcript' && error.code === 'NOT_FOUND'; +} + async function resultFrom(response: Response): Promise { let payload: unknown; try { @@ -194,6 +210,7 @@ export async function runYouTubeOperation( for (let index = 0; index < slots.length; index += 1) { const slot = slots[index]!; const startedAt = Date.now(); + const hasFallback = index < slots.length - 1; try { const response = await processorContainer(env, slot).fetch(new Request('http://youtube-processor/operations', { method: 'POST', @@ -202,7 +219,6 @@ export async function runYouTubeOperation( signal: AbortSignal.timeout(processorTimeoutMs(env)), })); - const hasFallback = index < slots.length - 1; if (hasFallback && RETRYABLE_CONTAINER_STATUSES.has(response.status)) { logProcessorAttempt(operation.kind, slot, index, response.status, 'fallback', startedAt); if (response.body) await response.body.cancel().catch(() => undefined); @@ -211,10 +227,20 @@ export async function runYouTubeOperation( } const result = await resultFrom>(response); + if (hasFallback && shouldFallbackResult(operation, result)) { + logProcessorAttempt(operation.kind, slot, index, response.status, 'fallback', startedAt); + await waitBeforeFallback(env, index); + continue; + } logProcessorAttempt(operation.kind, slot, index, response.status, 'success', startedAt); return result; } catch (error) { if (error instanceof YouTubeProcessorError) { + if (hasFallback && shouldFallbackError(operation, error)) { + logProcessorAttempt(operation.kind, slot, index, error.status, 'fallback', startedAt); + await waitBeforeFallback(env, index); + continue; + } logProcessorAttempt(operation.kind, slot, index, error.status, 'processor-error', startedAt); throw error; } diff --git a/platform/test/youtube-processor-client.test.ts b/platform/test/youtube-processor-client.test.ts index 75b835a..46a719e 100644 --- a/platform/test/youtube-processor-client.test.ts +++ b/platform/test/youtube-processor-client.test.ts @@ -48,6 +48,117 @@ describe('YouTube processor client', () => { expect(requested[0]).not.toBe(requested[1]); }); + test('fails over when caption tracks are empty and partial', async () => { + const operation = { kind: 'caption-tracks', id: 'abcdefghijk' } satisfies YouTubeOperation; + const empty = { + tracks: [], sourceTracks: [], translationLanguages: [], autoTranslationTargets: [], + meta: { source: 'allthingsyoutube', fetchedAt: '2026-08-19T00:00:00.000Z', partial: true, warnings: [] }, + }; + const complete = { + tracks: [{ id: 'a.en', name: 'English', languageCode: 'en', kind: 'asr', provenance: 'asr' }], + sourceTracks: [{ id: 'a.en', name: 'English', languageCode: 'en', kind: 'asr', provenance: 'asr' }], + translationLanguages: [], autoTranslationTargets: [], + meta: { source: 'allthingsyoutube', fetchedAt: '2026-08-19T00:00:01.000Z', partial: false, warnings: [] }, + }; + const { env, requested } = environment([ + Response.json({ value: empty }), + Response.json({ value: complete }), + ]); + + await expect(runYouTubeOperation(env, operation)).resolves.toMatchObject({ + tracks: [{ id: 'a.en' }], + meta: { partial: false }, + }); + expect(requested).toHaveLength(2); + expect(requested[0]).not.toBe(requested[1]); + }); + + test('preserves empty caption tracks after every slot returns a partial result', async () => { + const operation = { kind: 'caption-tracks', id: 'abcdefghijk' } satisfies YouTubeOperation; + const empty = () => Response.json({ value: { + tracks: [], sourceTracks: [], translationLanguages: [], autoTranslationTargets: [], + meta: { source: 'allthingsyoutube', fetchedAt: '2026-08-19T00:00:00.000Z', partial: true, warnings: [] }, + } }); + const { env, requested } = environment([empty(), empty()]); + + await expect(runYouTubeOperation(env, operation)).resolves.toMatchObject({ + tracks: [], + meta: { partial: true }, + }); + expect(requested).toHaveLength(2); + }); + + test('does not fail over a complete empty caption result', async () => { + const operation = { kind: 'caption-tracks', id: 'abcdefghijk' } satisfies YouTubeOperation; + const { env, requested } = environment([ + Response.json({ value: { + tracks: [], sourceTracks: [], translationLanguages: [], autoTranslationTargets: [], + meta: { source: 'allthingsyoutube', fetchedAt: '2026-08-19T00:00:00.000Z', partial: false, warnings: [] }, + } }), + Response.json({ value: { tracks: [{ id: 'a.en' }] } }), + ]); + + await expect(runYouTubeOperation(env, operation)).resolves.toMatchObject({ + tracks: [], + meta: { partial: false }, + }); + expect(requested).toHaveLength(1); + }); + + test('fails over when a transcript slot reports missing captions', async () => { + const operation = { + kind: 'transcript', id: 'abcdefghijk', granularity: 'word', + } satisfies YouTubeOperation; + const { env, requested } = environment([ + Response.json({ error: { + code: 'NOT_FOUND', message: 'No caption track is available.', status: 404, retryable: false, + } }, { status: 404 }), + Response.json({ value: { + videoId: 'abcdefghijk', + track: { id: 'a.en', name: 'English', languageCode: 'en', kind: 'asr', provenance: 'asr' }, + segments: [], granularity: 'word', text: 'Recovered transcript', + meta: { source: 'allthingsyoutube', fetchedAt: '2026-08-19T00:00:01.000Z', partial: false, warnings: [] }, + } }), + ]); + + await expect(runYouTubeOperation(env, operation)).resolves.toMatchObject({ + videoId: 'abcdefghijk', + text: 'Recovered transcript', + }); + expect(requested).toHaveLength(2); + expect(requested[0]).not.toBe(requested[1]); + }); + + test('preserves missing captions after every transcript slot agrees', async () => { + const operation = { + kind: 'transcript', id: 'abcdefghijk', granularity: 'word', + } satisfies YouTubeOperation; + const missing = () => Response.json({ error: { + code: 'NOT_FOUND', message: 'No caption track is available.', status: 404, retryable: false, + } }, { status: 404 }); + const { env, requested } = environment([missing(), missing()]); + + await expect(runYouTubeOperation(env, operation)).rejects.toMatchObject({ + code: 'NOT_FOUND', status: 404, retryable: false, + }); + expect(requested).toHaveLength(2); + }); + + test('does not fail over unrelated not-found errors', async () => { + const operation = { kind: 'video', id: 'abcdefghijk' } satisfies YouTubeOperation; + const { env, requested } = environment([ + Response.json({ error: { + code: 'NOT_FOUND', message: 'Video not found.', status: 404, retryable: false, + } }, { status: 404 }), + Response.json({ value: { id: 'abcdefghijk' } }), + ]); + + await expect(runYouTubeOperation(env, operation)).rejects.toMatchObject({ + code: 'NOT_FOUND', status: 404, + }); + expect(requested).toHaveLength(1); + }); + test('preserves structured processor errors', async () => { const operation = { kind: 'video', id: 'abcdefghijk' } satisfies YouTubeOperation; const failure = Response.json({ error: {