From c62b1e55e1ae570d4f90e4edd0bf2318d1a4ac5d Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Fri, 31 Jul 2026 00:17:11 +0700 Subject: [PATCH 1/9] fix(player): restore safe sentence audio export - fall back only when the source has exactly one audio stream - allow audio export from HDR video sources - show clear warnings when sentence audio generation or Anki storage fails --- .../ui/player/controls/PlayerSceneMiningUi.kt | 2 + .../ui/player/scene/SceneCaptureRequest.kt | 19 ++- .../player/scene/SceneSentenceAudioService.kt | 17 +-- .../player/scene/SceneCaptureRequestTest.kt | 56 +++++++++ .../scene/SceneSentenceAudioServiceTest.kt | 119 ++++++++++++++++++ .../ui/reader/setting/ReaderOcrSourceTest.kt | 4 +- .../java/chimahon/anki/AnkiCardCreator.kt | 38 ++++-- .../src/main/java/chimahon/anki/AnkiMedia.kt | 2 + .../moko-resources/base/strings.xml | 2 + 9 files changed, 228 insertions(+), 31 deletions(-) create mode 100644 app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index 6ce11160af..759e66db5a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -47,6 +47,8 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List KMR.strings.anki_scene_fallback_generation AnkiMediaWarning.AnimatedStorageFailed -> KMR.strings.anki_scene_fallback_storage AnkiMediaWarning.StillStorageFailed -> KMR.strings.anki_scene_still_storage_failed + AnkiMediaWarning.SentenceAudioGenerationFailed -> KMR.strings.anki_sentence_audio_generation_failed + AnkiMediaWarning.SentenceAudioStorageFailed -> KMR.strings.anki_sentence_audio_storage_failed }, ) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt index 9e322e58ea..a139a0ec7b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt @@ -143,6 +143,7 @@ internal data class SceneMpvSnapshot( val selectedAudioId: Int?, val selectedExternalAudioValue: String?, val selectedAudioIsExternal: Boolean, + val audioTrackCount: Int, val seekable: Boolean?, val selectedAudioFfmpegIndex: Int? = null, ) @@ -182,6 +183,7 @@ internal class SceneMpvSnapshotReader( selectedAudioId = selectedAudio.id, selectedExternalAudioValue = selectedAudio.externalValue, selectedAudioIsExternal = selectedAudio.isExternal, + audioTrackCount = selectedAudio.trackCount, seekable = properties.boolean("seekable"), selectedAudioFfmpegIndex = selectedAudio.ffmpegIndex, ) @@ -203,15 +205,18 @@ internal class SceneMpvSnapshotReader( } private fun selectedAudio(): SelectedAudioSnapshot { + val trackCount = properties.int("track-list/count") + ?: return SelectedAudioSnapshot() + val audioTrackCount = (0 until trackCount).count { index -> + properties.string("track-list/$index/type") == "audio" + } val selectedAudioId = properties.string("aid")?.toIntOrNull() ?: properties.int("aid") - ?: return SelectedAudioSnapshot() - val trackCount = properties.int("track-list/count") - ?: return SelectedAudioSnapshot(id = selectedAudioId) + ?: return SelectedAudioSnapshot(trackCount = audioTrackCount) val index = (0 until trackCount).firstOrNull { index -> properties.string("track-list/$index/type") == "audio" && properties.int("track-list/$index/id") == selectedAudioId - } ?: return SelectedAudioSnapshot(id = selectedAudioId) + } ?: return SelectedAudioSnapshot(id = selectedAudioId, trackCount = audioTrackCount) val externalValue = properties.string("track-list/$index/external-filename") ?.takeIf(String::isNotBlank) return SelectedAudioSnapshot( @@ -219,6 +224,7 @@ internal class SceneMpvSnapshotReader( externalValue = externalValue, isExternal = properties.boolean("track-list/$index/external") == true || externalValue != null, + trackCount = audioTrackCount, ffmpegIndex = properties.int("track-list/$index/ff-index") ?.takeIf { it >= 0 }, ) @@ -228,6 +234,7 @@ internal class SceneMpvSnapshotReader( val id: Int? = null, val externalValue: String? = null, val isExternal: Boolean = false, + val trackCount: Int = 0, val ffmpegIndex: Int? = null, ) @@ -308,7 +315,8 @@ internal class SceneCaptureRequestFactory( SceneVideoInputResolver.resolve(beforeVideo.sentenceAudio) } beforeMpv.selectedAudioId != null && - beforeMpv.selectedAudioFfmpegIndex == null -> null + beforeMpv.selectedAudioFfmpegIndex == null && + beforeMpv.audioTrackCount != 1 -> null else -> videoInput } val request = SceneCaptureRequest( @@ -337,6 +345,7 @@ internal class SceneCaptureRequestFactory( before.selectedAudioId == after.selectedAudioId && before.selectedExternalAudioValue == after.selectedExternalAudioValue && before.selectedAudioIsExternal == after.selectedAudioIsExternal && + before.audioTrackCount == after.audioTrackCount && before.seekable == after.seekable && before.selectedAudioFfmpegIndex == after.selectedAudioFfmpegIndex } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 7a9a972ce9..69d0da7f6b 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -26,12 +26,11 @@ internal class FrozenSceneSentenceAudioService private constructor( ) override suspend fun prepare(request: SceneCaptureRequest): AnkiMediaSource? { - val videoInput = request.videoInput ?: return null val input = request.sentenceAudioInput ?: return null val range = request.resolvedTiming?.audioRange ?: return null return withTimeoutOrNull(AUDIO_TIMEOUT_MILLIS) { withContext(Dispatchers.IO) { - if (!isVideoSafe(videoInput) || !isAudioSafe(input)) { + if (!isAudioSafe(input)) { return@withContext null } val lease = inputAcquirer.acquire(input) ?: return@withContext null @@ -73,20 +72,6 @@ internal class FrozenSceneSentenceAudioService private constructor( } } - private suspend fun isVideoSafe(input: SceneVideoInputSpec): Boolean { - val lease = inputAcquirer.acquire(input) ?: return false - val cleanup = SceneNativeCleanup(lease::close) - return try { - val probe = commandExecutor.executeFfprobe( - SceneFfmpegArguments.videoProbe(input, lease.ffmpegValue, lease.tlsCaFile), - cleanup::nativeFinished, - ) - probe is SceneCommandResult.Success && SceneMediaProbe.inspect(probe.output) - } finally { - cleanup.release() - } - } - private suspend fun isAudioSafe(input: SceneVideoInputSpec): Boolean { val lease = inputAcquirer.acquire(input) ?: return false val cleanup = SceneNativeCleanup(lease::close) diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt index 459501d71a..0db5785b6c 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt @@ -99,6 +99,31 @@ class SceneCaptureRequestTest { request.close() } + @Test + fun `single selected audio without a frozen ffmpeg index uses the only audio stream`() = runTest { + val bitmap = mockBitmap() + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader( + SequencePropertyReader( + listOf( + snapshotWithSingleAudioWithoutFfmpegIndex(), + snapshotWithSingleAudioWithoutFfmpegIndex(), + ), + ), + ), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { inputSnapshot() }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + assertNotNull(request) + assertNotNull(request!!.sentenceAudioInput) + request.close() + } + private fun mockBitmap(): Bitmap { return mockk(relaxed = true).also { every { it.isRecycled } returns false @@ -150,6 +175,37 @@ class SceneCaptureRequestTest { ) } + private fun snapshotWithSingleAudioWithoutFfmpegIndex(): SnapshotValues { + return SnapshotValues( + doubles = mapOf( + "time-pos" to 5.0, + "duration" to 60.0, + "sub-start/full" to 4.0, + "sub-end/full" to 6.0, + "sub-speed" to 1.0, + "sub-delay" to 0.0, + ), + strings = mapOf( + "path" to "https://media.example/video.mp4", + "aid" to "2", + "track-list/0/type" to "video", + "track-list/1/type" to "audio", + ), + booleans = mapOf( + "track-list/0/selected" to true, + "track-list/0/external" to false, + "track-list/1/external" to false, + "seekable" to true, + ), + ints = mapOf( + "track-list/count" to 2, + "track-list/0/id" to 1, + "track-list/0/ff-index" to 0, + "track-list/1/id" to 2, + ), + ) + } + private data class SnapshotValues( val doubles: Map, val strings: Map, diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt new file mode 100644 index 0000000000..7f99d09382 --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -0,0 +1,119 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import android.graphics.Bitmap +import chimahon.anki.AnkiMediaSource +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +class SceneSentenceAudioServiceTest { + @TempDir + lateinit var tempDirectory: File + + @Test + fun `exports safe audio when the video stream is HDR`() = runBlocking { + val executor = RecordingExecutor() + val request = request() + + try { + val result = service(executor).prepare(request) + val audio = result as? AnkiMediaSource.Bytes + + assertNotNull(audio) + assertEquals("m4a", audio!!.extension) + assertEquals(1, executor.ffmpegCalls) + assertEquals(listOf("a:0"), executor.ffprobeSelectors) + } finally { + request.close() + } + } + + private fun service(executor: RecordingExecutor): FrozenSceneSentenceAudioService { + val constructor = FrozenSceneSentenceAudioService::class.java.getDeclaredConstructor( + File::class.java, + SceneInputAcquirer::class.java, + SceneCommandExecutor::class.java, + ).apply { + isAccessible = true + } + return constructor.newInstance( + tempDirectory, + SceneInputAcquirer { input -> + object : SceneInputLease { + override val ffmpegValue = input.value + override val tlsCaFile = "/files/cacert.pem" + + override fun close() = Unit + } + }, + executor, + ) + } + + private fun request(): SceneCaptureRequest { + val video = SceneVideoInputSpec( + value = "https://media.example/video.mkv", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = emptyList(), + ) + val audio = SceneVideoInputSpec( + value = "https://media.example/audio.m4a", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = emptyList(), + ) + val bitmap = mockk(relaxed = true) + every { bitmap.isRecycled } returns false + return SceneCaptureRequest( + videoInput = video, + sentenceAudioInput = audio, + resolvedTiming = SceneResolvedTiming( + animationRange = SceneTimeRange(1.25, 4.25), + audioRange = SceneTimeRange(1.25, 4.25), + ), + stillFallback = OwnedBitmap(bitmap), + ) + } + + private class RecordingExecutor : SceneCommandExecutor { + var ffmpegCalls = 0 + val ffprobeSelectors = mutableListOf() + + override suspend fun executeFfmpeg( + arguments: Array, + onNativeFinished: () -> Unit, + ): SceneCommandResult { + return try { + ffmpegCalls++ + val output = File(arguments.last()) + output.writeBytes(byteArrayOf(1, 2, 3)) + SceneCommandResult.Success() + } finally { + onNativeFinished() + } + } + + override suspend fun executeFfprobe( + arguments: Array, + onNativeFinished: () -> Unit, + ): SceneCommandResult { + return try { + val selector = arguments[arguments.indexOf("-select_streams") + 1] + ffprobeSelectors += selector + when (selector) { + "v:0" -> SceneCommandResult.Success( + "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\nbits_per_raw_sample=10", + ) + "a:0" -> SceneCommandResult.Success("codec_type=audio\ncodec_name=aac") + else -> error("Unexpected stream selector: $selector") + } + } finally { + onNativeFinished() + } + } + } +} diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt index 6aa3801feb..7ee1a7b3e9 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt @@ -37,8 +37,8 @@ class ReaderOcrSourceTest { @Test fun `local source is only offered when available`() { - assertEquals(ReaderOcrSource.entries, ReaderOcrSource.availableSources(localOcrAvailable = true)) - assertFalse(ReaderOcrSource.LOCAL in ReaderOcrSource.availableSources(localOcrAvailable = false)) + assertEquals(ReaderOcrSource.entries, ReaderOcrSource.availableSources(localOcrAvailable = true, mokuroAvailable = true)) + assertFalse(ReaderOcrSource.LOCAL in ReaderOcrSource.availableSources(localOcrAvailable = false, mokuroAvailable = true)) } @Test diff --git a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt index d9b64fd478..79082ea998 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt @@ -384,16 +384,23 @@ object AnkiCardCreator { ownedPreparedFiles += screenshotPreparation.animation } + var sentenceAudioGenerationFailed = false + var sentenceAudioStorageFailed = false val sentenceAudioSource = if (hasSentenceAudioMarker) { - val lazySource = try { - mediaRequest?.sentenceAudioProvider?.prepare() - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - android.util.Log.w(TAG, "addToAnki: failed to prepare sentence audio", e) + val provider = mediaRequest?.sentenceAudioProvider + val lazySource = if (provider != null) { + try { + provider.prepare() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + android.util.Log.w(TAG, "addToAnki: failed to prepare sentence audio", e) + null + } + } else { null } - lazySource ?: sentenceAudioBytes?.let { bytes -> + val eagerSource = sentenceAudioBytes?.let { bytes -> val filename = generateSentenceAudioFilename(bytes, sentenceAudioExtension) AnkiMediaSource.Bytes( data = bytes, @@ -401,6 +408,10 @@ object AnkiCardCreator { extension = AnkiMediaNaming.safeExtension(filename, "m4a"), ) } + if (provider != null && lazySource == null && eagerSource == null) { + sentenceAudioGenerationFailed = true + } + lazySource ?: eagerSource } else { null } @@ -472,6 +483,9 @@ object AnkiCardCreator { sentenceAudioSource, "sentence audio", ) + if (sentenceAudioSource != null && sentenceAudioFilename == null) { + sentenceAudioStorageFailed = true + } val fieldsWithDictionaryMedia = resolveDictionaryMediaPlaceholders( fieldsWithPlaceholders, exportMedia, @@ -515,7 +529,15 @@ object AnkiCardCreator { titleId = titleId, ) if (syncOnCreate) bridge.triggerSync() - AnkiResult.Success(noteId, screenshotResult.warnings) + AnkiResult.Success( + noteId, + screenshotResult.warnings + listOfNotNull( + AnkiMediaWarning.SentenceAudioGenerationFailed + .takeIf { sentenceAudioGenerationFailed }, + AnkiMediaWarning.SentenceAudioStorageFailed + .takeIf { sentenceAudioStorageFailed }, + ), + ) } } } catch (e: CancellationException) { diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index 8f97351d72..bbaf2aafd4 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -48,6 +48,8 @@ sealed interface AnkiMediaWarning { data object SceneGenerationFailed : AnkiMediaWarning data object AnimatedStorageFailed : AnkiMediaWarning data object StillStorageFailed : AnkiMediaWarning + data object SentenceAudioGenerationFailed : AnkiMediaWarning + data object SentenceAudioStorageFailed : AnkiMediaWarning } sealed interface AnkiScreenshotPreparation { diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index eff6a5eda9..453e862c09 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -259,6 +259,8 @@ The animated scene could not be created. Chimahon used the captured still screenshot when available. AnkiDroid could not store the animated scene. Chimahon used the captured still screenshot when available. The screenshot could not be stored. The card was created without it. + Sentence audio could not be exported from this video source. The card was created without it. + AnkiDroid could not store the sentence audio. The card was created without it. Media Learning Sync From 01eb2be454f745ef64453f675891e1de84128294 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Fri, 31 Jul 2026 01:11:34 +0700 Subject: [PATCH 2/9] fix(player): report sentence audio export failures --- .../ui/player/controls/PlayerSceneMiningUi.kt | 26 ++- .../ui/player/scene/SceneCaptureRequest.kt | 30 +++- .../player/scene/SceneSentenceAudioService.kt | 62 +++++--- .../player/scene/SceneCaptureRequestTest.kt | 6 + .../scene/SceneSentenceAudioServiceTest.kt | 149 +++++++++++++++--- .../java/chimahon/anki/AnkiCardCreator.kt | 14 +- .../src/main/java/chimahon/anki/AnkiMedia.kt | 27 +++- .../moko-resources/base/strings.xml | 7 + 8 files changed, 263 insertions(+), 58 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index 759e66db5a..e75c5254f1 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -7,6 +7,7 @@ import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.ui.window.DialogProperties import chimahon.anki.AnkiMediaWarning +import chimahon.anki.AnkiSentenceAudioFailure import eu.kanade.tachiyomi.ui.player.scene.PlayerSceneMiningProgress import eu.kanade.tachiyomi.util.system.toast import tachiyomi.i18n.kmk.KMR @@ -47,7 +48,30 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List KMR.strings.anki_scene_fallback_generation AnkiMediaWarning.AnimatedStorageFailed -> KMR.strings.anki_scene_fallback_storage AnkiMediaWarning.StillStorageFailed -> KMR.strings.anki_scene_still_storage_failed - AnkiMediaWarning.SentenceAudioGenerationFailed -> KMR.strings.anki_sentence_audio_generation_failed + is AnkiMediaWarning.SentenceAudioGenerationFailed -> when (warning.failure) { + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_track_mapping_unavailable + } + AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_source_unavailable + } + AnkiSentenceAudioFailure.TIMING_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_timing_unavailable + } + AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED -> { + KMR.strings.anki_sentence_audio_probe_failed + } + AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE -> { + KMR.strings.anki_sentence_audio_stream_unreadable + } + AnkiSentenceAudioFailure.EXTRACTION_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_failed + } + AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT -> { + KMR.strings.anki_sentence_audio_extraction_timed_out + } + AnkiSentenceAudioFailure.UNKNOWN -> KMR.strings.anki_sentence_audio_generation_failed + } AnkiMediaWarning.SentenceAudioStorageFailed -> KMR.strings.anki_sentence_audio_storage_failed }, ) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt index a139a0ec7b..52e7e21990 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt @@ -1,6 +1,7 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap +import chimahon.anki.AnkiSentenceAudioFailure import `is`.xyz.mpv.MPVLib import java.io.Closeable import kotlin.math.abs @@ -91,6 +92,7 @@ internal class OwnedBitmap private constructor( internal class SceneCaptureRequest( val videoInput: SceneVideoInputSpec?, val sentenceAudioInput: SceneVideoInputSpec?, + val sentenceAudioFailure: AnkiSentenceAudioFailure? = null, val resolvedTiming: SceneResolvedTiming?, private val stillFallback: OwnedBitmap, ) : Closeable { @@ -310,18 +312,31 @@ internal class SceneCaptureRequestFactory( if (!sameCaptureState(beforeMpv, afterMpv) || beforeVideo != afterVideo) return null val videoInput = SceneVideoInputResolver.resolve(beforeVideo.video) - val sentenceAudioInput = when { + val sentenceAudio = when { beforeVideo.sentenceAudio != null -> { - SceneVideoInputResolver.resolve(beforeVideo.sentenceAudio) + val input = SceneVideoInputResolver.resolve(beforeVideo.sentenceAudio) + SentenceAudioInputResolution( + input = input, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { input == null }, + ) } beforeMpv.selectedAudioId != null && beforeMpv.selectedAudioFfmpegIndex == null && - beforeMpv.audioTrackCount != 1 -> null - else -> videoInput + beforeMpv.audioTrackCount != 1 -> { + SentenceAudioInputResolution( + input = null, + failure = AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ) + } + else -> SentenceAudioInputResolution( + input = videoInput, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { videoInput == null }, + ) } val request = SceneCaptureRequest( videoInput = videoInput, - sentenceAudioInput = sentenceAudioInput, + sentenceAudioInput = sentenceAudio.input, + sentenceAudioFailure = sentenceAudio.failure, resolvedTiming = resolveTiming(beforeMpv), stillFallback = OwnedBitmap(fallback), ) @@ -350,6 +365,11 @@ internal class SceneCaptureRequestFactory( before.selectedAudioFfmpegIndex == after.selectedAudioFfmpegIndex } + private data class SentenceAudioInputResolution( + val input: SceneVideoInputSpec?, + val failure: AnkiSentenceAudioFailure?, + ) + private fun nullableDoubleEquals(first: Double?, second: Double?): Boolean { if (first == null || second == null) return first == second return closeEnough(first, second, DOUBLE_SNAPSHOT_TOLERANCE_SECONDS) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 69d0da7f6b..1b82951f9a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -2,6 +2,8 @@ package eu.kanade.tachiyomi.ui.player.scene import android.content.Context import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPreparation import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -11,13 +13,14 @@ import java.security.MessageDigest import java.util.UUID internal fun interface SceneSentenceAudioService { - suspend fun prepare(request: SceneCaptureRequest): AnkiMediaSource? + suspend fun prepare(request: SceneCaptureRequest): AnkiSentenceAudioPreparation } -internal class FrozenSceneSentenceAudioService private constructor( +internal class FrozenSceneSentenceAudioService internal constructor( private val cacheDirectory: File, private val inputAcquirer: SceneInputAcquirer, private val commandExecutor: SceneCommandExecutor, + private val timeoutMillis: Long = AUDIO_TIMEOUT_MILLIS, ) : SceneSentenceAudioService { constructor(context: Context) : this( cacheDirectory = context.cacheDir, @@ -25,15 +28,16 @@ internal class FrozenSceneSentenceAudioService private constructor( commandExecutor = FfmpegKitSceneCommandExecutor(), ) - override suspend fun prepare(request: SceneCaptureRequest): AnkiMediaSource? { - val input = request.sentenceAudioInput ?: return null - val range = request.resolvedTiming?.audioRange ?: return null - return withTimeoutOrNull(AUDIO_TIMEOUT_MILLIS) { + override suspend fun prepare(request: SceneCaptureRequest): AnkiSentenceAudioPreparation { + val input = request.sentenceAudioInput + ?: return unavailable(request.sentenceAudioFailure ?: AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + val range = request.resolvedTiming?.audioRange + ?: return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) + return withTimeoutOrNull(timeoutMillis) { withContext(Dispatchers.IO) { - if (!isAudioSafe(input)) { - return@withContext null - } - val lease = inputAcquirer.acquire(input) ?: return@withContext null + inspectAudio(input)?.let { return@withContext unavailable(it) } + val lease = inputAcquirer.acquire(input) + ?: return@withContext unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) val output = File(cacheDirectory, "chimahon_sentence_audio_${UUID.randomUUID()}.m4a") val inputCleanup = SceneNativeCleanup(lease::close) val outputCleanup = SceneNativeCleanup(output::delete) @@ -52,40 +56,58 @@ internal class FrozenSceneSentenceAudioService private constructor( outputCleanup.nativeFinished() } if (result !is SceneCommandResult.Success || !output.isFile || output.length() == 0L) { - return@withContext null + return@withContext unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED) } val bytes = output.readBytes() - AnkiMediaSource.Bytes( - data = bytes, - preferredBaseName = "chimahon_sentence_${bytes.sha256()}", - extension = "m4a", + AnkiSentenceAudioPreparation.Ready( + AnkiMediaSource.Bytes( + data = bytes, + preferredBaseName = "chimahon_sentence_${bytes.sha256()}", + extension = "m4a", + ), ) } catch (e: CancellationException) { throw e } catch (_: Exception) { - return@withContext null + unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED) } finally { inputCleanup.release() outputCleanup.release() } } - } + } ?: unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) } - private suspend fun isAudioSafe(input: SceneVideoInputSpec): Boolean { - val lease = inputAcquirer.acquire(input) ?: return false + private suspend fun inspectAudio(input: SceneVideoInputSpec): AnkiSentenceAudioFailure? { + val lease = inputAcquirer.acquire(input) ?: return AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE val cleanup = SceneNativeCleanup(lease::close) return try { val probe = commandExecutor.executeFfprobe( SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile), cleanup::nativeFinished, ) - probe is SceneCommandResult.Success && SceneMediaProbe.inspectAudio(probe.output) + when (probe) { + SceneCommandResult.Failed -> AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED + is SceneCommandResult.Success -> { + if (SceneMediaProbe.inspectAudio(probe.output)) { + null + } else { + AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE + } + } + } + } catch (e: CancellationException) { + throw e + } catch (_: Exception) { + AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED } finally { cleanup.release() } } + private fun unavailable(failure: AnkiSentenceAudioFailure) = + AnkiSentenceAudioPreparation.Unavailable(failure) + private fun ByteArray.sha256(): String { return MessageDigest.getInstance("SHA-256") .digest(this) diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt index 0db5785b6c..913621f10b 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt @@ -1,10 +1,12 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap +import chimahon.anki.AnkiSentenceAudioFailure import io.mockk.every import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test @@ -96,6 +98,10 @@ class SceneCaptureRequestTest { assertNotNull(request) assertNull(request!!.sentenceAudioInput) + assertEquals( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + request.sentenceAudioFailure, + ) request.close() } diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt index 7f99d09382..489ac6dbc4 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -2,8 +2,11 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPreparation import io.mockk.every import io.mockk.mockk +import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull @@ -22,7 +25,7 @@ class SceneSentenceAudioServiceTest { try { val result = service(executor).prepare(request) - val audio = result as? AnkiMediaSource.Bytes + val audio = (result as? AnkiSentenceAudioPreparation.Ready)?.source as? AnkiMediaSource.Bytes assertNotNull(audio) assertEquals("m4a", audio!!.extension) @@ -33,17 +36,102 @@ class SceneSentenceAudioServiceTest { } } - private fun service(executor: RecordingExecutor): FrozenSceneSentenceAudioService { - val constructor = FrozenSceneSentenceAudioService::class.java.getDeclaredConstructor( - File::class.java, - SceneInputAcquirer::class.java, - SceneCommandExecutor::class.java, - ).apply { - isAccessible = true + @Test + fun `reports unavailable source when no sentence audio input was captured`() = runBlocking { + val request = request(sentenceAudioInput = null) + + try { + val result = service(RecordingExecutor()).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports unreadable stream when FFprobe finds no audio`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(audioProbeResult = SceneCommandResult.Success("codec_type=video")), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE), + result, + ) + } finally { + request.close() } - return constructor.newInstance( - tempDirectory, - SceneInputAcquirer { input -> + } + + @Test + fun `reports probe failure when FFprobe fails`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(audioProbeResult = SceneCommandResult.Failed), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports extraction failure when FFmpeg fails`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(ffmpegResult = SceneCommandResult.Failed), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports extraction timeout when FFmpeg does not finish in time`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(ffmpegDelayMillis = 50), + timeoutMillis = 1, + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT), + result, + ) + } finally { + request.close() + } + } + + private fun service( + executor: RecordingExecutor, + timeoutMillis: Long = 60_000L, + ): FrozenSceneSentenceAudioService { + return FrozenSceneSentenceAudioService( + cacheDirectory = tempDirectory, + inputAcquirer = SceneInputAcquirer { input -> object : SceneInputLease { override val ffmpegValue = input.value override val tlsCaFile = "/files/cacert.pem" @@ -51,26 +139,22 @@ class SceneSentenceAudioServiceTest { override fun close() = Unit } }, - executor, + commandExecutor = executor, + timeoutMillis = timeoutMillis, ) } - private fun request(): SceneCaptureRequest { + private fun request(sentenceAudioInput: SceneVideoInputSpec? = audioInput()): SceneCaptureRequest { val video = SceneVideoInputSpec( value = "https://media.example/video.mkv", kind = SceneVideoInputKind.REMOTE_HTTP, headers = emptyList(), ) - val audio = SceneVideoInputSpec( - value = "https://media.example/audio.m4a", - kind = SceneVideoInputKind.REMOTE_HTTP, - headers = emptyList(), - ) val bitmap = mockk(relaxed = true) every { bitmap.isRecycled } returns false return SceneCaptureRequest( videoInput = video, - sentenceAudioInput = audio, + sentenceAudioInput = sentenceAudioInput, resolvedTiming = SceneResolvedTiming( animationRange = SceneTimeRange(1.25, 4.25), audioRange = SceneTimeRange(1.25, 4.25), @@ -79,7 +163,21 @@ class SceneSentenceAudioServiceTest { ) } - private class RecordingExecutor : SceneCommandExecutor { + private fun audioInput(): SceneVideoInputSpec { + return SceneVideoInputSpec( + value = "https://media.example/audio.m4a", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = emptyList(), + ) + } + + private class RecordingExecutor( + private val audioProbeResult: SceneCommandResult = SceneCommandResult.Success( + "codec_type=audio\ncodec_name=aac", + ), + private val ffmpegResult: SceneCommandResult = SceneCommandResult.Success(), + private val ffmpegDelayMillis: Long = 0, + ) : SceneCommandExecutor { var ffmpegCalls = 0 val ffprobeSelectors = mutableListOf() @@ -89,9 +187,12 @@ class SceneSentenceAudioServiceTest { ): SceneCommandResult { return try { ffmpegCalls++ - val output = File(arguments.last()) - output.writeBytes(byteArrayOf(1, 2, 3)) - SceneCommandResult.Success() + if (ffmpegDelayMillis > 0) delay(ffmpegDelayMillis) + if (ffmpegResult is SceneCommandResult.Success) { + val output = File(arguments.last()) + output.writeBytes(byteArrayOf(1, 2, 3)) + } + ffmpegResult } finally { onNativeFinished() } @@ -108,7 +209,7 @@ class SceneSentenceAudioServiceTest { "v:0" -> SceneCommandResult.Success( "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\nbits_per_raw_sample=10", ) - "a:0" -> SceneCommandResult.Success("codec_type=audio\ncodec_name=aac") + "a:0" -> audioProbeResult else -> error("Unexpected stream selector: $selector") } } finally { diff --git a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt index 79082ea998..96a98ec486 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt @@ -384,22 +384,24 @@ object AnkiCardCreator { ownedPreparedFiles += screenshotPreparation.animation } - var sentenceAudioGenerationFailed = false + var sentenceAudioGenerationFailure: AnkiSentenceAudioFailure? = null var sentenceAudioStorageFailed = false val sentenceAudioSource = if (hasSentenceAudioMarker) { val provider = mediaRequest?.sentenceAudioProvider - val lazySource = if (provider != null) { + val lazyPreparation = if (provider != null) { try { provider.prepare() } catch (e: CancellationException) { throw e } catch (e: Exception) { android.util.Log.w(TAG, "addToAnki: failed to prepare sentence audio", e) - null + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.UNKNOWN) } } else { null } + val lazySource = (lazyPreparation as? AnkiSentenceAudioPreparation.Ready)?.source + val lazyFailure = (lazyPreparation as? AnkiSentenceAudioPreparation.Unavailable)?.failure val eagerSource = sentenceAudioBytes?.let { bytes -> val filename = generateSentenceAudioFilename(bytes, sentenceAudioExtension) AnkiMediaSource.Bytes( @@ -409,7 +411,7 @@ object AnkiCardCreator { ) } if (provider != null && lazySource == null && eagerSource == null) { - sentenceAudioGenerationFailed = true + sentenceAudioGenerationFailure = lazyFailure ?: AnkiSentenceAudioFailure.UNKNOWN } lazySource ?: eagerSource } else { @@ -532,8 +534,8 @@ object AnkiCardCreator { AnkiResult.Success( noteId, screenshotResult.warnings + listOfNotNull( - AnkiMediaWarning.SentenceAudioGenerationFailed - .takeIf { sentenceAudioGenerationFailed }, + sentenceAudioGenerationFailure + ?.let(AnkiMediaWarning::SentenceAudioGenerationFailed), AnkiMediaWarning.SentenceAudioStorageFailed .takeIf { sentenceAudioStorageFailed }, ), diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index bbaf2aafd4..455a878219 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -44,11 +44,34 @@ sealed interface AnkiMediaSource { ) : AnkiMediaSource } +enum class AnkiSentenceAudioFailure { + TRACK_MAPPING_UNAVAILABLE, + SOURCE_UNAVAILABLE, + TIMING_UNAVAILABLE, + AUDIO_PROBE_FAILED, + AUDIO_STREAM_UNREADABLE, + EXTRACTION_FAILED, + EXTRACTION_TIMED_OUT, + UNKNOWN, +} + +sealed interface AnkiSentenceAudioPreparation { + data class Ready( + val source: AnkiMediaSource, + ) : AnkiSentenceAudioPreparation + + data class Unavailable( + val failure: AnkiSentenceAudioFailure, + ) : AnkiSentenceAudioPreparation +} + sealed interface AnkiMediaWarning { data object SceneGenerationFailed : AnkiMediaWarning data object AnimatedStorageFailed : AnkiMediaWarning data object StillStorageFailed : AnkiMediaWarning - data object SentenceAudioGenerationFailed : AnkiMediaWarning + data class SentenceAudioGenerationFailed( + val failure: AnkiSentenceAudioFailure, + ) : AnkiMediaWarning data object SentenceAudioStorageFailed : AnkiMediaWarning } @@ -72,7 +95,7 @@ fun interface LazyAnkiScreenshotProvider { } fun interface LazyAnkiMediaProvider { - suspend fun prepare(): AnkiMediaSource? + suspend fun prepare(): AnkiSentenceAudioPreparation } data class AnkiMediaRequest( diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index 453e862c09..8bce703c8f 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -260,6 +260,13 @@ AnkiDroid could not store the animated scene. Chimahon used the captured still screenshot when available. The screenshot could not be stored. The card was created without it. Sentence audio could not be exported from this video source. The card was created without it. + The selected audio track could not be identified safely. Sentence audio was not added to avoid choosing the wrong track. + The selected video source could not be read for sentence audio. The card was created without it. + The subtitle timing was unavailable. Sentence audio was not added. + The selected audio stream could not be inspected. The card was created without sentence audio. + No readable audio stream was found in this video. The card was created without sentence audio. + Sentence audio extraction failed for this video. The card was created without it. + Sentence audio extraction timed out. The card was created without it. AnkiDroid could not store the sentence audio. The card was created without it. Media Learning From 443f37882705c52a797e188c72905364314d6f84 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Fri, 31 Jul 2026 01:47:36 +0700 Subject: [PATCH 3/9] fix(player): recover sentence audio from invalid MPV stream indexes - validate the selected stream before exporting sentence audio - fall back only when FFprobe finds exactly one readable audio stream - distinguish missing, non-audio, and protected stream failures --- .../ui/player/controls/PlayerSceneMiningUi.kt | 9 + .../ui/player/scene/SceneMediaProbe.kt | 78 +++++++- .../player/scene/SceneSentenceAudioService.kt | 121 +++++++++++-- .../ui/player/scene/SceneVideoInput.kt | 19 ++ .../scene/SceneSentenceAudioServiceTest.kt | 168 +++++++++++++++++- .../src/main/java/chimahon/anki/AnkiMedia.kt | 3 + .../moko-resources/base/strings.xml | 3 + 7 files changed, 372 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index e75c5254f1..eec062b5fb 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -61,6 +61,15 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List { KMR.strings.anki_sentence_audio_probe_failed } + AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_stream_index_unavailable + } + AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO -> { + KMR.strings.anki_sentence_audio_stream_not_audio + } + AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED -> { + KMR.strings.anki_sentence_audio_stream_protected + } AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE -> { KMR.strings.anki_sentence_audio_stream_unreadable } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt index 1466b296f1..9a60fcad7e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt @@ -4,8 +4,7 @@ import java.util.Locale internal object SceneMediaProbe { fun inspect(output: String): Boolean { - val normalized = output.lowercase(Locale.ROOT) - if (PROTECTION_MARKERS.any(normalized::contains)) { + if (output.hasProtectionMarker()) { return false } val values = output.lineSequence() @@ -42,9 +41,78 @@ internal object SceneMediaProbe { return true } - fun inspectAudio(output: String): Boolean { - val normalized = output.lowercase(Locale.ROOT) - return PROTECTION_MARKERS.none(normalized::contains) && "codec_type=audio" in normalized + fun inspectAudio(output: String): Boolean = inspectSelectedAudio(output) is AudioInspection.Readable + + fun inspectSelectedAudio(output: String): AudioInspection { + val values = output.keyValues() + if (values.isEmpty()) return AudioInspection.StreamMissing + if (output.hasProtectionMarker()) return AudioInspection.Protected + return if (values["codec_type"] == "audio") { + AudioInspection.Readable + } else { + AudioInspection.NotAudio + } + } + + fun audioStreams(output: String): List { + return output.streamBlocks().mapNotNull { streamOutput -> + val values = streamOutput.keyValues() + if (values["codec_type"] != "audio") return@mapNotNull null + AudioStream( + index = values["index"]?.toIntOrNull(), + protected = streamOutput.hasProtectionMarker(), + ) + } + } + + sealed interface AudioInspection { + data object Readable : AudioInspection + + data object StreamMissing : AudioInspection + + data object NotAudio : AudioInspection + + data object Protected : AudioInspection + } + + data class AudioStream( + val index: Int?, + val protected: Boolean, + ) + + private fun String.keyValues(): Map { + return lineSequence() + .mapNotNull { line -> + val separator = line.indexOf('=') + if (separator <= 0) { + null + } else { + line.substring(0, separator).trim().lowercase(Locale.ROOT) to + line.substring(separator + 1).trim().lowercase(Locale.ROOT) + } + } + .toMap() + } + + private fun String.streamBlocks(): List { + val blocks = mutableListOf() + var current: StringBuilder? = null + lineSequence().forEach { line -> + when (line.trim()) { + "[STREAM]" -> current = StringBuilder() + "[/STREAM]" -> current?.let { stream -> + blocks += stream.toString() + current = null + } + else -> current?.append(line)?.append('\n') + } + } + return blocks.ifEmpty { listOf(this) } + } + + private fun String.hasProtectionMarker(): Boolean { + val normalized = lowercase(Locale.ROOT) + return PROTECTION_MARKERS.any(normalized::contains) } private val HDR_TRANSFERS = setOf("smpte2084", "arib-std-b67") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 1b82951f9a..16cf307114 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -35,8 +35,11 @@ internal class FrozenSceneSentenceAudioService internal constructor( ?: return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) return withTimeoutOrNull(timeoutMillis) { withContext(Dispatchers.IO) { - inspectAudio(input)?.let { return@withContext unavailable(it) } - val lease = inputAcquirer.acquire(input) + val resolvedInput = when (val resolution = resolveAudioInput(input)) { + is AudioInputResolution.Ready -> resolution.input + is AudioInputResolution.Unavailable -> return@withContext unavailable(resolution.failure) + } + val lease = inputAcquirer.acquire(resolvedInput) ?: return@withContext unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) val output = File(cacheDirectory, "chimahon_sentence_audio_${UUID.randomUUID()}.m4a") val inputCleanup = SceneNativeCleanup(lease::close) @@ -45,7 +48,7 @@ internal class FrozenSceneSentenceAudioService internal constructor( output.delete() val result = commandExecutor.executeFfmpeg( SceneFfmpegArguments.sentenceAudio( - input = input, + input = resolvedInput, acquiredInputValue = lease.ffmpegValue, range = range, outputFile = output.absolutePath, @@ -78,33 +81,121 @@ internal class FrozenSceneSentenceAudioService internal constructor( } ?: unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) } - private suspend fun inspectAudio(input: SceneVideoInputSpec): AnkiSentenceAudioFailure? { - val lease = inputAcquirer.acquire(input) ?: return AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE + private suspend fun resolveAudioInput(input: SceneVideoInputSpec): AudioInputResolution { + return when (val probe = executeAudioProbe(input, allAudioStreams = false)) { + AudioProbeResult.SourceUnavailable -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + when (val inspection = SceneMediaProbe.inspectSelectedAudio(probe.output)) { + SceneMediaProbe.AudioInspection.Readable -> AudioInputResolution.Ready(input) + SceneMediaProbe.AudioInspection.Protected -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED) + } + SceneMediaProbe.AudioInspection.StreamMissing, + SceneMediaProbe.AudioInspection.NotAudio -> { + resolveFallbackAudioInput(input, inspection) + } + } + } + } + } + + private suspend fun resolveFallbackAudioInput( + input: SceneVideoInputSpec, + selectedInspection: SceneMediaProbe.AudioInspection, + ): AudioInputResolution { + return when (val probe = executeAudioProbe(input, allAudioStreams = true)) { + AudioProbeResult.SourceUnavailable -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + val streams = SceneMediaProbe.audioStreams(probe.output) + val onlyStream = streams.singleOrNull() + when { + onlyStream?.index != null && !onlyStream.protected -> { + AudioInputResolution.Ready(input.copy(audioStreamIndex = onlyStream.index)) + } + streams.size > 1 -> { + AudioInputResolution.Unavailable( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ) + } + onlyStream?.protected == true -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED) + } + onlyStream != null -> { + AudioInputResolution.Unavailable( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ) + } + selectedInspection is SceneMediaProbe.AudioInspection.StreamMissing -> { + AudioInputResolution.Unavailable( + AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE, + ) + } + else -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO) + } + } + } + } + } + + private suspend fun executeAudioProbe( + input: SceneVideoInputSpec, + allAudioStreams: Boolean, + ): AudioProbeResult { + val lease = inputAcquirer.acquire(input) ?: return AudioProbeResult.SourceUnavailable val cleanup = SceneNativeCleanup(lease::close) return try { val probe = commandExecutor.executeFfprobe( - SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile), + if (allAudioStreams) { + SceneFfmpegArguments.allAudioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } else { + SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + }, cleanup::nativeFinished, ) when (probe) { - SceneCommandResult.Failed -> AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED - is SceneCommandResult.Success -> { - if (SceneMediaProbe.inspectAudio(probe.output)) { - null - } else { - AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE - } - } + SceneCommandResult.Failed -> AudioProbeResult.ExecutionFailed + is SceneCommandResult.Success -> AudioProbeResult.Success(probe.output) } } catch (e: CancellationException) { throw e } catch (_: Exception) { - AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED + AudioProbeResult.ExecutionFailed } finally { cleanup.release() } } + private sealed interface AudioInputResolution { + data class Ready( + val input: SceneVideoInputSpec, + ) : AudioInputResolution + + data class Unavailable( + val failure: AnkiSentenceAudioFailure, + ) : AudioInputResolution + } + + private sealed interface AudioProbeResult { + data object SourceUnavailable : AudioProbeResult + + data object ExecutionFailed : AudioProbeResult + + data class Success( + val output: String, + ) : AudioProbeResult + } + private fun unavailable(failure: AnkiSentenceAudioFailure) = AnkiSentenceAudioPreparation.Unavailable(failure) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt index b6f19d112a..e17d12ec83 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt @@ -248,6 +248,25 @@ internal object SceneFfmpegArguments { }.toTypedArray() } + fun allAudioProbe( + input: SceneVideoInputSpec, + acquiredInputValue: String, + tlsCaFile: String? = null, + ): Array { + return buildList { + addInputOptions(input, tlsCaFile) + add("-v") + add("error") + add("-select_streams") + add("a") + add("-show_entries") + add("stream=index,codec_type,codec_name:stream_side_data") + add("-of") + add("default=noprint_wrappers=0") + add(acquiredInputValue) + }.toTypedArray() + } + fun sentenceAudio( input: SceneVideoInputSpec, acquiredInputValue: String, diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt index 489ac6dbc4..a0db022bab 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -53,16 +53,163 @@ class SceneSentenceAudioServiceTest { } @Test - fun `reports unreadable stream when FFprobe finds no audio`() = runBlocking { + fun `reports selected non-audio stream when FFprobe finds no audio`() = runBlocking { val request = request() try { val result = service( - RecordingExecutor(audioProbeResult = SceneCommandResult.Success("codec_type=video")), + RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + ), + ), ).prepare(request) assertEquals( - AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE), + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `normalizes a wrong MPV stream index when exactly one audio stream is readable`() = runBlocking { + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + codec_name=aac + [/STREAM] + """.trimIndent(), + ), + ), + ) + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("0", "a"), executor.ffprobeSelectors) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `reports missing selected stream when no audio stream can be resolved`() = runBlocking { + val request = request(audioInput(audioStreamIndex = 4)) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "4" to SceneCommandResult.Success(), + "a" to SceneCommandResult.Success(), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE, + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports selected non-audio stream when no audio stream can be resolved`() = runBlocking { + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success(), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO, + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports protected selected audio stream without remapping it`() = runBlocking { + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success( + "index=0\ncodec_type=audio\nside_data_type=Encryption info", + ), + ), + ) + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service(executor).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED, + ), + result, + ) + assertEquals(listOf("0"), executor.ffprobeSelectors) + assertEquals(0, executor.ffmpegCalls) + } finally { + request.close() + } + } + + @Test + fun `refuses to replace a wrong MPV stream index when multiple audio streams exist`() = runBlocking { + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + [/STREAM] + [STREAM] + index=2 + codec_type=audio + [/STREAM] + """.trimIndent(), + ), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ), result, ) } finally { @@ -76,7 +223,7 @@ class SceneSentenceAudioServiceTest { try { val result = service( - RecordingExecutor(audioProbeResult = SceneCommandResult.Failed), + RecordingExecutor(audioProbeResults = mapOf("a:0" to SceneCommandResult.Failed)), ).prepare(request) assertEquals( @@ -163,23 +310,25 @@ class SceneSentenceAudioServiceTest { ) } - private fun audioInput(): SceneVideoInputSpec { + private fun audioInput(audioStreamIndex: Int? = null): SceneVideoInputSpec { return SceneVideoInputSpec( value = "https://media.example/audio.m4a", kind = SceneVideoInputKind.REMOTE_HTTP, headers = emptyList(), + audioStreamIndex = audioStreamIndex, ) } private class RecordingExecutor( - private val audioProbeResult: SceneCommandResult = SceneCommandResult.Success( - "codec_type=audio\ncodec_name=aac", + private val audioProbeResults: Map = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), ), private val ffmpegResult: SceneCommandResult = SceneCommandResult.Success(), private val ffmpegDelayMillis: Long = 0, ) : SceneCommandExecutor { var ffmpegCalls = 0 val ffprobeSelectors = mutableListOf() + val ffmpegAudioMaps = mutableListOf() override suspend fun executeFfmpeg( arguments: Array, @@ -187,6 +336,7 @@ class SceneSentenceAudioServiceTest { ): SceneCommandResult { return try { ffmpegCalls++ + ffmpegAudioMaps += arguments[arguments.indexOf("-map") + 1] if (ffmpegDelayMillis > 0) delay(ffmpegDelayMillis) if (ffmpegResult is SceneCommandResult.Success) { val output = File(arguments.last()) @@ -209,8 +359,8 @@ class SceneSentenceAudioServiceTest { "v:0" -> SceneCommandResult.Success( "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\nbits_per_raw_sample=10", ) - "a:0" -> audioProbeResult - else -> error("Unexpected stream selector: $selector") + else -> audioProbeResults[selector] + ?: error("Unexpected stream selector: $selector") } } finally { onNativeFinished() diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index 455a878219..63d9b24dd7 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -49,6 +49,9 @@ enum class AnkiSentenceAudioFailure { SOURCE_UNAVAILABLE, TIMING_UNAVAILABLE, AUDIO_PROBE_FAILED, + AUDIO_STREAM_INDEX_UNAVAILABLE, + AUDIO_STREAM_NOT_AUDIO, + AUDIO_STREAM_PROTECTED, AUDIO_STREAM_UNREADABLE, EXTRACTION_FAILED, EXTRACTION_TIMED_OUT, diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index 8bce703c8f..c2ea771b62 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -264,6 +264,9 @@ The selected video source could not be read for sentence audio. The card was created without it. The subtitle timing was unavailable. Sentence audio was not added. The selected audio stream could not be inspected. The card was created without sentence audio. + The audio track selected in the player was not found in this video. The card was created without sentence audio. + The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. + The selected audio stream is protected and cannot be exported. The card was created without sentence audio. No readable audio stream was found in this video. The card was created without sentence audio. Sentence audio extraction failed for this video. The card was created without it. Sentence audio extraction timed out. The card was created without it. From 231d967ff995522057f9da71c7f6284e94f1ec62 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Fri, 31 Jul 2026 02:19:30 +0700 Subject: [PATCH 4/9] fix(player): diagnose sentence audio export sources - distinguish original, MPV playable, and external audio inputs - detect missing audio streams separately from restricted codecs - show source-specific sentence audio export warnings --- .../tachiyomi/ui/player/PlayerViewModel.kt | 1 + .../ui/player/controls/PlayerSceneMiningUi.kt | 29 ++++++ .../player/scene/SceneSentenceAudioService.kt | 91 ++++++++++++++----- .../ui/player/scene/SceneVideoInput.kt | 43 ++++++++- .../scene/SceneSentenceAudioServiceTest.kt | 64 +++++++++++-- .../ui/player/scene/SceneVideoInputTest.kt | 46 +++++++++- .../java/chimahon/anki/AnkiCardCreator.kt | 14 ++- .../src/main/java/chimahon/anki/AnkiMedia.kt | 14 +++ .../moko-resources/base/strings.xml | 8 ++ 9 files changed, 270 insertions(+), 40 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt index 830de880c6..56ef258427 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt @@ -3281,6 +3281,7 @@ class PlayerViewModel @JvmOverloads internal constructor( ffmpegVideoArgs = emptyList(), seekable = mpv.seekable ?: stableLocalFile, audioStreamIndex = null, + isExternalAudio = true, ) } else { null diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index eec062b5fb..45b362870e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -8,6 +8,7 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.window.DialogProperties import chimahon.anki.AnkiMediaWarning import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource import eu.kanade.tachiyomi.ui.player.scene.PlayerSceneMiningProgress import eu.kanade.tachiyomi.util.system.toast import tachiyomi.i18n.kmk.KMR @@ -61,6 +62,34 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List { KMR.strings.anki_sentence_audio_probe_failed } + AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_streams_not_found_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_streams_not_found_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO -> { + KMR.strings.anki_sentence_audio_streams_not_found_external + } + null -> KMR.strings.anki_sentence_audio_streams_not_found + } + } + AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_codec_restricted_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_codec_restricted_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO -> { + KMR.strings.anki_sentence_audio_codec_restricted_external + } + null -> KMR.strings.anki_sentence_audio_codec_restricted + } + } AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE -> { KMR.strings.anki_sentence_audio_stream_index_unavailable } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 16cf307114..98182274fd 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -2,7 +2,9 @@ package eu.kanade.tachiyomi.ui.player.scene import android.content.Context import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioDiagnostic import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource import chimahon.anki.AnkiSentenceAudioPreparation import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers @@ -37,7 +39,9 @@ internal class FrozenSceneSentenceAudioService internal constructor( withContext(Dispatchers.IO) { val resolvedInput = when (val resolution = resolveAudioInput(input)) { is AudioInputResolution.Ready -> resolution.input - is AudioInputResolution.Unavailable -> return@withContext unavailable(resolution.failure) + is AudioInputResolution.Unavailable -> { + return@withContext unavailable(resolution.failure, resolution.diagnostic) + } } val lease = inputAcquirer.acquire(resolvedInput) ?: return@withContext unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) @@ -82,7 +86,7 @@ internal class FrozenSceneSentenceAudioService internal constructor( } private suspend fun resolveAudioInput(input: SceneVideoInputSpec): AudioInputResolution { - return when (val probe = executeAudioProbe(input, allAudioStreams = false)) { + return when (val probe = executeAudioProbe(input, AudioProbeMode.SELECTED_RESTRICTED)) { AudioProbeResult.SourceUnavailable -> { AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) } @@ -97,7 +101,7 @@ internal class FrozenSceneSentenceAudioService internal constructor( } SceneMediaProbe.AudioInspection.StreamMissing, SceneMediaProbe.AudioInspection.NotAudio -> { - resolveFallbackAudioInput(input, inspection) + resolveFallbackAudioInput(input) } } } @@ -106,9 +110,8 @@ internal class FrozenSceneSentenceAudioService internal constructor( private suspend fun resolveFallbackAudioInput( input: SceneVideoInputSpec, - selectedInspection: SceneMediaProbe.AudioInspection, ): AudioInputResolution { - return when (val probe = executeAudioProbe(input, allAudioStreams = true)) { + return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_RESTRICTED)) { AudioProbeResult.SourceUnavailable -> { AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) } @@ -123,43 +126,59 @@ internal class FrozenSceneSentenceAudioService internal constructor( AudioInputResolution.Ready(input.copy(audioStreamIndex = onlyStream.index)) } streams.size > 1 -> { - AudioInputResolution.Unavailable( - AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, - ) + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE) } onlyStream?.protected == true -> { AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED) } onlyStream != null -> { - AudioInputResolution.Unavailable( - AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, - ) - } - selectedInspection is SceneMediaProbe.AudioInspection.StreamMissing -> { - AudioInputResolution.Unavailable( - AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE, - ) + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE) } else -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO) + resolveEmptyAudioInventory(input) } } } } } + private suspend fun resolveEmptyAudioInventory(input: SceneVideoInputSpec): AudioInputResolution { + return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY)) { + AudioProbeResult.SourceUnavailable -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + val failure = if (SceneMediaProbe.audioStreams(probe.output).isNotEmpty()) { + AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED + } else { + AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND + } + diagnosticUnavailable(input, failure) + } + } + } + private suspend fun executeAudioProbe( input: SceneVideoInputSpec, - allAudioStreams: Boolean, + mode: AudioProbeMode, ): AudioProbeResult { val lease = inputAcquirer.acquire(input) ?: return AudioProbeResult.SourceUnavailable val cleanup = SceneNativeCleanup(lease::close) return try { val probe = commandExecutor.executeFfprobe( - if (allAudioStreams) { - SceneFfmpegArguments.allAudioProbe(input, lease.ffmpegValue, lease.tlsCaFile) - } else { - SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + when (mode) { + AudioProbeMode.SELECTED_RESTRICTED -> { + SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } + AudioProbeMode.ALL_RESTRICTED -> { + SceneFfmpegArguments.allAudioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } + AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY -> { + SceneFfmpegArguments.audioDiscoveryProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } }, cleanup::nativeFinished, ) @@ -183,9 +202,16 @@ internal class FrozenSceneSentenceAudioService internal constructor( data class Unavailable( val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, ) : AudioInputResolution } + private enum class AudioProbeMode { + SELECTED_RESTRICTED, + ALL_RESTRICTED, + ALL_UNRESTRICTED_DISCOVERY, + } + private sealed interface AudioProbeResult { data object SourceUnavailable : AudioProbeResult @@ -196,8 +222,25 @@ internal class FrozenSceneSentenceAudioService internal constructor( ) : AudioProbeResult } - private fun unavailable(failure: AnkiSentenceAudioFailure) = - AnkiSentenceAudioPreparation.Unavailable(failure) + private fun diagnosticUnavailable( + input: SceneVideoInputSpec, + failure: AnkiSentenceAudioFailure, + ) = AudioInputResolution.Unavailable(failure, input.toDiagnostic()) + + private fun unavailable( + failure: AnkiSentenceAudioFailure, + diagnostic: AnkiSentenceAudioDiagnostic? = null, + ) = AnkiSentenceAudioPreparation.Unavailable(failure, diagnostic) + + private fun SceneVideoInputSpec.toDiagnostic(): AnkiSentenceAudioDiagnostic { + return AnkiSentenceAudioDiagnostic( + inputSource = when (origin) { + SceneVideoInputOrigin.ORIGINAL_VIDEO -> AnkiSentenceAudioInputSource.ORIGINAL_VIDEO + SceneVideoInputOrigin.PLAYABLE_VIDEO -> AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO + SceneVideoInputOrigin.EXTERNAL_AUDIO -> AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO + }, + ) + } private fun ByteArray.sha256(): String { return MessageDigest.getInstance("SHA-256") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt index e17d12ec83..368a5f5a28 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt @@ -11,12 +11,19 @@ internal enum class SceneVideoInputKind { REMOTE_HTTP, } +internal enum class SceneVideoInputOrigin { + ORIGINAL_VIDEO, + PLAYABLE_VIDEO, + EXTERNAL_AUDIO, +} + internal data class SceneVideoInputSpec( val value: String, val kind: SceneVideoInputKind, val headers: List>, val videoStreamIndex: Int? = null, val audioStreamIndex: Int? = null, + val origin: SceneVideoInputOrigin = SceneVideoInputOrigin.ORIGINAL_VIDEO, ) internal data class SceneVideoInputSnapshot( @@ -28,6 +35,7 @@ internal data class SceneVideoInputSnapshot( val seekable: Boolean?, val videoStreamIndex: Int? = null, val audioStreamIndex: Int? = null, + val isExternalAudio: Boolean = false, ) internal object SceneVideoInputResolver { @@ -43,12 +51,18 @@ internal object SceneVideoInputResolver { val original = snapshot.originalVideoValue.takeIf(String::isNotBlank) if (original != null && isTransient(original)) return null - val normalized = original?.let(::normalizeInput) + val normalizedOriginal = original?.let(::normalizeInput) + val normalized = normalizedOriginal ?: snapshot.playableValue?.takeIf(String::isNotBlank)?.let { playable -> if (isTransient(playable)) return null normalizeInput(playable) } ?: return null + val origin = when { + snapshot.isExternalAudio -> SceneVideoInputOrigin.EXTERNAL_AUDIO + normalizedOriginal != null -> SceneVideoInputOrigin.ORIGINAL_VIDEO + else -> SceneVideoInputOrigin.PLAYABLE_VIDEO + } val headers = when (normalized.second) { SceneVideoInputKind.REMOTE_HTTP -> validateRemoteInput(normalized.first, snapshot.headers) @@ -64,6 +78,7 @@ internal object SceneVideoInputResolver { headers = headers, videoStreamIndex = snapshot.videoStreamIndex?.takeIf { it >= 0 }, audioStreamIndex = snapshot.audioStreamIndex?.takeIf { it >= 0 }, + origin = origin, ) } @@ -267,6 +282,25 @@ internal object SceneFfmpegArguments { }.toTypedArray() } + fun audioDiscoveryProbe( + input: SceneVideoInputSpec, + acquiredInputValue: String, + tlsCaFile: String? = null, + ): Array { + return buildList { + addInputOptions(input, tlsCaFile, restrictDecoders = false) + add("-v") + add("error") + add("-select_streams") + add("a") + add("-show_entries") + add("stream=index,codec_type,codec_name:stream_side_data") + add("-of") + add("default=noprint_wrappers=0") + add(acquiredInputValue) + }.toTypedArray() + } + fun sentenceAudio( input: SceneVideoInputSpec, acquiredInputValue: String, @@ -299,9 +333,12 @@ internal object SceneFfmpegArguments { private fun MutableList.addInputOptions( input: SceneVideoInputSpec, tlsCaFile: String?, + restrictDecoders: Boolean = true, ) { - add("-codec_whitelist") - add(ALLOWED_INPUT_DECODERS) + if (restrictDecoders) { + add("-codec_whitelist") + add(ALLOWED_INPUT_DECODERS) + } if (input.kind == SceneVideoInputKind.REMOTE_HTTP) { require(!tlsCaFile.isNullOrBlank()) { "Remote scene input requires a CA bundle" } add("-tls_verify") diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt index a0db022bab..1e6c3179ff 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -2,7 +2,9 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioDiagnostic import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource import chimahon.anki.AnkiSentenceAudioPreparation import io.mockk.every import io.mockk.mockk @@ -53,7 +55,7 @@ class SceneSentenceAudioServiceTest { } @Test - fun `reports selected non-audio stream when FFprobe finds no audio`() = runBlocking { + fun `reports no audio in the original export input after unrestricted discovery`() = runBlocking { val request = request() try { @@ -63,11 +65,17 @@ class SceneSentenceAudioServiceTest { "a:0" to SceneCommandResult.Success("codec_type=video"), "a" to SceneCommandResult.Success(), ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), ), ).prepare(request) assertEquals( - AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO), + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), result, ) } finally { @@ -105,8 +113,13 @@ class SceneSentenceAudioServiceTest { } @Test - fun `reports missing selected stream when no audio stream can be resolved`() = runBlocking { - val request = request(audioInput(audioStreamIndex = 4)) + fun `reports codec restriction when discovery finds audio in MPV playable input`() = runBlocking { + val request = request( + audioInput( + audioStreamIndex = 4, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ), + ) try { val result = service( @@ -115,12 +128,24 @@ class SceneSentenceAudioServiceTest { "4" to SceneCommandResult.Success(), "a" to SceneCommandResult.Success(), ), + unrestrictedAudioProbeResult = SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + codec_name=unsupported_audio + [/STREAM] + """.trimIndent(), + ), ), ).prepare(request) assertEquals( AnkiSentenceAudioPreparation.Unavailable( - AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE, + failure = AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), ), result, ) @@ -130,8 +155,13 @@ class SceneSentenceAudioServiceTest { } @Test - fun `reports selected non-audio stream when no audio stream can be resolved`() = runBlocking { - val request = request(audioInput(audioStreamIndex = 0)) + fun `reports no audio in MPV selected external audio input`() = runBlocking { + val request = request( + audioInput( + audioStreamIndex = 0, + origin = SceneVideoInputOrigin.EXTERNAL_AUDIO, + ), + ) try { val result = service( @@ -140,12 +170,16 @@ class SceneSentenceAudioServiceTest { "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), "a" to SceneCommandResult.Success(), ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), ), ).prepare(request) assertEquals( AnkiSentenceAudioPreparation.Unavailable( - AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO, + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + ), ), result, ) @@ -310,12 +344,16 @@ class SceneSentenceAudioServiceTest { ) } - private fun audioInput(audioStreamIndex: Int? = null): SceneVideoInputSpec { + private fun audioInput( + audioStreamIndex: Int? = null, + origin: SceneVideoInputOrigin = SceneVideoInputOrigin.ORIGINAL_VIDEO, + ): SceneVideoInputSpec { return SceneVideoInputSpec( value = "https://media.example/audio.m4a", kind = SceneVideoInputKind.REMOTE_HTTP, headers = emptyList(), audioStreamIndex = audioStreamIndex, + origin = origin, ) } @@ -323,11 +361,13 @@ class SceneSentenceAudioServiceTest { private val audioProbeResults: Map = mapOf( "a:0" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), ), + private val unrestrictedAudioProbeResult: SceneCommandResult? = null, private val ffmpegResult: SceneCommandResult = SceneCommandResult.Success(), private val ffmpegDelayMillis: Long = 0, ) : SceneCommandExecutor { var ffmpegCalls = 0 val ffprobeSelectors = mutableListOf() + val unrestrictedFfprobeSelectors = mutableListOf() val ffmpegAudioMaps = mutableListOf() override suspend fun executeFfmpeg( @@ -355,6 +395,12 @@ class SceneSentenceAudioServiceTest { return try { val selector = arguments[arguments.indexOf("-select_streams") + 1] ffprobeSelectors += selector + val restrictDecoders = "-codec_whitelist" in arguments + if (!restrictDecoders) { + unrestrictedFfprobeSelectors += selector + return unrestrictedAudioProbeResult + ?: error("Unexpected unrestricted stream selector: $selector") + } when (selector) { "v:0" -> SceneCommandResult.Success( "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\nbits_per_raw_sample=10", diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt index 243daccfde..3f31545b3c 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt @@ -61,6 +61,29 @@ class SceneVideoInputTest { cases.forEach { assertNull(resolve(it)) } } + @Test + fun `resolved inputs retain whether export uses original playable or external audio`() { + val original = requireNotNull( + resolve(snapshot("/video/original.mkv", playableValue = "/video/playable.mkv")), + ) + val playable = requireNotNull( + resolve(snapshot("", playableValue = "/video/playable.mkv")), + ) + val externalAudio = requireNotNull( + resolve( + snapshot( + "/audio/episode.m4a", + playableValue = "/audio/episode.m4a", + isExternalAudio = true, + ), + ), + ) + + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, original.origin) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playable.origin) + assertEquals(SceneVideoInputOrigin.EXTERNAL_AUDIO, externalAudio.origin) + } + @Test fun `AVIF command has the single bounded native recipe`() { val input = supportedInput() @@ -125,6 +148,7 @@ class SceneVideoInputTest { ), SceneFfmpegArguments.videoProbe(input, input.value, caFile), SceneFfmpegArguments.audioProbe(input, input.value, caFile), + SceneFfmpegArguments.allAudioProbe(input, input.value, caFile), SceneFfmpegArguments.sentenceAudio(input, input.value, range, "/cache/audio.m4a", caFile), ) @@ -138,6 +162,23 @@ class SceneVideoInputTest { } } + @Test + fun `audio discovery probe omits the decoder whitelist and only reads stream metadata`() { + val input = supportedInput() + val arguments = SceneFfmpegArguments.audioDiscoveryProbe( + input = input, + acquiredInputValue = input.value, + tlsCaFile = "/files/cacert.pem", + ).toList() + + assertFalse("-codec_whitelist" in arguments) + assertEquals("a", arguments[arguments.indexOf("-select_streams") + 1]) + assertEquals( + "stream=index,codec_type,codec_name:stream_side_data", + arguments[arguments.indexOf("-show_entries") + 1], + ) + } + @Test fun `sentence audio maps the frozen selected stream`() { val input = supportedInput().copy(videoStreamIndex = 2, audioStreamIndex = 3) @@ -179,16 +220,19 @@ class SceneVideoInputTest { private fun snapshot( value: String, + playableValue: String? = value, headers: List> = emptyList(), ffmpegStreamArgs: List> = emptyList(), seekable: Boolean = true, + isExternalAudio: Boolean = false, ) = SceneVideoInputSnapshot( originalVideoValue = value, - playableValue = value, + playableValue = playableValue, headers = headers, ffmpegStreamArgs = ffmpegStreamArgs, ffmpegVideoArgs = emptyList(), seekable = seekable, + isExternalAudio = isExternalAudio, ) private companion object { diff --git a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt index 96a98ec486..8bfdc65b99 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt @@ -385,6 +385,7 @@ object AnkiCardCreator { } var sentenceAudioGenerationFailure: AnkiSentenceAudioFailure? = null + var sentenceAudioGenerationDiagnostic: AnkiSentenceAudioDiagnostic? = null var sentenceAudioStorageFailed = false val sentenceAudioSource = if (hasSentenceAudioMarker) { val provider = mediaRequest?.sentenceAudioProvider @@ -401,7 +402,9 @@ object AnkiCardCreator { null } val lazySource = (lazyPreparation as? AnkiSentenceAudioPreparation.Ready)?.source - val lazyFailure = (lazyPreparation as? AnkiSentenceAudioPreparation.Unavailable)?.failure + val lazyUnavailable = lazyPreparation as? AnkiSentenceAudioPreparation.Unavailable + val lazyFailure = lazyUnavailable?.failure + val lazyDiagnostic = lazyUnavailable?.diagnostic val eagerSource = sentenceAudioBytes?.let { bytes -> val filename = generateSentenceAudioFilename(bytes, sentenceAudioExtension) AnkiMediaSource.Bytes( @@ -412,6 +415,7 @@ object AnkiCardCreator { } if (provider != null && lazySource == null && eagerSource == null) { sentenceAudioGenerationFailure = lazyFailure ?: AnkiSentenceAudioFailure.UNKNOWN + sentenceAudioGenerationDiagnostic = lazyDiagnostic } lazySource ?: eagerSource } else { @@ -534,8 +538,12 @@ object AnkiCardCreator { AnkiResult.Success( noteId, screenshotResult.warnings + listOfNotNull( - sentenceAudioGenerationFailure - ?.let(AnkiMediaWarning::SentenceAudioGenerationFailed), + sentenceAudioGenerationFailure?.let { failure -> + AnkiMediaWarning.SentenceAudioGenerationFailed( + failure, + sentenceAudioGenerationDiagnostic, + ) + }, AnkiMediaWarning.SentenceAudioStorageFailed .takeIf { sentenceAudioStorageFailed }, ), diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index 63d9b24dd7..ab36f14b82 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -49,6 +49,8 @@ enum class AnkiSentenceAudioFailure { SOURCE_UNAVAILABLE, TIMING_UNAVAILABLE, AUDIO_PROBE_FAILED, + AUDIO_STREAMS_NOT_FOUND, + AUDIO_CODEC_RESTRICTED, AUDIO_STREAM_INDEX_UNAVAILABLE, AUDIO_STREAM_NOT_AUDIO, AUDIO_STREAM_PROTECTED, @@ -58,6 +60,16 @@ enum class AnkiSentenceAudioFailure { UNKNOWN, } +enum class AnkiSentenceAudioInputSource { + ORIGINAL_VIDEO, + MPV_PLAYABLE_VIDEO, + MPV_EXTERNAL_AUDIO, +} + +data class AnkiSentenceAudioDiagnostic( + val inputSource: AnkiSentenceAudioInputSource, +) + sealed interface AnkiSentenceAudioPreparation { data class Ready( val source: AnkiMediaSource, @@ -65,6 +77,7 @@ sealed interface AnkiSentenceAudioPreparation { data class Unavailable( val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, ) : AnkiSentenceAudioPreparation } @@ -74,6 +87,7 @@ sealed interface AnkiMediaWarning { data object StillStorageFailed : AnkiMediaWarning data class SentenceAudioGenerationFailed( val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, ) : AnkiMediaWarning data object SentenceAudioStorageFailed : AnkiMediaWarning } diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index c2ea771b62..fc34af8338 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -264,6 +264,14 @@ The selected video source could not be read for sentence audio. The card was created without it. The subtitle timing was unavailable. Sentence audio was not added. The selected audio stream could not be inspected. The card was created without sentence audio. + FFmpeg could not find an audio stream in the source used for export. The card was created without sentence audio. + FFmpeg could not find an audio stream in the original video source used for export. MPV may be playing a different source. The card was created without sentence audio. + FFmpeg could not find an audio stream in MPV's playable video source. The card was created without sentence audio. + MPV selected external audio, but FFmpeg could not find an audio stream in that source. The card was created without sentence audio. + FFmpeg found audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. + FFmpeg found audio in the original video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + FFmpeg found audio in MPV's playable video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + MPV selected external audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. The audio track selected in the player was not found in this video. The card was created without sentence audio. The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. The selected audio stream is protected and cannot be exported. The card was created without sentence audio. From 513c9488f2b57e55c253713b55b30bd4cfd23656 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Fri, 31 Jul 2026 02:40:19 +0700 Subject: [PATCH 5/9] fix(player): fall back to MPV playable audio source - retry sentence audio export from MPV's playable source only when the original has no audio stream - preserve safe track mapping and source-specific failure diagnostics --- .../ui/player/scene/SceneCaptureRequest.kt | 11 +++ .../player/scene/SceneSentenceAudioService.kt | 23 ++++- .../ui/player/scene/SceneVideoInput.kt | 5 ++ .../player/scene/SceneCaptureRequestTest.kt | 35 +++++++- .../scene/SceneSentenceAudioServiceTest.kt | 83 ++++++++++++++++++- .../ui/player/scene/SceneVideoInputTest.kt | 16 ++++ 6 files changed, 167 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt index 52e7e21990..be74bc2f0a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt @@ -94,6 +94,7 @@ internal class SceneCaptureRequest( val sentenceAudioInput: SceneVideoInputSpec?, val sentenceAudioFailure: AnkiSentenceAudioFailure? = null, val resolvedTiming: SceneResolvedTiming?, + val sentenceAudioFallbackInput: SceneVideoInputSpec? = null, private val stillFallback: OwnedBitmap, ) : Closeable { fun fallbackBitmapOrNull(): Bitmap? = stillFallback.bitmapOrNull() @@ -333,11 +334,21 @@ internal class SceneCaptureRequestFactory( failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { videoInput == null }, ) } + val sentenceAudioFallbackInput = sentenceAudio.input + ?.takeIf { + beforeVideo.sentenceAudio == null && + it.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO + } + ?.let { originalInput -> + SceneVideoInputResolver.resolvePlayable(beforeVideo.video) + ?.takeIf { it.value != originalInput.value } + } val request = SceneCaptureRequest( videoInput = videoInput, sentenceAudioInput = sentenceAudio.input, sentenceAudioFailure = sentenceAudio.failure, resolvedTiming = resolveTiming(beforeMpv), + sentenceAudioFallbackInput = sentenceAudioFallbackInput, stillFallback = OwnedBitmap(fallback), ) transferred = true diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 98182274fd..2d01fe7efb 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -37,7 +37,9 @@ internal class FrozenSceneSentenceAudioService internal constructor( ?: return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) return withTimeoutOrNull(timeoutMillis) { withContext(Dispatchers.IO) { - val resolvedInput = when (val resolution = resolveAudioInput(input)) { + val resolvedInput = when ( + val resolution = resolveAudioInput(input, request.sentenceAudioFallbackInput) + ) { is AudioInputResolution.Ready -> resolution.input is AudioInputResolution.Unavailable -> { return@withContext unavailable(resolution.failure, resolution.diagnostic) @@ -85,7 +87,24 @@ internal class FrozenSceneSentenceAudioService internal constructor( } ?: unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) } - private suspend fun resolveAudioInput(input: SceneVideoInputSpec): AudioInputResolution { + private suspend fun resolveAudioInput( + input: SceneVideoInputSpec, + fallbackInput: SceneVideoInputSpec?, + ): AudioInputResolution { + val resolution = resolveSingleAudioInput(input) + if ( + resolution !is AudioInputResolution.Unavailable || + resolution.failure != AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND || + input.origin != SceneVideoInputOrigin.ORIGINAL_VIDEO + ) { + return resolution + } + val playableFallback = fallbackInput ?: return resolution + if (playableFallback.origin != SceneVideoInputOrigin.PLAYABLE_VIDEO) return resolution + return resolveSingleAudioInput(playableFallback) + } + + private suspend fun resolveSingleAudioInput(input: SceneVideoInputSpec): AudioInputResolution { return when (val probe = executeAudioProbe(input, AudioProbeMode.SELECTED_RESTRICTED)) { AudioProbeResult.SourceUnavailable -> { AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt index 368a5f5a28..14b516f4fa 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt @@ -82,6 +82,11 @@ internal object SceneVideoInputResolver { ) } + fun resolvePlayable(snapshot: SceneVideoInputSnapshot): SceneVideoInputSpec? { + if (snapshot.isExternalAudio) return null + return resolve(snapshot.copy(originalVideoValue = "")) + } + private fun validateRemoteInput( value: String, headers: List>, diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt index 913621f10b..d83bea3409 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt @@ -130,6 +130,35 @@ class SceneCaptureRequestTest { request.close() } + @Test + fun `original sentence audio input keeps a different MPV playable source as fallback`() = runTest { + val bitmap = mockBitmap() + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader(SequencePropertyReader(listOf(snapshot(), snapshot()))), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { + inputSnapshot( + originalVideoValue = "https://media.example/original.m3u8", + playableValue = "https://media.example/playable.m3u8", + ) + }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + assertNotNull(request) + val capturedRequest = requireNotNull(request) + val originalInput = requireNotNull(capturedRequest.sentenceAudioInput) + val playableInput = requireNotNull(capturedRequest.sentenceAudioFallbackInput) + assertEquals("https://media.example/original.m3u8", originalInput.value) + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, originalInput.origin) + assertEquals("https://media.example/playable.m3u8", playableInput.value) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playableInput.origin) + capturedRequest.close() + } + private fun mockBitmap(): Bitmap { return mockk(relaxed = true).also { every { it.isRecycled } returns false @@ -137,11 +166,13 @@ class SceneCaptureRequestTest { } private fun inputSnapshot( + originalVideoValue: String = "https://media.example/video.mp4", + playableValue: String = originalVideoValue, headers: List> = emptyList(), ): SceneCaptureInputSnapshot { val video = SceneVideoInputSnapshot( - originalVideoValue = "https://media.example/video.mp4", - playableValue = "https://media.example/video.mp4", + originalVideoValue = originalVideoValue, + playableValue = playableValue, headers = headers, ffmpegStreamArgs = emptyList(), ffmpegVideoArgs = emptyList(), diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt index 1e6c3179ff..3a6578b1ed 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -83,6 +83,78 @@ class SceneSentenceAudioServiceTest { } } + @Test + fun `exports audio from MPV playable input when original source has no audio`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("a:0", "a", "a", "1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `reports MPV playable input when fallback also has no audio`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + "1" to SceneCommandResult.Success("codec_type=video"), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + @Test fun `normalizes a wrong MPV stream index when exactly one audio stream is readable`() = runBlocking { val executor = RecordingExecutor( @@ -325,7 +397,10 @@ class SceneSentenceAudioServiceTest { ) } - private fun request(sentenceAudioInput: SceneVideoInputSpec? = audioInput()): SceneCaptureRequest { + private fun request( + sentenceAudioInput: SceneVideoInputSpec? = audioInput(), + sentenceAudioFallbackInput: SceneVideoInputSpec? = null, + ): SceneCaptureRequest { val video = SceneVideoInputSpec( value = "https://media.example/video.mkv", kind = SceneVideoInputKind.REMOTE_HTTP, @@ -336,6 +411,7 @@ class SceneSentenceAudioServiceTest { return SceneCaptureRequest( videoInput = video, sentenceAudioInput = sentenceAudioInput, + sentenceAudioFallbackInput = sentenceAudioFallbackInput, resolvedTiming = SceneResolvedTiming( animationRange = SceneTimeRange(1.25, 4.25), audioRange = SceneTimeRange(1.25, 4.25), @@ -345,11 +421,12 @@ class SceneSentenceAudioServiceTest { } private fun audioInput( + value: String = "https://media.example/audio.m4a", audioStreamIndex: Int? = null, origin: SceneVideoInputOrigin = SceneVideoInputOrigin.ORIGINAL_VIDEO, ): SceneVideoInputSpec { return SceneVideoInputSpec( - value = "https://media.example/audio.m4a", + value = value, kind = SceneVideoInputKind.REMOTE_HTTP, headers = emptyList(), audioStreamIndex = audioStreamIndex, @@ -369,6 +446,7 @@ class SceneSentenceAudioServiceTest { val ffprobeSelectors = mutableListOf() val unrestrictedFfprobeSelectors = mutableListOf() val ffmpegAudioMaps = mutableListOf() + val ffmpegInputs = mutableListOf() override suspend fun executeFfmpeg( arguments: Array, @@ -376,6 +454,7 @@ class SceneSentenceAudioServiceTest { ): SceneCommandResult { return try { ffmpegCalls++ + ffmpegInputs += arguments[arguments.indexOf("-i") + 1] ffmpegAudioMaps += arguments[arguments.indexOf("-map") + 1] if (ffmpegDelayMillis > 0) delay(ffmpegDelayMillis) if (ffmpegResult is SceneCommandResult.Success) { diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt index 3f31545b3c..26658d1aea 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt @@ -84,6 +84,22 @@ class SceneVideoInputTest { assertEquals(SceneVideoInputOrigin.EXTERNAL_AUDIO, externalAudio.origin) } + @Test + fun `playable resolution keeps MPV source separate from an original video source`() { + val input = snapshot( + value = "https://media.example/original.m3u8", + playableValue = "https://media.example/playable.m3u8", + ) + + val original = requireNotNull(resolve(input)) + val playable = requireNotNull(SceneVideoInputResolver.resolvePlayable(input)) + + assertEquals("https://media.example/original.m3u8", original.value) + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, original.origin) + assertEquals("https://media.example/playable.m3u8", playable.value) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playable.origin) + } + @Test fun `AVIF command has the single bounded native recipe`() { val input = supportedInput() From 61c3f3ff48a3577603406f29cc35677de5b50570 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Sat, 1 Aug 2026 00:51:20 +0700 Subject: [PATCH 6/9] fix(player) --- .../ui/player/controls/PlayerSceneMiningUi.kt | 55 +++- .../scene/AndroidSceneCaptureService.kt | 3 +- .../scene/FfmpegKitSceneCommandExecutor.kt | 94 +++++- .../ui/player/scene/SceneCaptureRequest.kt | 20 +- .../player/scene/SceneSentenceAudioService.kt | 291 ++++++++++++++++-- .../ui/player/scene/SceneVideoInput.kt | 36 ++- .../scene/SentenceAudioDiagnosticLog.kt | 171 ++++++++++ .../scene/AndroidSceneCaptureServiceTest.kt | 4 - .../player/scene/SceneCaptureRequestTest.kt | 24 ++ .../scene/SceneFfmpegFailureClassifierTest.kt | 61 ++++ .../scene/SceneSentenceAudioServiceTest.kt | 263 +++++++++++++++- .../ui/player/scene/SceneVideoInputTest.kt | 41 ++- .../scene/SentenceAudioDiagnosticLogTest.kt | 102 ++++++ .../src/main/java/chimahon/anki/AnkiMedia.kt | 17 + .../moko-resources/base/strings.xml | 15 +- 15 files changed, 1140 insertions(+), 57 deletions(-) create mode 100644 app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt create mode 100644 app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt create mode 100644 app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index 45b362870e..cd41796a1d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -9,6 +9,7 @@ import androidx.compose.ui.window.DialogProperties import chimahon.anki.AnkiMediaWarning import chimahon.anki.AnkiSentenceAudioFailure import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback import eu.kanade.tachiyomi.ui.player.scene.PlayerSceneMiningProgress import eu.kanade.tachiyomi.util.system.toast import tachiyomi.i18n.kmk.KMR @@ -60,12 +61,35 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List { - KMR.strings.anki_sentence_audio_probe_failed + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_probe_failed_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_probe_failed_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + null -> KMR.strings.anki_sentence_audio_probe_failed + } } AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND -> { when (warning.diagnostic?.inputSource) { AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { - KMR.strings.anki_sentence_audio_streams_not_found_original + when (warning.diagnostic?.playableFallback) { + AnkiSentenceAudioPlayableFallback.MISSING -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_missing + } + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_same + } + AnkiSentenceAudioPlayableFallback.UNAVAILABLE -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_unavailable + } + null -> KMR.strings.anki_sentence_audio_streams_not_found_original + } } AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { KMR.strings.anki_sentence_audio_streams_not_found_playable @@ -105,6 +129,33 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List { KMR.strings.anki_sentence_audio_extraction_failed } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING -> { + KMR.strings.anki_sentence_audio_extraction_output_missing + } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_output_read_failed + } + AnkiSentenceAudioFailure.EXTRACTION_STREAM_MAPPING_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_stream_mapping_failed + } + AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_extraction_source_read_failed_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_extraction_source_read_failed_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + null -> KMR.strings.anki_sentence_audio_extraction_source_read_failed + } + } + AnkiSentenceAudioFailure.EXTRACTION_SEEK_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_seek_failed + } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_WRITE_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_output_write_failed + } AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT -> { KMR.strings.anki_sentence_audio_extraction_timed_out } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt index 899b811ff7..443a858b20 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt @@ -66,7 +66,8 @@ internal class AndroidSceneCaptureService private constructor( outputCleanup.nativeFinished() } when (result) { - SceneCommandResult.Failed -> { + SceneCommandResult.Failed, + is SceneCommandResult.FfmpegFailed -> { return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null) } is SceneCommandResult.Success -> Unit diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt index 3bc5f60bcd..7665eefa53 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt @@ -16,9 +16,75 @@ import kotlin.coroutines.resume internal sealed interface SceneCommandResult { data class Success(val output: String = "") : SceneCommandResult + data class FfmpegFailed( + val failure: SceneFfmpegFailure, + val nativeDiagnostics: SceneNativeFailureDiagnostics? = null, + ) : SceneCommandResult + data object Failed : SceneCommandResult } +/** + * Raw native details remain in-memory until the temporary, app-private sentence-audio journal + * redacts them. They must never be passed to UI warnings or regular application logs. + */ +internal data class SceneNativeFailureDiagnostics( + val returnCode: Int?, + val failStackTrace: String?, + val logs: String?, +) + +internal enum class SceneFfmpegFailure { + STREAM_MAPPING, + SOURCE_READ, + SEEK, + OUTPUT_WRITE, + UNKNOWN, +} + +/** + * Converts untrusted native diagnostics into a closed, user-safe category. Raw logs may include + * URLs or request headers, so they are never exposed to the UI; the temporary sentence-audio + * journal is responsible for redacting any diagnostic detail it writes. + */ +internal fun classifySceneFfmpegFailure( + failStackTrace: String?, + logs: String?, +): SceneFfmpegFailure { + val detail = sequenceOf(failStackTrace, logs) + .filterNotNull() + .joinToString(separator = "\n") + .lowercase() + + return when { + "stream map" in detail && "matches no streams" in detail -> { + SceneFfmpegFailure.STREAM_MAPPING + } + "could not seek" in detail || "failed to seek" in detail || "invalid seek" in detail -> { + SceneFfmpegFailure.SEEK + } + "could not write header for output" in detail || + "error opening output" in detail || + "failed to avio_open" in detail || + "error writing trailer" in detail || + "error muxing a packet" in detail -> { + SceneFfmpegFailure.OUTPUT_WRITE + } + "http error" in detail || + "server returned" in detail || + "failed to open segment" in detail || + "error when loading first segment" in detail || + "unable to open resource" in detail || + "connection refused" in detail || + "connection reset by peer" in detail || + "network is unreachable" in detail || + "connection timed out" in detail -> { + SceneFfmpegFailure.SOURCE_READ + } + else -> SceneFfmpegFailure.UNKNOWN + } +} + internal interface SceneCommandExecutor { suspend fun executeFfmpeg( arguments: Array, @@ -79,7 +145,18 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { if (ReturnCode.isSuccess(session.returnCode)) { SceneCommandResult.Success() } else { - SceneCommandResult.Failed + val diagnostics = SceneNativeFailureDiagnostics( + returnCode = session.returnCode?.getValue(), + failStackTrace = session.failStackTrace, + logs = session.getAllLogsAsString(FAILURE_LOG_WAIT_MILLIS), + ) + SceneCommandResult.FfmpegFailed( + classifySceneFfmpegFailure( + failStackTrace = diagnostics.failStackTrace, + logs = diagnostics.logs, + ), + nativeDiagnostics = diagnostics, + ) } }, onNativeFinished = onNativeFinished, @@ -105,7 +182,18 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { if (ReturnCode.isSuccess(session.returnCode)) { SceneCommandResult.Success(session.output.orEmpty()) } else { - SceneCommandResult.Failed + val diagnostics = SceneNativeFailureDiagnostics( + returnCode = session.returnCode?.getValue(), + failStackTrace = session.failStackTrace, + logs = session.getAllLogsAsString(FAILURE_LOG_WAIT_MILLIS), + ) + SceneCommandResult.FfmpegFailed( + classifySceneFfmpegFailure( + failStackTrace = diagnostics.failStackTrace, + logs = diagnostics.logs, + ), + nativeDiagnostics = diagnostics, + ) } }, onNativeFinished = onNativeFinished, @@ -201,6 +289,8 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { } private companion object { + // Temporary diagnostic window: only native failures wait for late FFmpeg/FFprobe lines. + const val FAILURE_LOG_WAIT_MILLIS = 1_000 val DISCARD_LOG_CALLBACK = LogCallback {} val DISCARD_STATISTICS_CALLBACK = StatisticsCallback {} } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt index be74bc2f0a..49030a7d92 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt @@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPlayableFallback import `is`.xyz.mpv.MPVLib import java.io.Closeable import kotlin.math.abs @@ -95,6 +96,7 @@ internal class SceneCaptureRequest( val sentenceAudioFailure: AnkiSentenceAudioFailure? = null, val resolvedTiming: SceneResolvedTiming?, val sentenceAudioFallbackInput: SceneVideoInputSpec? = null, + val sentenceAudioFallbackStatus: AnkiSentenceAudioPlayableFallback? = null, private val stillFallback: OwnedBitmap, ) : Closeable { fun fallbackBitmapOrNull(): Bitmap? = stillFallback.bitmapOrNull() @@ -334,21 +336,33 @@ internal class SceneCaptureRequestFactory( failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { videoInput == null }, ) } - val sentenceAudioFallbackInput = sentenceAudio.input + val sentenceAudioFallbackResolution = sentenceAudio.input ?.takeIf { beforeVideo.sentenceAudio == null && it.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO } ?.let { originalInput -> - SceneVideoInputResolver.resolvePlayable(beforeVideo.video) - ?.takeIf { it.value != originalInput.value } + SceneVideoInputResolver.resolvePlayableFallback(beforeVideo.video, originalInput) } + val sentenceAudioFallbackInput = (sentenceAudioFallbackResolution + as? ScenePlayableFallbackResolution.Available) + ?.input + val sentenceAudioFallbackStatus = when (sentenceAudioFallbackResolution) { + ScenePlayableFallbackResolution.Missing -> AnkiSentenceAudioPlayableFallback.MISSING + ScenePlayableFallbackResolution.SameAsOriginal -> { + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL + } + ScenePlayableFallbackResolution.Unavailable -> AnkiSentenceAudioPlayableFallback.UNAVAILABLE + is ScenePlayableFallbackResolution.Available, + null -> null + } val request = SceneCaptureRequest( videoInput = videoInput, sentenceAudioInput = sentenceAudio.input, sentenceAudioFailure = sentenceAudio.failure, resolvedTiming = resolveTiming(beforeMpv), sentenceAudioFallbackInput = sentenceAudioFallbackInput, + sentenceAudioFallbackStatus = sentenceAudioFallbackStatus, stillFallback = OwnedBitmap(fallback), ) transferred = true diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 2d01fe7efb..e04f14b76e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -5,6 +5,7 @@ import chimahon.anki.AnkiMediaSource import chimahon.anki.AnkiSentenceAudioDiagnostic import chimahon.anki.AnkiSentenceAudioFailure import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback import chimahon.anki.AnkiSentenceAudioPreparation import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers @@ -23,22 +24,47 @@ internal class FrozenSceneSentenceAudioService internal constructor( private val inputAcquirer: SceneInputAcquirer, private val commandExecutor: SceneCommandExecutor, private val timeoutMillis: Long = AUDIO_TIMEOUT_MILLIS, + private val diagnosticLogger: SentenceAudioDiagnosticLogger = NoOpSentenceAudioDiagnosticLogger, ) : SceneSentenceAudioService { constructor(context: Context) : this( cacheDirectory = context.cacheDir, inputAcquirer = AndroidSceneInputAcquirer(context), commandExecutor = FfmpegKitSceneCommandExecutor(), + diagnosticLogger = createSentenceAudioDiagnosticLogger(), ) override suspend fun prepare(request: SceneCaptureRequest): AnkiSentenceAudioPreparation { - val input = request.sentenceAudioInput - ?: return unavailable(request.sentenceAudioFailure ?: AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) - val range = request.resolvedTiming?.audioRange - ?: return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) - return withTimeoutOrNull(timeoutMillis) { + val input = request.sentenceAudioInput ?: run { + val failure = request.sentenceAudioFailure ?: AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = failure, + ), + ) + return unavailable(failure) + } + val range = request.resolvedTiming?.audioRange ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = input, + fallback = input.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.TIMING_UNAVAILABLE, + ), + ) + return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) + } + val preparation = withTimeoutOrNull(timeoutMillis) { withContext(Dispatchers.IO) { val resolvedInput = when ( - val resolution = resolveAudioInput(input, request.sentenceAudioFallbackInput) + val resolution = resolveAudioInput( + input = input, + fallbackInput = request.sentenceAudioFallbackInput, + fallbackStatus = request.sentenceAudioFallbackStatus, + ) ) { is AudioInputResolution.Ready -> resolution.input is AudioInputResolution.Unavailable -> { @@ -46,7 +72,20 @@ internal class FrozenSceneSentenceAudioService internal constructor( } } val lease = inputAcquirer.acquire(resolvedInput) - ?: return@withContext unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ) + return@withContext diagnosticUnavailable( + resolvedInput, + AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ).toPreparation() + } val output = File(cacheDirectory, "chimahon_sentence_audio_${UUID.randomUUID()}.m4a") val inputCleanup = SceneNativeCleanup(lease::close) val outputCleanup = SceneNativeCleanup(output::delete) @@ -64,10 +103,55 @@ internal class FrozenSceneSentenceAudioService internal constructor( inputCleanup.nativeFinished() outputCleanup.nativeFinished() } - if (result !is SceneCommandResult.Success || !output.isFile || output.length() == 0L) { - return@withContext unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED) + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = result.toSentenceAudioFailure() + .takeIf { result !is SceneCommandResult.Success }, + result = result, + ), + ) + if (result !is SceneCommandResult.Success) { + return@withContext unavailable( + failure = result.toSentenceAudioFailure(), + diagnostic = result.toSentenceAudioDiagnostic(resolvedInput), + ) + } + if (!output.isFile || output.length() == 0L) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.OUTPUT_VALIDATION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + result = result, + ), + ) + return@withContext unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + diagnostic = resolvedInput.toDiagnostic(), + ) + } + val bytes = try { + output.readBytes() + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.OUTPUT_READ, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED, + result = result, + exceptionType = e.javaClass.name, + ), + ) + return@withContext unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED, + diagnostic = resolvedInput.toDiagnostic(), + ) } - val bytes = output.readBytes() AnkiSentenceAudioPreparation.Ready( AnkiMediaSource.Bytes( data = bytes, @@ -77,40 +161,88 @@ internal class FrozenSceneSentenceAudioService internal constructor( ) } catch (e: CancellationException) { throw e - } catch (_: Exception) { - unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED) + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_FAILED, + exceptionType = e.javaClass.name, + ), + ) + unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_FAILED, + diagnostic = resolvedInput.toDiagnostic(), + ) } finally { inputCleanup.release() outputCleanup.release() } } - } ?: unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) + } + return preparation ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = input, + fallback = input.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT, + ), + ) + unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) + } } private suspend fun resolveAudioInput( input: SceneVideoInputSpec, fallbackInput: SceneVideoInputSpec?, + fallbackStatus: AnkiSentenceAudioPlayableFallback?, ): AudioInputResolution { val resolution = resolveSingleAudioInput(input) if ( resolution !is AudioInputResolution.Unavailable || - resolution.failure != AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND || + !resolution.failure.isPlayableFallbackRetryable() || input.origin != SceneVideoInputOrigin.ORIGINAL_VIDEO ) { return resolution } - val playableFallback = fallbackInput ?: return resolution - if (playableFallback.origin != SceneVideoInputOrigin.PLAYABLE_VIDEO) return resolution + val playableFallback = fallbackInput?.takeIf { + it.origin == SceneVideoInputOrigin.PLAYABLE_VIDEO && it.value != input.value + } + if (playableFallback == null) { + val effectiveFallbackStatus = fallbackStatus ?: when { + fallbackInput?.value == input.value -> AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL + else -> null + } + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.FALLBACK_DECISION, + input = input, + fallback = effectiveFallbackStatus.toDiagnosticFallback(), + failure = resolution.failure, + ), + ) + return resolution.withFallbackStatus(input, effectiveFallbackStatus) + } + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.FALLBACK_DECISION, + input = input, + fallback = SentenceAudioDiagnosticFallback.ATTEMPTED, + failure = resolution.failure, + ), + ) return resolveSingleAudioInput(playableFallback) } private suspend fun resolveSingleAudioInput(input: SceneVideoInputSpec): AudioInputResolution { return when (val probe = executeAudioProbe(input, AudioProbeMode.SELECTED_RESTRICTED)) { AudioProbeResult.SourceUnavailable -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) } AudioProbeResult.ExecutionFailed -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) } is AudioProbeResult.Success -> { when (val inspection = SceneMediaProbe.inspectSelectedAudio(probe.output)) { @@ -132,10 +264,10 @@ internal class FrozenSceneSentenceAudioService internal constructor( ): AudioInputResolution { return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_RESTRICTED)) { AudioProbeResult.SourceUnavailable -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) } AudioProbeResult.ExecutionFailed -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) } is AudioProbeResult.Success -> { val streams = SceneMediaProbe.audioStreams(probe.output) @@ -164,10 +296,10 @@ internal class FrozenSceneSentenceAudioService internal constructor( private suspend fun resolveEmptyAudioInventory(input: SceneVideoInputSpec): AudioInputResolution { return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY)) { AudioProbeResult.SourceUnavailable -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) } AudioProbeResult.ExecutionFailed -> { - AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) } is AudioProbeResult.Success -> { val failure = if (SceneMediaProbe.audioStreams(probe.output).isNotEmpty()) { @@ -184,7 +316,19 @@ internal class FrozenSceneSentenceAudioService internal constructor( input: SceneVideoInputSpec, mode: AudioProbeMode, ): AudioProbeResult { - val lease = inputAcquirer.acquire(input) ?: return AudioProbeResult.SourceUnavailable + val stage = mode.toDiagnosticStage() + val fallback = input.toDiagnosticFallback() + val lease = inputAcquirer.acquire(input) ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ) + return AudioProbeResult.SourceUnavailable + } val cleanup = SceneNativeCleanup(lease::close) return try { val probe = commandExecutor.executeFfprobe( @@ -201,13 +345,33 @@ internal class FrozenSceneSentenceAudioService internal constructor( }, cleanup::nativeFinished, ) + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED + .takeIf { probe !is SceneCommandResult.Success }, + result = probe, + ), + ) when (probe) { - SceneCommandResult.Failed -> AudioProbeResult.ExecutionFailed + SceneCommandResult.Failed, + is SceneCommandResult.FfmpegFailed -> AudioProbeResult.ExecutionFailed is SceneCommandResult.Success -> AudioProbeResult.Success(probe.output) } } catch (e: CancellationException) { throw e - } catch (_: Exception) { + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + exceptionType = e.javaClass.name, + ), + ) AudioProbeResult.ExecutionFailed } finally { cleanup.release() @@ -246,21 +410,96 @@ internal class FrozenSceneSentenceAudioService internal constructor( failure: AnkiSentenceAudioFailure, ) = AudioInputResolution.Unavailable(failure, input.toDiagnostic()) + private fun AudioInputResolution.Unavailable.withFallbackStatus( + input: SceneVideoInputSpec, + fallbackStatus: AnkiSentenceAudioPlayableFallback?, + ): AudioInputResolution.Unavailable { + return fallbackStatus?.let { copy(diagnostic = input.toDiagnostic(it)) } ?: this + } + + private fun AudioInputResolution.Unavailable.toPreparation(): AnkiSentenceAudioPreparation { + return unavailable(failure, diagnostic) + } + private fun unavailable( failure: AnkiSentenceAudioFailure, diagnostic: AnkiSentenceAudioDiagnostic? = null, ) = AnkiSentenceAudioPreparation.Unavailable(failure, diagnostic) - private fun SceneVideoInputSpec.toDiagnostic(): AnkiSentenceAudioDiagnostic { + private fun SceneVideoInputSpec.toDiagnostic( + playableFallback: AnkiSentenceAudioPlayableFallback? = null, + ): AnkiSentenceAudioDiagnostic { return AnkiSentenceAudioDiagnostic( inputSource = when (origin) { SceneVideoInputOrigin.ORIGINAL_VIDEO -> AnkiSentenceAudioInputSource.ORIGINAL_VIDEO SceneVideoInputOrigin.PLAYABLE_VIDEO -> AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO SceneVideoInputOrigin.EXTERNAL_AUDIO -> AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO }, + playableFallback = playableFallback, ) } + private fun AudioProbeMode.toDiagnosticStage(): SentenceAudioDiagnosticStage { + return when (this) { + AudioProbeMode.SELECTED_RESTRICTED -> SentenceAudioDiagnosticStage.SELECTED_AUDIO_PROBE + AudioProbeMode.ALL_RESTRICTED -> SentenceAudioDiagnosticStage.ALL_AUDIO_PROBE + AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY -> SentenceAudioDiagnosticStage.AUDIO_DISCOVERY_PROBE + } + } + + private fun SceneVideoInputSpec.toDiagnosticFallback(): SentenceAudioDiagnosticFallback { + return if (origin == SceneVideoInputOrigin.PLAYABLE_VIDEO) { + SentenceAudioDiagnosticFallback.ATTEMPTED + } else { + SentenceAudioDiagnosticFallback.NOT_APPLICABLE + } + } + + private fun AnkiSentenceAudioPlayableFallback?.toDiagnosticFallback(): SentenceAudioDiagnosticFallback { + return when (this) { + AnkiSentenceAudioPlayableFallback.MISSING -> SentenceAudioDiagnosticFallback.MISSING + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL -> { + SentenceAudioDiagnosticFallback.SAME_AS_ORIGINAL + } + AnkiSentenceAudioPlayableFallback.UNAVAILABLE -> SentenceAudioDiagnosticFallback.UNAVAILABLE + null -> SentenceAudioDiagnosticFallback.NOT_APPLICABLE + } + } + + private fun AnkiSentenceAudioFailure.isPlayableFallbackRetryable(): Boolean { + return this == AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE || + this == AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED || + this == AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND + } + + private fun SceneCommandResult.toSentenceAudioFailure(): AnkiSentenceAudioFailure { + return when (this) { + is SceneCommandResult.Success, + SceneCommandResult.Failed -> AnkiSentenceAudioFailure.EXTRACTION_FAILED + is SceneCommandResult.FfmpegFailed -> when (failure) { + SceneFfmpegFailure.STREAM_MAPPING -> { + AnkiSentenceAudioFailure.EXTRACTION_STREAM_MAPPING_FAILED + } + SceneFfmpegFailure.SOURCE_READ -> { + AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED + } + SceneFfmpegFailure.SEEK -> AnkiSentenceAudioFailure.EXTRACTION_SEEK_FAILED + SceneFfmpegFailure.OUTPUT_WRITE -> { + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_WRITE_FAILED + } + SceneFfmpegFailure.UNKNOWN -> AnkiSentenceAudioFailure.EXTRACTION_FAILED + } + } + } + + private fun SceneCommandResult.toSentenceAudioDiagnostic( + input: SceneVideoInputSpec, + ): AnkiSentenceAudioDiagnostic? { + return (this as? SceneCommandResult.FfmpegFailed) + ?.takeUnless { it.failure == SceneFfmpegFailure.UNKNOWN } + ?.let { input.toDiagnostic() } + } + private fun ByteArray.sha256(): String { return MessageDigest.getInstance("SHA-256") .digest(this) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt index 14b516f4fa..6e0fa8a4ce 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt @@ -38,6 +38,19 @@ internal data class SceneVideoInputSnapshot( val isExternalAudio: Boolean = false, ) +internal sealed interface ScenePlayableFallbackResolution { + data class Available( + val input: SceneVideoInputSpec, + ) : ScenePlayableFallbackResolution + + data object Missing : ScenePlayableFallbackResolution + + data object SameAsOriginal : ScenePlayableFallbackResolution + + /** The player exposed a value, but it cannot safely be used for export. */ + data object Unavailable : ScenePlayableFallbackResolution +} + internal object SceneVideoInputResolver { fun resolve(snapshot: SceneVideoInputSnapshot): SceneVideoInputSpec? { if (snapshot.originalVideoValue.isBlank() && snapshot.playableValue.isNullOrBlank()) { @@ -87,6 +100,25 @@ internal object SceneVideoInputResolver { return resolve(snapshot.copy(originalVideoValue = "")) } + fun resolvePlayableFallback( + snapshot: SceneVideoInputSnapshot, + original: SceneVideoInputSpec, + ): ScenePlayableFallbackResolution { + if (snapshot.playableValue.isNullOrBlank()) { + return ScenePlayableFallbackResolution.Missing + } + val playable = resolvePlayable(snapshot) ?: return ScenePlayableFallbackResolution.Unavailable + return if ( + playable.value == original.value && + playable.kind == original.kind && + playable.headers == original.headers + ) { + ScenePlayableFallbackResolution.SameAsOriginal + } else { + ScenePlayableFallbackResolution.Available(playable) + } + } + private fun validateRemoteInput( value: String, headers: List>, @@ -346,10 +378,6 @@ internal object SceneFfmpegArguments { } if (input.kind == SceneVideoInputKind.REMOTE_HTTP) { require(!tlsCaFile.isNullOrBlank()) { "Remote scene input requires a CA bundle" } - add("-tls_verify") - add("1") - add("-ca_file") - add(tlsCaFile) add("-protocol_whitelist") add(REMOTE_PROTOCOLS) add("-rw_timeout") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt new file mode 100644 index 0000000000..3a566ffd57 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt @@ -0,0 +1,171 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import chimahon.anki.AnkiSentenceAudioFailure +import com.hippo.unifile.UniFile +import logcat.LogPriority +import logcat.logcat +import tachiyomi.domain.storage.service.StorageManager +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.io.IOException +import java.nio.charset.StandardCharsets +import kotlin.math.max + +/** + * Temporary diagnostic journal for sentence-audio failures. + * + * It is written to the existing `logs` directory below the storage folder selected by the user. + * It intentionally records only source metadata and redacted native diagnostics. It must be + * removed after the current stream-export investigation is complete. + */ +internal enum class SentenceAudioDiagnosticStage { + REQUEST_VALIDATION, + FALLBACK_DECISION, + SELECTED_AUDIO_PROBE, + ALL_AUDIO_PROBE, + AUDIO_DISCOVERY_PROBE, + AUDIO_EXTRACTION, + OUTPUT_VALIDATION, + OUTPUT_READ, +} + +internal enum class SentenceAudioDiagnosticFallback { + NOT_APPLICABLE, + MISSING, + SAME_AS_ORIGINAL, + UNAVAILABLE, + ATTEMPTED, +} + +internal data class SentenceAudioDiagnosticEvent( + val stage: SentenceAudioDiagnosticStage, + val input: SceneVideoInputSpec?, + val fallback: SentenceAudioDiagnosticFallback, + val failure: AnkiSentenceAudioFailure? = null, + val result: SceneCommandResult? = null, + val exceptionType: String? = null, +) + +internal fun interface SentenceAudioDiagnosticLogger { + fun record(event: SentenceAudioDiagnosticEvent) +} + +internal object NoOpSentenceAudioDiagnosticLogger : SentenceAudioDiagnosticLogger { + override fun record(event: SentenceAudioDiagnosticEvent) = Unit +} + +internal fun createSentenceAudioDiagnosticLogger( + storageManager: StorageManager = Injekt.get(), +): SentenceAudioDiagnosticLogger { + return StorageFolderSentenceAudioDiagnosticLogger( + directory = storageManager::getLogsDirectory, + onWriteFailure = { error -> + logcat("SentenceAudioDiagnostic", LogPriority.ERROR) { + "Sentence-audio diagnostic log could not be written to the selected storage folder: ${error.javaClass.simpleName}" + } + }, + ) +} + +internal class StorageFolderSentenceAudioDiagnosticLogger( + private val directory: () -> UniFile?, + private val maxBytes: Int = MAX_LOG_BYTES, + private val onWriteFailure: (Throwable) -> Unit = {}, +) : SentenceAudioDiagnosticLogger { + init { + require(maxBytes > 0) { "maxBytes must be positive" } + } + + override fun record(event: SentenceAudioDiagnosticEvent) { + runCatching { + val entry = SentenceAudioDiagnosticJournal.render(event).toByteArray(StandardCharsets.UTF_8) + synchronized(lock) { + val folder = directory() + ?: throw IOException("The selected storage folder is unavailable") + val file = folder.findFile(FILE_NAME) + ?: folder.createFile(FILE_NAME) + ?: throw IOException("Could not create the sentence-audio diagnostic log") + val existing = file.openInputStream().use { it.readBytes() } + file.openOutputStream().use { output -> + output.write(SentenceAudioDiagnosticJournal.retain(existing, entry, maxBytes)) + } + } + }.onFailure(onWriteFailure) + } + + private companion object { + val lock = Any() + } +} + +internal object SentenceAudioDiagnosticJournal { + fun retain(existing: ByteArray, entry: ByteArray, maxBytes: Int): ByteArray { + val combined = existing + entry + return if (combined.size <= maxBytes) { + combined + } else { + combined.copyOfRange(max(0, combined.size - maxBytes), combined.size) + } + } + + fun render(event: SentenceAudioDiagnosticEvent): String { + return buildString { + appendLine("recorded_at_utc=${System.currentTimeMillis()}") + appendLine("stage=${event.stage.name}") + event.input?.let { input -> + appendLine("input_source=${input.origin.name}") + appendLine("input_kind=${input.kind.name}") + appendLine("audio_stream_index=${input.audioStreamIndex ?: "none"}") + } + appendLine("fallback=${event.fallback.name}") + event.failure?.let { appendLine("failure=${it.name}") } + event.result?.let { appendResult(it) } + event.exceptionType?.let { appendLine("exception_type=$it") } + appendLine("---") + } + } + + private fun StringBuilder.appendResult(result: SceneCommandResult) { + when (result) { + is SceneCommandResult.Success -> appendLine("command_result=SUCCESS") + SceneCommandResult.Failed -> appendLine("command_result=EXECUTION_FAILED") + is SceneCommandResult.FfmpegFailed -> { + appendLine("command_result=NATIVE_FAILED") + appendLine("native_failure=${result.failure.name}") + result.nativeDiagnostics?.let { details -> + details.returnCode?.let { appendLine("return_code=$it") } + val detail = sequenceOf(details.failStackTrace, details.logs) + .filterNotNull() + .joinToString(separator = "\n") + .redactNativeDiagnostics() + .takeLast(MAX_NATIVE_DIAGNOSTIC_CHARS) + .trim() + if (detail.isNotEmpty()) { + appendLine("native_diagnostic_begin") + appendLine(detail) + appendLine("native_diagnostic_end") + } + } + } + } + } + + private fun String.redactNativeDiagnostics(): String { + return replace(URL_PATTERN, "") + .replace(SENSITIVE_QUERY_PATTERN, "${'$'}1=") + .replace(SENSITIVE_HEADER_PATTERN, "${'$'}1") + .replace(LOCAL_PATH_PATTERN, "") + } +} + +private const val FILE_NAME = "chimahon_sentence_audio_debug.log" +private const val MAX_LOG_BYTES = 64 * 1024 +private const val MAX_NATIVE_DIAGNOSTIC_CHARS = 32 * 1024 +private val URL_PATTERN = Regex("""(?i)\b(?:https?|file)://[^\s\"'<>]+""") +private val SENSITIVE_QUERY_PATTERN = Regex( + """(?i)\b(access_token|api_key|auth|authorization|credential|credentials|key|policy|signature|signed|sig|token|x-amz-[^=\s]+|x-goog-[^=\s]+)=([^&\s]+)""", +) +private val SENSITIVE_HEADER_PATTERN = Regex( + """(?im)^((?:authorization|cookie|referer|origin|user-agent|accept(?:-[a-z-]+)?|cache-control|pragma|proxy-authorization|x-[a-z0-9-]+)\s*:\s*).*$""", +) +private val LOCAL_PATH_PATTERN = Regex("""(?i)(?:[a-z]:\\|/(?:data|storage|sdcard|mnt|cache|files)/)[^\s\"'<>]+""") diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt index 8e0ec461ec..e5b8b15b5f 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt @@ -114,10 +114,6 @@ class AndroidSceneCaptureServiceTest { return arrayOf( "-codec_whitelist", SceneFfmpegArguments.ALLOWED_INPUT_DECODERS, - "-tls_verify", - "1", - "-ca_file", - "/files/cacert.pem", "-protocol_whitelist", "http,https,tls,tcp,crypto", "-rw_timeout", diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt index d83bea3409..9887152e36 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt @@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPlayableFallback import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -159,6 +160,29 @@ class SceneCaptureRequestTest { capturedRequest.close() } + @Test + fun `original sentence audio input records when MPV playable source is the same`() = runTest { + val bitmap = mockBitmap() + val source = "https://media.example/original.m3u8" + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader(SequencePropertyReader(listOf(snapshot(source), snapshot(source)))), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { inputSnapshot(originalVideoValue = source, playableValue = source) }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + val capturedRequest = requireNotNull(request) + assertNull(capturedRequest.sentenceAudioFallbackInput) + assertEquals( + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL, + capturedRequest.sentenceAudioFallbackStatus, + ) + capturedRequest.close() + } + private fun mockBitmap(): Bitmap { return mockk(relaxed = true).also { every { it.isRecycled } returns false diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt new file mode 100644 index 0000000000..8aa235de98 --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt @@ -0,0 +1,61 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SceneFfmpegFailureClassifierTest { + @Test + fun `classifies an HTTP failure while reading the video source`() { + assertEquals( + SceneFfmpegFailure.SOURCE_READ, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "[https @ 0x1] HTTP error 403 Forbidden", + ), + ) + } + + @Test + fun `classifies a missing mapped stream`() { + assertEquals( + SceneFfmpegFailure.STREAM_MAPPING, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Stream map '0:4' matches no streams.", + ), + ) + } + + @Test + fun `classifies an input seek failure`() { + assertEquals( + SceneFfmpegFailure.SEEK, + classifySceneFfmpegFailure( + failStackTrace = "Could not seek to position 12.500", + logs = null, + ), + ) + } + + @Test + fun `classifies a temporary output write failure`() { + assertEquals( + SceneFfmpegFailure.OUTPUT_WRITE, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Could not write header for output file #0: Invalid argument", + ), + ) + } + + @Test + fun `keeps unrecognised FFmpeg output generic`() { + assertEquals( + SceneFfmpegFailure.UNKNOWN, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Conversion failed.", + ), + ) + } +} diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt index 3a6578b1ed..47dda9acef 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -5,6 +5,7 @@ import chimahon.anki.AnkiMediaSource import chimahon.anki.AnkiSentenceAudioDiagnostic import chimahon.anki.AnkiSentenceAudioFailure import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback import chimahon.anki.AnkiSentenceAudioPreparation import io.mockk.every import io.mockk.mockk @@ -54,6 +55,30 @@ class SceneSentenceAudioServiceTest { } } + @Test + fun `records unavailable source when no sentence audio input was captured`() = runBlocking { + val diagnostics = RecordingDiagnosticLogger() + val request = request(sentenceAudioInput = null) + + try { + service(RecordingExecutor(), diagnosticLogger = diagnostics).prepare(request) + + assertEquals( + listOf( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ), + diagnostics.events, + ) + } finally { + request.close() + } + } + @Test fun `reports no audio in the original export input after unrestricted discovery`() = runBlocking { val request = request() @@ -116,6 +141,76 @@ class SceneSentenceAudioServiceTest { } } + @Test + fun `exports audio from MPV playable input when original FFprobe fails`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Failed, + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("a:0", "1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `exports audio from MPV playable input when original source cannot be acquired`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ) + + try { + val result = service( + executor = executor, + inputAcquirer = SceneInputAcquirer { input -> + if (input.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO) { + null + } else { + leaseFor(input) + } + }, + ).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + @Test fun `reports MPV playable input when fallback also has no audio`() = runBlocking { val original = audioInput(value = "https://media.example/original.m3u8") @@ -155,6 +250,72 @@ class SceneSentenceAudioServiceTest { } } + @Test + fun `reports original probe failure and missing playable fallback`() = runBlocking { + val request = request( + sentenceAudioFallbackStatus = AnkiSentenceAudioPlayableFallback.MISSING, + ) + + try { + val result = service( + RecordingExecutor(audioProbeResults = mapOf("a:0" to SceneCommandResult.Failed)), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + playableFallback = AnkiSentenceAudioPlayableFallback.MISSING, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `does not retry external audio after its probe fails`() = runBlocking { + val external = audioInput( + audioStreamIndex = 0, + origin = SceneVideoInputOrigin.EXTERNAL_AUDIO, + ) + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = external, + sentenceAudioFallbackInput = playable, + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Failed, + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + ), + ), + result, + ) + } finally { + request.close() + } + } + @Test fun `normalizes a wrong MPV stream index when exactly one audio stream is readable`() = runBlocking { val executor = RecordingExecutor( @@ -333,7 +494,12 @@ class SceneSentenceAudioServiceTest { ).prepare(request) assertEquals( - AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED), + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), result, ) } finally { @@ -359,20 +525,81 @@ class SceneSentenceAudioServiceTest { } } + @Test + fun `reports missing output when FFmpeg succeeds without creating sentence audio`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(writeAudioOutput = false), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports source read failure when FFmpeg cannot read MPV playable source`() = runBlocking { + val request = request( + audioInput(origin = SceneVideoInputOrigin.PLAYABLE_VIDEO), + ) + + try { + val result = service( + RecordingExecutor( + ffmpegResult = SceneCommandResult.FfmpegFailed(SceneFfmpegFailure.SOURCE_READ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + @Test fun `reports extraction timeout when FFmpeg does not finish in time`() = runBlocking { + val diagnostics = RecordingDiagnosticLogger() val request = request() try { val result = service( RecordingExecutor(ffmpegDelayMillis = 50), timeoutMillis = 1, + diagnosticLogger = diagnostics, ).prepare(request) assertEquals( AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT), result, ) + assertEquals( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = request.sentenceAudioInput, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT, + ), + diagnostics.events.last(), + ) } finally { request.close() } @@ -381,25 +608,31 @@ class SceneSentenceAudioServiceTest { private fun service( executor: RecordingExecutor, timeoutMillis: Long = 60_000L, + inputAcquirer: SceneInputAcquirer = SceneInputAcquirer { input -> leaseFor(input) }, + diagnosticLogger: SentenceAudioDiagnosticLogger = NoOpSentenceAudioDiagnosticLogger, ): FrozenSceneSentenceAudioService { return FrozenSceneSentenceAudioService( cacheDirectory = tempDirectory, - inputAcquirer = SceneInputAcquirer { input -> - object : SceneInputLease { - override val ffmpegValue = input.value - override val tlsCaFile = "/files/cacert.pem" - - override fun close() = Unit - } - }, + inputAcquirer = inputAcquirer, commandExecutor = executor, timeoutMillis = timeoutMillis, + diagnosticLogger = diagnosticLogger, ) } + private fun leaseFor(input: SceneVideoInputSpec): SceneInputLease { + return object : SceneInputLease { + override val ffmpegValue = input.value + override val tlsCaFile = "/files/cacert.pem" + + override fun close() = Unit + } + } + private fun request( sentenceAudioInput: SceneVideoInputSpec? = audioInput(), sentenceAudioFallbackInput: SceneVideoInputSpec? = null, + sentenceAudioFallbackStatus: AnkiSentenceAudioPlayableFallback? = null, ): SceneCaptureRequest { val video = SceneVideoInputSpec( value = "https://media.example/video.mkv", @@ -412,6 +645,7 @@ class SceneSentenceAudioServiceTest { videoInput = video, sentenceAudioInput = sentenceAudioInput, sentenceAudioFallbackInput = sentenceAudioFallbackInput, + sentenceAudioFallbackStatus = sentenceAudioFallbackStatus, resolvedTiming = SceneResolvedTiming( animationRange = SceneTimeRange(1.25, 4.25), audioRange = SceneTimeRange(1.25, 4.25), @@ -441,6 +675,7 @@ class SceneSentenceAudioServiceTest { private val unrestrictedAudioProbeResult: SceneCommandResult? = null, private val ffmpegResult: SceneCommandResult = SceneCommandResult.Success(), private val ffmpegDelayMillis: Long = 0, + private val writeAudioOutput: Boolean = true, ) : SceneCommandExecutor { var ffmpegCalls = 0 val ffprobeSelectors = mutableListOf() @@ -457,7 +692,7 @@ class SceneSentenceAudioServiceTest { ffmpegInputs += arguments[arguments.indexOf("-i") + 1] ffmpegAudioMaps += arguments[arguments.indexOf("-map") + 1] if (ffmpegDelayMillis > 0) delay(ffmpegDelayMillis) - if (ffmpegResult is SceneCommandResult.Success) { + if (ffmpegResult is SceneCommandResult.Success && writeAudioOutput) { val output = File(arguments.last()) output.writeBytes(byteArrayOf(1, 2, 3)) } @@ -492,4 +727,12 @@ class SceneSentenceAudioServiceTest { } } } + + private class RecordingDiagnosticLogger : SentenceAudioDiagnosticLogger { + val events = mutableListOf() + + override fun record(event: SentenceAudioDiagnosticEvent) { + events += event + } + } } diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt index 26658d1aea..c797c486eb 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt @@ -100,6 +100,43 @@ class SceneVideoInputTest { assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playable.origin) } + @Test + fun `playable fallback resolution distinguishes usable missing same and unsafe sources`() { + val originalValue = "https://media.example/original.m3u8" + val original = requireNotNull(resolve(snapshot(originalValue))) + + val available = SceneVideoInputResolver.resolvePlayableFallback( + snapshot( + value = originalValue, + playableValue = "https://media.example/playable.m3u8", + ), + original, + ) + val missing = SceneVideoInputResolver.resolvePlayableFallback( + snapshot(value = originalValue, playableValue = ""), + original, + ) + val same = SceneVideoInputResolver.resolvePlayableFallback( + snapshot(value = originalValue, playableValue = originalValue), + original, + ) + val unsafe = SceneVideoInputResolver.resolvePlayableFallback( + snapshot( + value = originalValue, + playableValue = "https://media.example/playable.m3u8?token=secret", + ), + original, + ) + + assertEquals( + "https://media.example/playable.m3u8", + (available as ScenePlayableFallbackResolution.Available).input.value, + ) + assertEquals(ScenePlayableFallbackResolution.Missing, missing) + assertEquals(ScenePlayableFallbackResolution.SameAsOriginal, same) + assertEquals(ScenePlayableFallbackResolution.Unavailable, unsafe) + } + @Test fun `AVIF command has the single bounded native recipe`() { val input = supportedInput() @@ -131,10 +168,6 @@ class SceneVideoInputTest { assertTrue( arguments.containsAll( listOf( - "-tls_verify", - "1", - "-ca_file", - "/files/cacert.pem", "-protocol_whitelist", "http,https,tls,tcp,crypto", "-rw_timeout", diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt new file mode 100644 index 0000000000..944ff97b6d --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt @@ -0,0 +1,102 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import chimahon.anki.AnkiSentenceAudioFailure +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SentenceAudioDiagnosticLogTest { + @Test + fun `records a redacted native failure with its sentence audio stage`() { + val input = SceneVideoInputSpec( + value = "https://media.example/episode.m3u8?token=secret", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = listOf("Authorization" to "Bearer secret"), + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + + val text = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = input, + fallback = SentenceAudioDiagnosticFallback.ATTEMPTED, + failure = AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED, + result = SceneCommandResult.FfmpegFailed( + failure = SceneFfmpegFailure.SOURCE_READ, + nativeDiagnostics = SceneNativeFailureDiagnostics( + returnCode = 1, + failStackTrace = null, + logs = "HTTP error 403 for https://media.example/episode.m3u8?token=secret\n" + + "Authorization: Bearer secret", + ), + ), + ), + ) + + assertTrue(text.contains("stage=AUDIO_EXTRACTION")) + assertTrue(text.contains("input_source=PLAYABLE_VIDEO")) + assertTrue(text.contains("input_kind=REMOTE_HTTP")) + assertTrue(text.contains("audio_stream_index=1")) + assertTrue(text.contains("fallback=ATTEMPTED")) + assertTrue(text.contains("failure=EXTRACTION_SOURCE_READ_FAILED")) + assertTrue(text.contains("return_code=1")) + assertTrue(text.contains("HTTP error 403")) + assertFalse(text.contains(input.value)) + assertFalse(text.contains("token=secret")) + assertFalse(text.contains("Bearer secret")) + } + + @Test + fun `keeps the newest entries within its byte budget`() { + var bytes = ByteArray(0) + + repeat(8) { + bytes = SentenceAudioDiagnosticJournal.retain( + existing = bytes, + entry = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.SELECTED_AUDIO_PROBE, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ).encodeToByteArray(), + maxBytes = 512, + ) + } + bytes = SentenceAudioDiagnosticJournal.retain( + existing = bytes, + entry = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ).encodeToByteArray(), + maxBytes = 512, + ) + + assertTrue(bytes.size <= 512) + assertTrue(bytes.decodeToString().contains("stage=AUDIO_EXTRACTION")) + } + + @Test + fun `reports a file writer failure instead of discarding it`() { + var writeFailure: Throwable? = null + val logger = StorageFolderSentenceAudioDiagnosticLogger( + directory = { null }, + onWriteFailure = { writeFailure = it }, + ) + + logger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ) + + assertNotNull(writeFailure) + } +} diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index ab36f14b82..2ede7313a3 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -56,6 +56,12 @@ enum class AnkiSentenceAudioFailure { AUDIO_STREAM_PROTECTED, AUDIO_STREAM_UNREADABLE, EXTRACTION_FAILED, + EXTRACTION_OUTPUT_MISSING, + EXTRACTION_OUTPUT_READ_FAILED, + EXTRACTION_STREAM_MAPPING_FAILED, + EXTRACTION_SOURCE_READ_FAILED, + EXTRACTION_SEEK_FAILED, + EXTRACTION_OUTPUT_WRITE_FAILED, EXTRACTION_TIMED_OUT, UNKNOWN, } @@ -66,8 +72,19 @@ enum class AnkiSentenceAudioInputSource { MPV_EXTERNAL_AUDIO, } +/** + * Safe, frozen reason why the original video could not be retried through MPV's playable source. + * It deliberately does not expose a path, URL, header, or player property. + */ +enum class AnkiSentenceAudioPlayableFallback { + MISSING, + SAME_AS_ORIGINAL, + UNAVAILABLE, +} + data class AnkiSentenceAudioDiagnostic( val inputSource: AnkiSentenceAudioInputSource, + val playableFallback: AnkiSentenceAudioPlayableFallback? = null, ) sealed interface AnkiSentenceAudioPreparation { diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index fc34af8338..1e319cb57d 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -264,8 +264,13 @@ The selected video source could not be read for sentence audio. The card was created without it. The subtitle timing was unavailable. Sentence audio was not added. The selected audio stream could not be inspected. The card was created without sentence audio. + FFprobe could not inspect audio in the original video source. The card was created without sentence audio. + FFprobe could not inspect audio in MPV's playable video source. The card was created without sentence audio. FFmpeg could not find an audio stream in the source used for export. The card was created without sentence audio. FFmpeg could not find an audio stream in the original video source used for export. MPV may be playing a different source. The card was created without sentence audio. + FFmpeg could not find an audio stream in the original video source, and MPV did not provide a separate playable source to try. The card was created without sentence audio. + FFmpeg could not find an audio stream in the original video source, and MPV's playable source was the same source. The card was created without sentence audio. + FFmpeg could not find an audio stream in the original video source, and MPV's separate playable source was not safe to use for export. The card was created without sentence audio. FFmpeg could not find an audio stream in MPV's playable video source. The card was created without sentence audio. MPV selected external audio, but FFmpeg could not find an audio stream in that source. The card was created without sentence audio. FFmpeg found audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. @@ -276,7 +281,15 @@ The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. The selected audio stream is protected and cannot be exported. The card was created without sentence audio. No readable audio stream was found in this video. The card was created without sentence audio. - Sentence audio extraction failed for this video. The card was created without it. + FFmpeg found an audio stream but could not extract sentence audio from this video. The card was created without it. + FFmpeg completed but did not create a usable sentence-audio file. The card was created without it. + FFmpeg created sentence audio, but Chimahon could not read its temporary file. The card was created without it. + FFmpeg could not map the selected audio track for sentence export. The card was created without it. + FFmpeg could not read the audio data from the video source during sentence export. The card was created without it. + FFmpeg could not read audio data from the original video source during sentence export. The card was created without it. + FFmpeg could not read audio data from MPV's playable video source during sentence export. The card was created without it. + FFmpeg could not seek the video source to the subtitle time for sentence export. The card was created without it. + FFmpeg could not write the temporary sentence audio file. The card was created without it. Sentence audio extraction timed out. The card was created without it. AnkiDroid could not store the sentence audio. The card was created without it. Media From 4ab295c91d7d460e869b7f041dcfd4c632dbad73 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Sat, 1 Aug 2026 01:02:21 +0700 Subject: [PATCH 7/9] Update strings.xml --- .../moko-resources/base/strings.xml | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index 1e319cb57d..18fae336bf 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -263,35 +263,35 @@ The selected audio track could not be identified safely. Sentence audio was not added to avoid choosing the wrong track. The selected video source could not be read for sentence audio. The card was created without it. The subtitle timing was unavailable. Sentence audio was not added. - The selected audio stream could not be inspected. The card was created without sentence audio. - FFprobe could not inspect audio in the original video source. The card was created without sentence audio. - FFprobe could not inspect audio in MPV's playable video source. The card was created without sentence audio. - FFmpeg could not find an audio stream in the source used for export. The card was created without sentence audio. - FFmpeg could not find an audio stream in the original video source used for export. MPV may be playing a different source. The card was created without sentence audio. - FFmpeg could not find an audio stream in the original video source, and MPV did not provide a separate playable source to try. The card was created without sentence audio. - FFmpeg could not find an audio stream in the original video source, and MPV's playable source was the same source. The card was created without sentence audio. - FFmpeg could not find an audio stream in the original video source, and MPV's separate playable source was not safe to use for export. The card was created without sentence audio. - FFmpeg could not find an audio stream in MPV's playable video source. The card was created without sentence audio. - MPV selected external audio, but FFmpeg could not find an audio stream in that source. The card was created without sentence audio. - FFmpeg found audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. - FFmpeg found audio in the original video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. - FFmpeg found audio in MPV's playable video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. - MPV selected external audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. - The audio track selected in the player was not found in this video. The card was created without sentence audio. - The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. - The selected audio stream is protected and cannot be exported. The card was created without sentence audio. - No readable audio stream was found in this video. The card was created without sentence audio. - FFmpeg found an audio stream but could not extract sentence audio from this video. The card was created without it. - FFmpeg completed but did not create a usable sentence-audio file. The card was created without it. - FFmpeg created sentence audio, but Chimahon could not read its temporary file. The card was created without it. - FFmpeg could not map the selected audio track for sentence export. The card was created without it. - FFmpeg could not read the audio data from the video source during sentence export. The card was created without it. - FFmpeg could not read audio data from the original video source during sentence export. The card was created without it. - FFmpeg could not read audio data from MPV's playable video source during sentence export. The card was created without it. - FFmpeg could not seek the video source to the subtitle time for sentence export. The card was created without it. - FFmpeg could not write the temporary sentence audio file. The card was created without it. - Sentence audio extraction timed out. The card was created without it. - AnkiDroid could not store the sentence audio. The card was created without it. + [Audio Probe Failed] The selected audio stream could not be inspected. The card was created without sentence audio. + [Audio Probe Failed (Original)] FFprobe could not inspect audio in the original video source. The card was created without sentence audio. + [Audio Probe Failed (Playable)] FFprobe could not inspect audio in MPV's playable video source. The card was created without sentence audio. + [Audio Streams Not Found] FFmpeg could not find an audio stream in the source used for export. The card was created without sentence audio. + [Audio Streams Not Found (Original)] FFmpeg could not find an audio stream in the original video source used for export. MPV may be playing a different source. The card was created without sentence audio. + [Audio Streams Not Found (Playable Missing)] FFmpeg could not find an audio stream in the original video source, and MPV did not provide a separate playable source to try. The card was created without sentence audio. + [Audio Streams Not Found (Playable Same)] FFmpeg could not find an audio stream in the original video source, and MPV's playable source was the same source. The card was created without sentence audio. + [Audio Streams Not Found (Playable Unsafe)] FFmpeg could not find an audio stream in the original video source, and MPV's separate playable source was not safe to use for export. The card was created without sentence audio. + [Audio Streams Not Found (Playable)] FFmpeg could not find an audio stream in MPV's playable video source. The card was created without sentence audio. + [Audio Streams Not Found (External)] MPV selected external audio, but FFmpeg could not find an audio stream in that source. The card was created without sentence audio. + [Codec Restricted] FFmpeg found audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (Original)] FFmpeg found audio in the original video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (Playable)] FFmpeg found audio in MPV's playable video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (External)] MPV selected external audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Stream Index Unavailable] The audio track selected in the player was not found in this video. The card was created without sentence audio. + [Track Not Audio] The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. + [Stream Protected] The selected audio stream is protected and cannot be exported. The card was created without sentence audio. + [Stream Unreadable] No readable audio stream was found in this video. The card was created without sentence audio. + [Audio Extraction Failed] FFmpeg found an audio stream but could not extract sentence audio from this video. The card was created without it. + [Extraction Output Missing] FFmpeg completed but did not create a usable sentence-audio file. The card was created without it. + [Extraction Output Read Failed] FFmpeg created sentence audio, but Chimahon could not read its temporary file. The card was created without it. + [Stream Mapping Failed] FFmpeg could not map the selected audio track for sentence export. The card was created without it. + [Source Read Failed] FFmpeg could not read the audio data from the video source during sentence export. The card was created without it. + [Source Read Failed (Original)] FFmpeg could not read audio data from the original video source during sentence export. The card was created without it. + [Source Read Failed (Playable)] FFmpeg could not read audio data from MPV's playable video source during sentence export. The card was created without it. + [Seek Failed] FFmpeg could not seek the video source to the subtitle time for sentence export. The card was created without it. + [Output Write Failed] FFmpeg could not write the temporary sentence audio file. The card was created without it. + [Extraction Timed Out] Sentence audio extraction timed out. The card was created without it. + [Audio Storage Failed] AnkiDroid could not store the sentence audio. The card was created without it. Media Learning Sync From 8b694aad703da4c5d83095ca7ee0f85c6de4f823 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Sat, 1 Aug 2026 01:18:38 +0700 Subject: [PATCH 8/9] Update SceneSentenceAudioService.kt --- .../ui/player/scene/SceneSentenceAudioService.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index e04f14b76e..64d003db6a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -302,8 +302,15 @@ internal class FrozenSceneSentenceAudioService internal constructor( diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) } is AudioProbeResult.Success -> { - val failure = if (SceneMediaProbe.audioStreams(probe.output).isNotEmpty()) { + val streams = SceneMediaProbe.audioStreams(probe.output) + val failure = if (streams.isNotEmpty()) { AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED + } else if ( + input.kind == SceneVideoInputKind.REMOTE_HTTP && + input.audioStreamIndex != null && + input.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO + ) { + return AudioInputResolution.Ready(input) } else { AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND } From 4032d014577bf150d9d32626b86c8a0aead1bb17 Mon Sep 17 00:00:00 2001 From: Jolimunium Date: Sat, 1 Aug 2026 01:40:22 +0700 Subject: [PATCH 9/9] Update SentenceAudioDiagnosticLog.kt --- .../ui/player/scene/SentenceAudioDiagnosticLog.kt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt index 3a566ffd57..b38126c5f5 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt @@ -57,14 +57,7 @@ internal object NoOpSentenceAudioDiagnosticLogger : SentenceAudioDiagnosticLogge internal fun createSentenceAudioDiagnosticLogger( storageManager: StorageManager = Injekt.get(), ): SentenceAudioDiagnosticLogger { - return StorageFolderSentenceAudioDiagnosticLogger( - directory = storageManager::getLogsDirectory, - onWriteFailure = { error -> - logcat("SentenceAudioDiagnostic", LogPriority.ERROR) { - "Sentence-audio diagnostic log could not be written to the selected storage folder: ${error.javaClass.simpleName}" - } - }, - ) + return NoOpSentenceAudioDiagnosticLogger } internal class StorageFolderSentenceAudioDiagnosticLogger(