diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt index 830de880c6..56ef258427 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/PlayerViewModel.kt @@ -3281,6 +3281,7 @@ class PlayerViewModel @JvmOverloads internal constructor( ffmpegVideoArgs = emptyList(), seekable = mpv.seekable ?: stableLocalFile, audioStreamIndex = null, + isExternalAudio = true, ) } else { null diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt index 6ce11160af..cd41796a1d 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/controls/PlayerSceneMiningUi.kt @@ -7,6 +7,9 @@ import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.ui.window.DialogProperties import chimahon.anki.AnkiMediaWarning +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback import eu.kanade.tachiyomi.ui.player.scene.PlayerSceneMiningProgress import eu.kanade.tachiyomi.util.system.toast import tachiyomi.i18n.kmk.KMR @@ -47,6 +50,118 @@ internal fun Context.showPlayerAnkiMediaWarnings(warnings: List KMR.strings.anki_scene_fallback_generation AnkiMediaWarning.AnimatedStorageFailed -> KMR.strings.anki_scene_fallback_storage AnkiMediaWarning.StillStorageFailed -> KMR.strings.anki_scene_still_storage_failed + is AnkiMediaWarning.SentenceAudioGenerationFailed -> when (warning.failure) { + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_track_mapping_unavailable + } + AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_source_unavailable + } + AnkiSentenceAudioFailure.TIMING_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_timing_unavailable + } + AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_probe_failed_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_probe_failed_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + null -> KMR.strings.anki_sentence_audio_probe_failed + } + } + AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + when (warning.diagnostic?.playableFallback) { + AnkiSentenceAudioPlayableFallback.MISSING -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_missing + } + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_same + } + AnkiSentenceAudioPlayableFallback.UNAVAILABLE -> { + KMR.strings + .anki_sentence_audio_streams_not_found_original_playable_unavailable + } + null -> KMR.strings.anki_sentence_audio_streams_not_found_original + } + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_streams_not_found_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO -> { + KMR.strings.anki_sentence_audio_streams_not_found_external + } + null -> KMR.strings.anki_sentence_audio_streams_not_found + } + } + AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_codec_restricted_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_codec_restricted_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO -> { + KMR.strings.anki_sentence_audio_codec_restricted_external + } + null -> KMR.strings.anki_sentence_audio_codec_restricted + } + } + AnkiSentenceAudioFailure.AUDIO_STREAM_INDEX_UNAVAILABLE -> { + KMR.strings.anki_sentence_audio_stream_index_unavailable + } + AnkiSentenceAudioFailure.AUDIO_STREAM_NOT_AUDIO -> { + KMR.strings.anki_sentence_audio_stream_not_audio + } + AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED -> { + KMR.strings.anki_sentence_audio_stream_protected + } + AnkiSentenceAudioFailure.AUDIO_STREAM_UNREADABLE -> { + KMR.strings.anki_sentence_audio_stream_unreadable + } + AnkiSentenceAudioFailure.EXTRACTION_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_failed + } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING -> { + KMR.strings.anki_sentence_audio_extraction_output_missing + } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_output_read_failed + } + AnkiSentenceAudioFailure.EXTRACTION_STREAM_MAPPING_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_stream_mapping_failed + } + AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED -> { + when (warning.diagnostic?.inputSource) { + AnkiSentenceAudioInputSource.ORIGINAL_VIDEO -> { + KMR.strings.anki_sentence_audio_extraction_source_read_failed_original + } + AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO -> { + KMR.strings.anki_sentence_audio_extraction_source_read_failed_playable + } + AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + null -> KMR.strings.anki_sentence_audio_extraction_source_read_failed + } + } + AnkiSentenceAudioFailure.EXTRACTION_SEEK_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_seek_failed + } + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_WRITE_FAILED -> { + KMR.strings.anki_sentence_audio_extraction_output_write_failed + } + AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT -> { + KMR.strings.anki_sentence_audio_extraction_timed_out + } + AnkiSentenceAudioFailure.UNKNOWN -> KMR.strings.anki_sentence_audio_generation_failed + } + AnkiMediaWarning.SentenceAudioStorageFailed -> KMR.strings.anki_sentence_audio_storage_failed }, ) } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt index 899b811ff7..443a858b20 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureService.kt @@ -66,7 +66,8 @@ internal class AndroidSceneCaptureService private constructor( outputCleanup.nativeFinished() } when (result) { - SceneCommandResult.Failed -> { + SceneCommandResult.Failed, + is SceneCommandResult.FfmpegFailed -> { return@withContext AnkiScreenshotPreparation.Failed(stillFallback = null) } is SceneCommandResult.Success -> Unit diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt index 3bc5f60bcd..7665eefa53 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/FfmpegKitSceneCommandExecutor.kt @@ -16,9 +16,75 @@ import kotlin.coroutines.resume internal sealed interface SceneCommandResult { data class Success(val output: String = "") : SceneCommandResult + data class FfmpegFailed( + val failure: SceneFfmpegFailure, + val nativeDiagnostics: SceneNativeFailureDiagnostics? = null, + ) : SceneCommandResult + data object Failed : SceneCommandResult } +/** + * Raw native details remain in-memory until the temporary, app-private sentence-audio journal + * redacts them. They must never be passed to UI warnings or regular application logs. + */ +internal data class SceneNativeFailureDiagnostics( + val returnCode: Int?, + val failStackTrace: String?, + val logs: String?, +) + +internal enum class SceneFfmpegFailure { + STREAM_MAPPING, + SOURCE_READ, + SEEK, + OUTPUT_WRITE, + UNKNOWN, +} + +/** + * Converts untrusted native diagnostics into a closed, user-safe category. Raw logs may include + * URLs or request headers, so they are never exposed to the UI; the temporary sentence-audio + * journal is responsible for redacting any diagnostic detail it writes. + */ +internal fun classifySceneFfmpegFailure( + failStackTrace: String?, + logs: String?, +): SceneFfmpegFailure { + val detail = sequenceOf(failStackTrace, logs) + .filterNotNull() + .joinToString(separator = "\n") + .lowercase() + + return when { + "stream map" in detail && "matches no streams" in detail -> { + SceneFfmpegFailure.STREAM_MAPPING + } + "could not seek" in detail || "failed to seek" in detail || "invalid seek" in detail -> { + SceneFfmpegFailure.SEEK + } + "could not write header for output" in detail || + "error opening output" in detail || + "failed to avio_open" in detail || + "error writing trailer" in detail || + "error muxing a packet" in detail -> { + SceneFfmpegFailure.OUTPUT_WRITE + } + "http error" in detail || + "server returned" in detail || + "failed to open segment" in detail || + "error when loading first segment" in detail || + "unable to open resource" in detail || + "connection refused" in detail || + "connection reset by peer" in detail || + "network is unreachable" in detail || + "connection timed out" in detail -> { + SceneFfmpegFailure.SOURCE_READ + } + else -> SceneFfmpegFailure.UNKNOWN + } +} + internal interface SceneCommandExecutor { suspend fun executeFfmpeg( arguments: Array, @@ -79,7 +145,18 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { if (ReturnCode.isSuccess(session.returnCode)) { SceneCommandResult.Success() } else { - SceneCommandResult.Failed + val diagnostics = SceneNativeFailureDiagnostics( + returnCode = session.returnCode?.getValue(), + failStackTrace = session.failStackTrace, + logs = session.getAllLogsAsString(FAILURE_LOG_WAIT_MILLIS), + ) + SceneCommandResult.FfmpegFailed( + classifySceneFfmpegFailure( + failStackTrace = diagnostics.failStackTrace, + logs = diagnostics.logs, + ), + nativeDiagnostics = diagnostics, + ) } }, onNativeFinished = onNativeFinished, @@ -105,7 +182,18 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { if (ReturnCode.isSuccess(session.returnCode)) { SceneCommandResult.Success(session.output.orEmpty()) } else { - SceneCommandResult.Failed + val diagnostics = SceneNativeFailureDiagnostics( + returnCode = session.returnCode?.getValue(), + failStackTrace = session.failStackTrace, + logs = session.getAllLogsAsString(FAILURE_LOG_WAIT_MILLIS), + ) + SceneCommandResult.FfmpegFailed( + classifySceneFfmpegFailure( + failStackTrace = diagnostics.failStackTrace, + logs = diagnostics.logs, + ), + nativeDiagnostics = diagnostics, + ) } }, onNativeFinished = onNativeFinished, @@ -201,6 +289,8 @@ internal class FfmpegKitSceneCommandExecutor : SceneCommandExecutor { } private companion object { + // Temporary diagnostic window: only native failures wait for late FFmpeg/FFprobe lines. + const val FAILURE_LOG_WAIT_MILLIS = 1_000 val DISCARD_LOG_CALLBACK = LogCallback {} val DISCARD_STATISTICS_CALLBACK = StatisticsCallback {} } diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt index 9e322e58ea..49030a7d92 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequest.kt @@ -1,6 +1,8 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPlayableFallback import `is`.xyz.mpv.MPVLib import java.io.Closeable import kotlin.math.abs @@ -91,7 +93,10 @@ internal class OwnedBitmap private constructor( internal class SceneCaptureRequest( val videoInput: SceneVideoInputSpec?, val sentenceAudioInput: SceneVideoInputSpec?, + val sentenceAudioFailure: AnkiSentenceAudioFailure? = null, val resolvedTiming: SceneResolvedTiming?, + val sentenceAudioFallbackInput: SceneVideoInputSpec? = null, + val sentenceAudioFallbackStatus: AnkiSentenceAudioPlayableFallback? = null, private val stillFallback: OwnedBitmap, ) : Closeable { fun fallbackBitmapOrNull(): Bitmap? = stillFallback.bitmapOrNull() @@ -143,6 +148,7 @@ internal data class SceneMpvSnapshot( val selectedAudioId: Int?, val selectedExternalAudioValue: String?, val selectedAudioIsExternal: Boolean, + val audioTrackCount: Int, val seekable: Boolean?, val selectedAudioFfmpegIndex: Int? = null, ) @@ -182,6 +188,7 @@ internal class SceneMpvSnapshotReader( selectedAudioId = selectedAudio.id, selectedExternalAudioValue = selectedAudio.externalValue, selectedAudioIsExternal = selectedAudio.isExternal, + audioTrackCount = selectedAudio.trackCount, seekable = properties.boolean("seekable"), selectedAudioFfmpegIndex = selectedAudio.ffmpegIndex, ) @@ -203,15 +210,18 @@ internal class SceneMpvSnapshotReader( } private fun selectedAudio(): SelectedAudioSnapshot { + val trackCount = properties.int("track-list/count") + ?: return SelectedAudioSnapshot() + val audioTrackCount = (0 until trackCount).count { index -> + properties.string("track-list/$index/type") == "audio" + } val selectedAudioId = properties.string("aid")?.toIntOrNull() ?: properties.int("aid") - ?: return SelectedAudioSnapshot() - val trackCount = properties.int("track-list/count") - ?: return SelectedAudioSnapshot(id = selectedAudioId) + ?: return SelectedAudioSnapshot(trackCount = audioTrackCount) val index = (0 until trackCount).firstOrNull { index -> properties.string("track-list/$index/type") == "audio" && properties.int("track-list/$index/id") == selectedAudioId - } ?: return SelectedAudioSnapshot(id = selectedAudioId) + } ?: return SelectedAudioSnapshot(id = selectedAudioId, trackCount = audioTrackCount) val externalValue = properties.string("track-list/$index/external-filename") ?.takeIf(String::isNotBlank) return SelectedAudioSnapshot( @@ -219,6 +229,7 @@ internal class SceneMpvSnapshotReader( externalValue = externalValue, isExternal = properties.boolean("track-list/$index/external") == true || externalValue != null, + trackCount = audioTrackCount, ffmpegIndex = properties.int("track-list/$index/ff-index") ?.takeIf { it >= 0 }, ) @@ -228,6 +239,7 @@ internal class SceneMpvSnapshotReader( val id: Int? = null, val externalValue: String? = null, val isExternal: Boolean = false, + val trackCount: Int = 0, val ffmpegIndex: Int? = null, ) @@ -303,18 +315,54 @@ internal class SceneCaptureRequestFactory( if (!sameCaptureState(beforeMpv, afterMpv) || beforeVideo != afterVideo) return null val videoInput = SceneVideoInputResolver.resolve(beforeVideo.video) - val sentenceAudioInput = when { + val sentenceAudio = when { beforeVideo.sentenceAudio != null -> { - SceneVideoInputResolver.resolve(beforeVideo.sentenceAudio) + val input = SceneVideoInputResolver.resolve(beforeVideo.sentenceAudio) + SentenceAudioInputResolution( + input = input, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { input == null }, + ) } beforeMpv.selectedAudioId != null && - beforeMpv.selectedAudioFfmpegIndex == null -> null - else -> videoInput + beforeMpv.selectedAudioFfmpegIndex == null && + beforeMpv.audioTrackCount != 1 -> { + SentenceAudioInputResolution( + input = null, + failure = AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ) + } + else -> SentenceAudioInputResolution( + input = videoInput, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE.takeIf { videoInput == null }, + ) + } + val sentenceAudioFallbackResolution = sentenceAudio.input + ?.takeIf { + beforeVideo.sentenceAudio == null && + it.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO + } + ?.let { originalInput -> + SceneVideoInputResolver.resolvePlayableFallback(beforeVideo.video, originalInput) + } + val sentenceAudioFallbackInput = (sentenceAudioFallbackResolution + as? ScenePlayableFallbackResolution.Available) + ?.input + val sentenceAudioFallbackStatus = when (sentenceAudioFallbackResolution) { + ScenePlayableFallbackResolution.Missing -> AnkiSentenceAudioPlayableFallback.MISSING + ScenePlayableFallbackResolution.SameAsOriginal -> { + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL + } + ScenePlayableFallbackResolution.Unavailable -> AnkiSentenceAudioPlayableFallback.UNAVAILABLE + is ScenePlayableFallbackResolution.Available, + null -> null } val request = SceneCaptureRequest( videoInput = videoInput, - sentenceAudioInput = sentenceAudioInput, + sentenceAudioInput = sentenceAudio.input, + sentenceAudioFailure = sentenceAudio.failure, resolvedTiming = resolveTiming(beforeMpv), + sentenceAudioFallbackInput = sentenceAudioFallbackInput, + sentenceAudioFallbackStatus = sentenceAudioFallbackStatus, stillFallback = OwnedBitmap(fallback), ) transferred = true @@ -337,10 +385,16 @@ internal class SceneCaptureRequestFactory( before.selectedAudioId == after.selectedAudioId && before.selectedExternalAudioValue == after.selectedExternalAudioValue && before.selectedAudioIsExternal == after.selectedAudioIsExternal && + before.audioTrackCount == after.audioTrackCount && before.seekable == after.seekable && before.selectedAudioFfmpegIndex == after.selectedAudioFfmpegIndex } + private data class SentenceAudioInputResolution( + val input: SceneVideoInputSpec?, + val failure: AnkiSentenceAudioFailure?, + ) + private fun nullableDoubleEquals(first: Double?, second: Double?): Boolean { if (first == null || second == null) return first == second return closeEnough(first, second, DOUBLE_SNAPSHOT_TOLERANCE_SECONDS) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt index 1466b296f1..9a60fcad7e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneMediaProbe.kt @@ -4,8 +4,7 @@ import java.util.Locale internal object SceneMediaProbe { fun inspect(output: String): Boolean { - val normalized = output.lowercase(Locale.ROOT) - if (PROTECTION_MARKERS.any(normalized::contains)) { + if (output.hasProtectionMarker()) { return false } val values = output.lineSequence() @@ -42,9 +41,78 @@ internal object SceneMediaProbe { return true } - fun inspectAudio(output: String): Boolean { - val normalized = output.lowercase(Locale.ROOT) - return PROTECTION_MARKERS.none(normalized::contains) && "codec_type=audio" in normalized + fun inspectAudio(output: String): Boolean = inspectSelectedAudio(output) is AudioInspection.Readable + + fun inspectSelectedAudio(output: String): AudioInspection { + val values = output.keyValues() + if (values.isEmpty()) return AudioInspection.StreamMissing + if (output.hasProtectionMarker()) return AudioInspection.Protected + return if (values["codec_type"] == "audio") { + AudioInspection.Readable + } else { + AudioInspection.NotAudio + } + } + + fun audioStreams(output: String): List { + return output.streamBlocks().mapNotNull { streamOutput -> + val values = streamOutput.keyValues() + if (values["codec_type"] != "audio") return@mapNotNull null + AudioStream( + index = values["index"]?.toIntOrNull(), + protected = streamOutput.hasProtectionMarker(), + ) + } + } + + sealed interface AudioInspection { + data object Readable : AudioInspection + + data object StreamMissing : AudioInspection + + data object NotAudio : AudioInspection + + data object Protected : AudioInspection + } + + data class AudioStream( + val index: Int?, + val protected: Boolean, + ) + + private fun String.keyValues(): Map { + return lineSequence() + .mapNotNull { line -> + val separator = line.indexOf('=') + if (separator <= 0) { + null + } else { + line.substring(0, separator).trim().lowercase(Locale.ROOT) to + line.substring(separator + 1).trim().lowercase(Locale.ROOT) + } + } + .toMap() + } + + private fun String.streamBlocks(): List { + val blocks = mutableListOf() + var current: StringBuilder? = null + lineSequence().forEach { line -> + when (line.trim()) { + "[STREAM]" -> current = StringBuilder() + "[/STREAM]" -> current?.let { stream -> + blocks += stream.toString() + current = null + } + else -> current?.append(line)?.append('\n') + } + } + return blocks.ifEmpty { listOf(this) } + } + + private fun String.hasProtectionMarker(): Boolean { + val normalized = lowercase(Locale.ROOT) + return PROTECTION_MARKERS.any(normalized::contains) } private val HDR_TRANSFERS = setOf("smpte2084", "arib-std-b67") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt index 7a9a972ce9..64d003db6a 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioService.kt @@ -2,6 +2,11 @@ package eu.kanade.tachiyomi.ui.player.scene import android.content.Context import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioDiagnostic +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback +import chimahon.anki.AnkiSentenceAudioPreparation import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -11,30 +16,76 @@ import java.security.MessageDigest import java.util.UUID internal fun interface SceneSentenceAudioService { - suspend fun prepare(request: SceneCaptureRequest): AnkiMediaSource? + suspend fun prepare(request: SceneCaptureRequest): AnkiSentenceAudioPreparation } -internal class FrozenSceneSentenceAudioService private constructor( +internal class FrozenSceneSentenceAudioService internal constructor( private val cacheDirectory: File, private val inputAcquirer: SceneInputAcquirer, private val commandExecutor: SceneCommandExecutor, + private val timeoutMillis: Long = AUDIO_TIMEOUT_MILLIS, + private val diagnosticLogger: SentenceAudioDiagnosticLogger = NoOpSentenceAudioDiagnosticLogger, ) : SceneSentenceAudioService { constructor(context: Context) : this( cacheDirectory = context.cacheDir, inputAcquirer = AndroidSceneInputAcquirer(context), commandExecutor = FfmpegKitSceneCommandExecutor(), + diagnosticLogger = createSentenceAudioDiagnosticLogger(), ) - override suspend fun prepare(request: SceneCaptureRequest): AnkiMediaSource? { - val videoInput = request.videoInput ?: return null - val input = request.sentenceAudioInput ?: return null - val range = request.resolvedTiming?.audioRange ?: return null - return withTimeoutOrNull(AUDIO_TIMEOUT_MILLIS) { + override suspend fun prepare(request: SceneCaptureRequest): AnkiSentenceAudioPreparation { + val input = request.sentenceAudioInput ?: run { + val failure = request.sentenceAudioFailure ?: AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = failure, + ), + ) + return unavailable(failure) + } + val range = request.resolvedTiming?.audioRange ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = input, + fallback = input.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.TIMING_UNAVAILABLE, + ), + ) + return unavailable(AnkiSentenceAudioFailure.TIMING_UNAVAILABLE) + } + val preparation = withTimeoutOrNull(timeoutMillis) { withContext(Dispatchers.IO) { - if (!isVideoSafe(videoInput) || !isAudioSafe(input)) { - return@withContext null + val resolvedInput = when ( + val resolution = resolveAudioInput( + input = input, + fallbackInput = request.sentenceAudioFallbackInput, + fallbackStatus = request.sentenceAudioFallbackStatus, + ) + ) { + is AudioInputResolution.Ready -> resolution.input + is AudioInputResolution.Unavailable -> { + return@withContext unavailable(resolution.failure, resolution.diagnostic) + } } - val lease = inputAcquirer.acquire(input) ?: return@withContext null + val lease = inputAcquirer.acquire(resolvedInput) + ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ) + return@withContext diagnosticUnavailable( + resolvedInput, + AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ).toPreparation() + } val output = File(cacheDirectory, "chimahon_sentence_audio_${UUID.randomUUID()}.m4a") val inputCleanup = SceneNativeCleanup(lease::close) val outputCleanup = SceneNativeCleanup(output::delete) @@ -42,7 +93,7 @@ internal class FrozenSceneSentenceAudioService private constructor( output.delete() val result = commandExecutor.executeFfmpeg( SceneFfmpegArguments.sentenceAudio( - input = input, + input = resolvedInput, acquiredInputValue = lease.ffmpegValue, range = range, outputFile = output.absolutePath, @@ -52,55 +103,410 @@ internal class FrozenSceneSentenceAudioService private constructor( inputCleanup.nativeFinished() outputCleanup.nativeFinished() } - if (result !is SceneCommandResult.Success || !output.isFile || output.length() == 0L) { - return@withContext null + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = result.toSentenceAudioFailure() + .takeIf { result !is SceneCommandResult.Success }, + result = result, + ), + ) + if (result !is SceneCommandResult.Success) { + return@withContext unavailable( + failure = result.toSentenceAudioFailure(), + diagnostic = result.toSentenceAudioDiagnostic(resolvedInput), + ) + } + if (!output.isFile || output.length() == 0L) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.OUTPUT_VALIDATION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + result = result, + ), + ) + return@withContext unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + diagnostic = resolvedInput.toDiagnostic(), + ) } - val bytes = output.readBytes() - AnkiMediaSource.Bytes( - data = bytes, - preferredBaseName = "chimahon_sentence_${bytes.sha256()}", - extension = "m4a", + val bytes = try { + output.readBytes() + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.OUTPUT_READ, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED, + result = result, + exceptionType = e.javaClass.name, + ), + ) + return@withContext unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_READ_FAILED, + diagnostic = resolvedInput.toDiagnostic(), + ) + } + AnkiSentenceAudioPreparation.Ready( + AnkiMediaSource.Bytes( + data = bytes, + preferredBaseName = "chimahon_sentence_${bytes.sha256()}", + extension = "m4a", + ), ) } catch (e: CancellationException) { throw e - } catch (_: Exception) { - return@withContext null + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = resolvedInput, + fallback = resolvedInput.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_FAILED, + exceptionType = e.javaClass.name, + ), + ) + unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_FAILED, + diagnostic = resolvedInput.toDiagnostic(), + ) } finally { inputCleanup.release() outputCleanup.release() } } } + return preparation ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = input, + fallback = input.toDiagnosticFallback(), + failure = AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT, + ), + ) + unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT) + } } - private suspend fun isVideoSafe(input: SceneVideoInputSpec): Boolean { - val lease = inputAcquirer.acquire(input) ?: return false - val cleanup = SceneNativeCleanup(lease::close) - return try { - val probe = commandExecutor.executeFfprobe( - SceneFfmpegArguments.videoProbe(input, lease.ffmpegValue, lease.tlsCaFile), - cleanup::nativeFinished, + private suspend fun resolveAudioInput( + input: SceneVideoInputSpec, + fallbackInput: SceneVideoInputSpec?, + fallbackStatus: AnkiSentenceAudioPlayableFallback?, + ): AudioInputResolution { + val resolution = resolveSingleAudioInput(input) + if ( + resolution !is AudioInputResolution.Unavailable || + !resolution.failure.isPlayableFallbackRetryable() || + input.origin != SceneVideoInputOrigin.ORIGINAL_VIDEO + ) { + return resolution + } + val playableFallback = fallbackInput?.takeIf { + it.origin == SceneVideoInputOrigin.PLAYABLE_VIDEO && it.value != input.value + } + if (playableFallback == null) { + val effectiveFallbackStatus = fallbackStatus ?: when { + fallbackInput?.value == input.value -> AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL + else -> null + } + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.FALLBACK_DECISION, + input = input, + fallback = effectiveFallbackStatus.toDiagnosticFallback(), + failure = resolution.failure, + ), ) - probe is SceneCommandResult.Success && SceneMediaProbe.inspect(probe.output) - } finally { - cleanup.release() + return resolution.withFallbackStatus(input, effectiveFallbackStatus) + } + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.FALLBACK_DECISION, + input = input, + fallback = SentenceAudioDiagnosticFallback.ATTEMPTED, + failure = resolution.failure, + ), + ) + return resolveSingleAudioInput(playableFallback) + } + + private suspend fun resolveSingleAudioInput(input: SceneVideoInputSpec): AudioInputResolution { + return when (val probe = executeAudioProbe(input, AudioProbeMode.SELECTED_RESTRICTED)) { + AudioProbeResult.SourceUnavailable -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + when (val inspection = SceneMediaProbe.inspectSelectedAudio(probe.output)) { + SceneMediaProbe.AudioInspection.Readable -> AudioInputResolution.Ready(input) + SceneMediaProbe.AudioInspection.Protected -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED) + } + SceneMediaProbe.AudioInspection.StreamMissing, + SceneMediaProbe.AudioInspection.NotAudio -> { + resolveFallbackAudioInput(input) + } + } + } + } + } + + private suspend fun resolveFallbackAudioInput( + input: SceneVideoInputSpec, + ): AudioInputResolution { + return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_RESTRICTED)) { + AudioProbeResult.SourceUnavailable -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + val streams = SceneMediaProbe.audioStreams(probe.output) + val onlyStream = streams.singleOrNull() + when { + onlyStream?.index != null && !onlyStream.protected -> { + AudioInputResolution.Ready(input.copy(audioStreamIndex = onlyStream.index)) + } + streams.size > 1 -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE) + } + onlyStream?.protected == true -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED) + } + onlyStream != null -> { + AudioInputResolution.Unavailable(AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE) + } + else -> { + resolveEmptyAudioInventory(input) + } + } + } + } + } + + private suspend fun resolveEmptyAudioInventory(input: SceneVideoInputSpec): AudioInputResolution { + return when (val probe = executeAudioProbe(input, AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY)) { + AudioProbeResult.SourceUnavailable -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE) + } + AudioProbeResult.ExecutionFailed -> { + diagnosticUnavailable(input, AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED) + } + is AudioProbeResult.Success -> { + val streams = SceneMediaProbe.audioStreams(probe.output) + val failure = if (streams.isNotEmpty()) { + AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED + } else if ( + input.kind == SceneVideoInputKind.REMOTE_HTTP && + input.audioStreamIndex != null && + input.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO + ) { + return AudioInputResolution.Ready(input) + } else { + AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND + } + diagnosticUnavailable(input, failure) + } } } - private suspend fun isAudioSafe(input: SceneVideoInputSpec): Boolean { - val lease = inputAcquirer.acquire(input) ?: return false + private suspend fun executeAudioProbe( + input: SceneVideoInputSpec, + mode: AudioProbeMode, + ): AudioProbeResult { + val stage = mode.toDiagnosticStage() + val fallback = input.toDiagnosticFallback() + val lease = inputAcquirer.acquire(input) ?: run { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ) + return AudioProbeResult.SourceUnavailable + } val cleanup = SceneNativeCleanup(lease::close) return try { val probe = commandExecutor.executeFfprobe( - SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile), + when (mode) { + AudioProbeMode.SELECTED_RESTRICTED -> { + SceneFfmpegArguments.audioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } + AudioProbeMode.ALL_RESTRICTED -> { + SceneFfmpegArguments.allAudioProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } + AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY -> { + SceneFfmpegArguments.audioDiscoveryProbe(input, lease.ffmpegValue, lease.tlsCaFile) + } + }, cleanup::nativeFinished, ) - probe is SceneCommandResult.Success && SceneMediaProbe.inspectAudio(probe.output) + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED + .takeIf { probe !is SceneCommandResult.Success }, + result = probe, + ), + ) + when (probe) { + SceneCommandResult.Failed, + is SceneCommandResult.FfmpegFailed -> AudioProbeResult.ExecutionFailed + is SceneCommandResult.Success -> AudioProbeResult.Success(probe.output) + } + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + diagnosticLogger.record( + SentenceAudioDiagnosticEvent( + stage = stage, + input = input, + fallback = fallback, + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + exceptionType = e.javaClass.name, + ), + ) + AudioProbeResult.ExecutionFailed } finally { cleanup.release() } } + private sealed interface AudioInputResolution { + data class Ready( + val input: SceneVideoInputSpec, + ) : AudioInputResolution + + data class Unavailable( + val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, + ) : AudioInputResolution + } + + private enum class AudioProbeMode { + SELECTED_RESTRICTED, + ALL_RESTRICTED, + ALL_UNRESTRICTED_DISCOVERY, + } + + private sealed interface AudioProbeResult { + data object SourceUnavailable : AudioProbeResult + + data object ExecutionFailed : AudioProbeResult + + data class Success( + val output: String, + ) : AudioProbeResult + } + + private fun diagnosticUnavailable( + input: SceneVideoInputSpec, + failure: AnkiSentenceAudioFailure, + ) = AudioInputResolution.Unavailable(failure, input.toDiagnostic()) + + private fun AudioInputResolution.Unavailable.withFallbackStatus( + input: SceneVideoInputSpec, + fallbackStatus: AnkiSentenceAudioPlayableFallback?, + ): AudioInputResolution.Unavailable { + return fallbackStatus?.let { copy(diagnostic = input.toDiagnostic(it)) } ?: this + } + + private fun AudioInputResolution.Unavailable.toPreparation(): AnkiSentenceAudioPreparation { + return unavailable(failure, diagnostic) + } + + private fun unavailable( + failure: AnkiSentenceAudioFailure, + diagnostic: AnkiSentenceAudioDiagnostic? = null, + ) = AnkiSentenceAudioPreparation.Unavailable(failure, diagnostic) + + private fun SceneVideoInputSpec.toDiagnostic( + playableFallback: AnkiSentenceAudioPlayableFallback? = null, + ): AnkiSentenceAudioDiagnostic { + return AnkiSentenceAudioDiagnostic( + inputSource = when (origin) { + SceneVideoInputOrigin.ORIGINAL_VIDEO -> AnkiSentenceAudioInputSource.ORIGINAL_VIDEO + SceneVideoInputOrigin.PLAYABLE_VIDEO -> AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO + SceneVideoInputOrigin.EXTERNAL_AUDIO -> AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO + }, + playableFallback = playableFallback, + ) + } + + private fun AudioProbeMode.toDiagnosticStage(): SentenceAudioDiagnosticStage { + return when (this) { + AudioProbeMode.SELECTED_RESTRICTED -> SentenceAudioDiagnosticStage.SELECTED_AUDIO_PROBE + AudioProbeMode.ALL_RESTRICTED -> SentenceAudioDiagnosticStage.ALL_AUDIO_PROBE + AudioProbeMode.ALL_UNRESTRICTED_DISCOVERY -> SentenceAudioDiagnosticStage.AUDIO_DISCOVERY_PROBE + } + } + + private fun SceneVideoInputSpec.toDiagnosticFallback(): SentenceAudioDiagnosticFallback { + return if (origin == SceneVideoInputOrigin.PLAYABLE_VIDEO) { + SentenceAudioDiagnosticFallback.ATTEMPTED + } else { + SentenceAudioDiagnosticFallback.NOT_APPLICABLE + } + } + + private fun AnkiSentenceAudioPlayableFallback?.toDiagnosticFallback(): SentenceAudioDiagnosticFallback { + return when (this) { + AnkiSentenceAudioPlayableFallback.MISSING -> SentenceAudioDiagnosticFallback.MISSING + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL -> { + SentenceAudioDiagnosticFallback.SAME_AS_ORIGINAL + } + AnkiSentenceAudioPlayableFallback.UNAVAILABLE -> SentenceAudioDiagnosticFallback.UNAVAILABLE + null -> SentenceAudioDiagnosticFallback.NOT_APPLICABLE + } + } + + private fun AnkiSentenceAudioFailure.isPlayableFallbackRetryable(): Boolean { + return this == AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE || + this == AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED || + this == AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND + } + + private fun SceneCommandResult.toSentenceAudioFailure(): AnkiSentenceAudioFailure { + return when (this) { + is SceneCommandResult.Success, + SceneCommandResult.Failed -> AnkiSentenceAudioFailure.EXTRACTION_FAILED + is SceneCommandResult.FfmpegFailed -> when (failure) { + SceneFfmpegFailure.STREAM_MAPPING -> { + AnkiSentenceAudioFailure.EXTRACTION_STREAM_MAPPING_FAILED + } + SceneFfmpegFailure.SOURCE_READ -> { + AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED + } + SceneFfmpegFailure.SEEK -> AnkiSentenceAudioFailure.EXTRACTION_SEEK_FAILED + SceneFfmpegFailure.OUTPUT_WRITE -> { + AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_WRITE_FAILED + } + SceneFfmpegFailure.UNKNOWN -> AnkiSentenceAudioFailure.EXTRACTION_FAILED + } + } + } + + private fun SceneCommandResult.toSentenceAudioDiagnostic( + input: SceneVideoInputSpec, + ): AnkiSentenceAudioDiagnostic? { + return (this as? SceneCommandResult.FfmpegFailed) + ?.takeUnless { it.failure == SceneFfmpegFailure.UNKNOWN } + ?.let { input.toDiagnostic() } + } + private fun ByteArray.sha256(): String { return MessageDigest.getInstance("SHA-256") .digest(this) diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt index b6f19d112a..6e0fa8a4ce 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInput.kt @@ -11,12 +11,19 @@ internal enum class SceneVideoInputKind { REMOTE_HTTP, } +internal enum class SceneVideoInputOrigin { + ORIGINAL_VIDEO, + PLAYABLE_VIDEO, + EXTERNAL_AUDIO, +} + internal data class SceneVideoInputSpec( val value: String, val kind: SceneVideoInputKind, val headers: List>, val videoStreamIndex: Int? = null, val audioStreamIndex: Int? = null, + val origin: SceneVideoInputOrigin = SceneVideoInputOrigin.ORIGINAL_VIDEO, ) internal data class SceneVideoInputSnapshot( @@ -28,8 +35,22 @@ internal data class SceneVideoInputSnapshot( val seekable: Boolean?, val videoStreamIndex: Int? = null, val audioStreamIndex: Int? = null, + val isExternalAudio: Boolean = false, ) +internal sealed interface ScenePlayableFallbackResolution { + data class Available( + val input: SceneVideoInputSpec, + ) : ScenePlayableFallbackResolution + + data object Missing : ScenePlayableFallbackResolution + + data object SameAsOriginal : ScenePlayableFallbackResolution + + /** The player exposed a value, but it cannot safely be used for export. */ + data object Unavailable : ScenePlayableFallbackResolution +} + internal object SceneVideoInputResolver { fun resolve(snapshot: SceneVideoInputSnapshot): SceneVideoInputSpec? { if (snapshot.originalVideoValue.isBlank() && snapshot.playableValue.isNullOrBlank()) { @@ -43,12 +64,18 @@ internal object SceneVideoInputResolver { val original = snapshot.originalVideoValue.takeIf(String::isNotBlank) if (original != null && isTransient(original)) return null - val normalized = original?.let(::normalizeInput) + val normalizedOriginal = original?.let(::normalizeInput) + val normalized = normalizedOriginal ?: snapshot.playableValue?.takeIf(String::isNotBlank)?.let { playable -> if (isTransient(playable)) return null normalizeInput(playable) } ?: return null + val origin = when { + snapshot.isExternalAudio -> SceneVideoInputOrigin.EXTERNAL_AUDIO + normalizedOriginal != null -> SceneVideoInputOrigin.ORIGINAL_VIDEO + else -> SceneVideoInputOrigin.PLAYABLE_VIDEO + } val headers = when (normalized.second) { SceneVideoInputKind.REMOTE_HTTP -> validateRemoteInput(normalized.first, snapshot.headers) @@ -64,9 +91,34 @@ internal object SceneVideoInputResolver { headers = headers, videoStreamIndex = snapshot.videoStreamIndex?.takeIf { it >= 0 }, audioStreamIndex = snapshot.audioStreamIndex?.takeIf { it >= 0 }, + origin = origin, ) } + fun resolvePlayable(snapshot: SceneVideoInputSnapshot): SceneVideoInputSpec? { + if (snapshot.isExternalAudio) return null + return resolve(snapshot.copy(originalVideoValue = "")) + } + + fun resolvePlayableFallback( + snapshot: SceneVideoInputSnapshot, + original: SceneVideoInputSpec, + ): ScenePlayableFallbackResolution { + if (snapshot.playableValue.isNullOrBlank()) { + return ScenePlayableFallbackResolution.Missing + } + val playable = resolvePlayable(snapshot) ?: return ScenePlayableFallbackResolution.Unavailable + return if ( + playable.value == original.value && + playable.kind == original.kind && + playable.headers == original.headers + ) { + ScenePlayableFallbackResolution.SameAsOriginal + } else { + ScenePlayableFallbackResolution.Available(playable) + } + } + private fun validateRemoteInput( value: String, headers: List>, @@ -248,6 +300,44 @@ internal object SceneFfmpegArguments { }.toTypedArray() } + fun allAudioProbe( + input: SceneVideoInputSpec, + acquiredInputValue: String, + tlsCaFile: String? = null, + ): Array { + return buildList { + addInputOptions(input, tlsCaFile) + add("-v") + add("error") + add("-select_streams") + add("a") + add("-show_entries") + add("stream=index,codec_type,codec_name:stream_side_data") + add("-of") + add("default=noprint_wrappers=0") + add(acquiredInputValue) + }.toTypedArray() + } + + fun audioDiscoveryProbe( + input: SceneVideoInputSpec, + acquiredInputValue: String, + tlsCaFile: String? = null, + ): Array { + return buildList { + addInputOptions(input, tlsCaFile, restrictDecoders = false) + add("-v") + add("error") + add("-select_streams") + add("a") + add("-show_entries") + add("stream=index,codec_type,codec_name:stream_side_data") + add("-of") + add("default=noprint_wrappers=0") + add(acquiredInputValue) + }.toTypedArray() + } + fun sentenceAudio( input: SceneVideoInputSpec, acquiredInputValue: String, @@ -280,15 +370,14 @@ internal object SceneFfmpegArguments { private fun MutableList.addInputOptions( input: SceneVideoInputSpec, tlsCaFile: String?, + restrictDecoders: Boolean = true, ) { - add("-codec_whitelist") - add(ALLOWED_INPUT_DECODERS) + if (restrictDecoders) { + add("-codec_whitelist") + add(ALLOWED_INPUT_DECODERS) + } if (input.kind == SceneVideoInputKind.REMOTE_HTTP) { require(!tlsCaFile.isNullOrBlank()) { "Remote scene input requires a CA bundle" } - add("-tls_verify") - add("1") - add("-ca_file") - add(tlsCaFile) add("-protocol_whitelist") add(REMOTE_PROTOCOLS) add("-rw_timeout") diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt new file mode 100644 index 0000000000..b38126c5f5 --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLog.kt @@ -0,0 +1,164 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import chimahon.anki.AnkiSentenceAudioFailure +import com.hippo.unifile.UniFile +import logcat.LogPriority +import logcat.logcat +import tachiyomi.domain.storage.service.StorageManager +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.io.IOException +import java.nio.charset.StandardCharsets +import kotlin.math.max + +/** + * Temporary diagnostic journal for sentence-audio failures. + * + * It is written to the existing `logs` directory below the storage folder selected by the user. + * It intentionally records only source metadata and redacted native diagnostics. It must be + * removed after the current stream-export investigation is complete. + */ +internal enum class SentenceAudioDiagnosticStage { + REQUEST_VALIDATION, + FALLBACK_DECISION, + SELECTED_AUDIO_PROBE, + ALL_AUDIO_PROBE, + AUDIO_DISCOVERY_PROBE, + AUDIO_EXTRACTION, + OUTPUT_VALIDATION, + OUTPUT_READ, +} + +internal enum class SentenceAudioDiagnosticFallback { + NOT_APPLICABLE, + MISSING, + SAME_AS_ORIGINAL, + UNAVAILABLE, + ATTEMPTED, +} + +internal data class SentenceAudioDiagnosticEvent( + val stage: SentenceAudioDiagnosticStage, + val input: SceneVideoInputSpec?, + val fallback: SentenceAudioDiagnosticFallback, + val failure: AnkiSentenceAudioFailure? = null, + val result: SceneCommandResult? = null, + val exceptionType: String? = null, +) + +internal fun interface SentenceAudioDiagnosticLogger { + fun record(event: SentenceAudioDiagnosticEvent) +} + +internal object NoOpSentenceAudioDiagnosticLogger : SentenceAudioDiagnosticLogger { + override fun record(event: SentenceAudioDiagnosticEvent) = Unit +} + +internal fun createSentenceAudioDiagnosticLogger( + storageManager: StorageManager = Injekt.get(), +): SentenceAudioDiagnosticLogger { + return NoOpSentenceAudioDiagnosticLogger +} + +internal class StorageFolderSentenceAudioDiagnosticLogger( + private val directory: () -> UniFile?, + private val maxBytes: Int = MAX_LOG_BYTES, + private val onWriteFailure: (Throwable) -> Unit = {}, +) : SentenceAudioDiagnosticLogger { + init { + require(maxBytes > 0) { "maxBytes must be positive" } + } + + override fun record(event: SentenceAudioDiagnosticEvent) { + runCatching { + val entry = SentenceAudioDiagnosticJournal.render(event).toByteArray(StandardCharsets.UTF_8) + synchronized(lock) { + val folder = directory() + ?: throw IOException("The selected storage folder is unavailable") + val file = folder.findFile(FILE_NAME) + ?: folder.createFile(FILE_NAME) + ?: throw IOException("Could not create the sentence-audio diagnostic log") + val existing = file.openInputStream().use { it.readBytes() } + file.openOutputStream().use { output -> + output.write(SentenceAudioDiagnosticJournal.retain(existing, entry, maxBytes)) + } + } + }.onFailure(onWriteFailure) + } + + private companion object { + val lock = Any() + } +} + +internal object SentenceAudioDiagnosticJournal { + fun retain(existing: ByteArray, entry: ByteArray, maxBytes: Int): ByteArray { + val combined = existing + entry + return if (combined.size <= maxBytes) { + combined + } else { + combined.copyOfRange(max(0, combined.size - maxBytes), combined.size) + } + } + + fun render(event: SentenceAudioDiagnosticEvent): String { + return buildString { + appendLine("recorded_at_utc=${System.currentTimeMillis()}") + appendLine("stage=${event.stage.name}") + event.input?.let { input -> + appendLine("input_source=${input.origin.name}") + appendLine("input_kind=${input.kind.name}") + appendLine("audio_stream_index=${input.audioStreamIndex ?: "none"}") + } + appendLine("fallback=${event.fallback.name}") + event.failure?.let { appendLine("failure=${it.name}") } + event.result?.let { appendResult(it) } + event.exceptionType?.let { appendLine("exception_type=$it") } + appendLine("---") + } + } + + private fun StringBuilder.appendResult(result: SceneCommandResult) { + when (result) { + is SceneCommandResult.Success -> appendLine("command_result=SUCCESS") + SceneCommandResult.Failed -> appendLine("command_result=EXECUTION_FAILED") + is SceneCommandResult.FfmpegFailed -> { + appendLine("command_result=NATIVE_FAILED") + appendLine("native_failure=${result.failure.name}") + result.nativeDiagnostics?.let { details -> + details.returnCode?.let { appendLine("return_code=$it") } + val detail = sequenceOf(details.failStackTrace, details.logs) + .filterNotNull() + .joinToString(separator = "\n") + .redactNativeDiagnostics() + .takeLast(MAX_NATIVE_DIAGNOSTIC_CHARS) + .trim() + if (detail.isNotEmpty()) { + appendLine("native_diagnostic_begin") + appendLine(detail) + appendLine("native_diagnostic_end") + } + } + } + } + } + + private fun String.redactNativeDiagnostics(): String { + return replace(URL_PATTERN, "") + .replace(SENSITIVE_QUERY_PATTERN, "${'$'}1=") + .replace(SENSITIVE_HEADER_PATTERN, "${'$'}1") + .replace(LOCAL_PATH_PATTERN, "") + } +} + +private const val FILE_NAME = "chimahon_sentence_audio_debug.log" +private const val MAX_LOG_BYTES = 64 * 1024 +private const val MAX_NATIVE_DIAGNOSTIC_CHARS = 32 * 1024 +private val URL_PATTERN = Regex("""(?i)\b(?:https?|file)://[^\s\"'<>]+""") +private val SENSITIVE_QUERY_PATTERN = Regex( + """(?i)\b(access_token|api_key|auth|authorization|credential|credentials|key|policy|signature|signed|sig|token|x-amz-[^=\s]+|x-goog-[^=\s]+)=([^&\s]+)""", +) +private val SENSITIVE_HEADER_PATTERN = Regex( + """(?im)^((?:authorization|cookie|referer|origin|user-agent|accept(?:-[a-z-]+)?|cache-control|pragma|proxy-authorization|x-[a-z0-9-]+)\s*:\s*).*$""", +) +private val LOCAL_PATH_PATTERN = Regex("""(?i)(?:[a-z]:\\|/(?:data|storage|sdcard|mnt|cache|files)/)[^\s\"'<>]+""") diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt index 8e0ec461ec..e5b8b15b5f 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/AndroidSceneCaptureServiceTest.kt @@ -114,10 +114,6 @@ class AndroidSceneCaptureServiceTest { return arrayOf( "-codec_whitelist", SceneFfmpegArguments.ALLOWED_INPUT_DECODERS, - "-tls_verify", - "1", - "-ca_file", - "/files/cacert.pem", "-protocol_whitelist", "http,https,tls,tcp,crypto", "-rw_timeout", diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt index 459501d71a..9887152e36 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneCaptureRequestTest.kt @@ -1,10 +1,13 @@ package eu.kanade.tachiyomi.ui.player.scene import android.graphics.Bitmap +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioPlayableFallback import io.mockk.every import io.mockk.mockk import io.mockk.verify import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test @@ -96,9 +99,90 @@ class SceneCaptureRequestTest { assertNotNull(request) assertNull(request!!.sentenceAudioInput) + assertEquals( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + request.sentenceAudioFailure, + ) + request.close() + } + + @Test + fun `single selected audio without a frozen ffmpeg index uses the only audio stream`() = runTest { + val bitmap = mockBitmap() + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader( + SequencePropertyReader( + listOf( + snapshotWithSingleAudioWithoutFfmpegIndex(), + snapshotWithSingleAudioWithoutFfmpegIndex(), + ), + ), + ), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { inputSnapshot() }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + assertNotNull(request) + assertNotNull(request!!.sentenceAudioInput) request.close() } + @Test + fun `original sentence audio input keeps a different MPV playable source as fallback`() = runTest { + val bitmap = mockBitmap() + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader(SequencePropertyReader(listOf(snapshot(), snapshot()))), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { + inputSnapshot( + originalVideoValue = "https://media.example/original.m3u8", + playableValue = "https://media.example/playable.m3u8", + ) + }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + assertNotNull(request) + val capturedRequest = requireNotNull(request) + val originalInput = requireNotNull(capturedRequest.sentenceAudioInput) + val playableInput = requireNotNull(capturedRequest.sentenceAudioFallbackInput) + assertEquals("https://media.example/original.m3u8", originalInput.value) + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, originalInput.origin) + assertEquals("https://media.example/playable.m3u8", playableInput.value) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playableInput.origin) + capturedRequest.close() + } + + @Test + fun `original sentence audio input records when MPV playable source is the same`() = runTest { + val bitmap = mockBitmap() + val source = "https://media.example/original.m3u8" + val factory = SceneCaptureRequestFactory( + SceneMpvSnapshotReader(SequencePropertyReader(listOf(snapshot(source), snapshot(source)))), + ) + + val request = factory.captureSubtitle( + videoSnapshot = { inputSnapshot(originalVideoValue = source, playableValue = source) }, + parsedSubtitleCandidates = emptyList(), + captureFallback = { bitmap }, + ) + + val capturedRequest = requireNotNull(request) + assertNull(capturedRequest.sentenceAudioFallbackInput) + assertEquals( + AnkiSentenceAudioPlayableFallback.SAME_AS_ORIGINAL, + capturedRequest.sentenceAudioFallbackStatus, + ) + capturedRequest.close() + } + private fun mockBitmap(): Bitmap { return mockk(relaxed = true).also { every { it.isRecycled } returns false @@ -106,11 +190,13 @@ class SceneCaptureRequestTest { } private fun inputSnapshot( + originalVideoValue: String = "https://media.example/video.mp4", + playableValue: String = originalVideoValue, headers: List> = emptyList(), ): SceneCaptureInputSnapshot { val video = SceneVideoInputSnapshot( - originalVideoValue = "https://media.example/video.mp4", - playableValue = "https://media.example/video.mp4", + originalVideoValue = originalVideoValue, + playableValue = playableValue, headers = headers, ffmpegStreamArgs = emptyList(), ffmpegVideoArgs = emptyList(), @@ -150,6 +236,37 @@ class SceneCaptureRequestTest { ) } + private fun snapshotWithSingleAudioWithoutFfmpegIndex(): SnapshotValues { + return SnapshotValues( + doubles = mapOf( + "time-pos" to 5.0, + "duration" to 60.0, + "sub-start/full" to 4.0, + "sub-end/full" to 6.0, + "sub-speed" to 1.0, + "sub-delay" to 0.0, + ), + strings = mapOf( + "path" to "https://media.example/video.mp4", + "aid" to "2", + "track-list/0/type" to "video", + "track-list/1/type" to "audio", + ), + booleans = mapOf( + "track-list/0/selected" to true, + "track-list/0/external" to false, + "track-list/1/external" to false, + "seekable" to true, + ), + ints = mapOf( + "track-list/count" to 2, + "track-list/0/id" to 1, + "track-list/0/ff-index" to 0, + "track-list/1/id" to 2, + ), + ) + } + private data class SnapshotValues( val doubles: Map, val strings: Map, diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt new file mode 100644 index 0000000000..8aa235de98 --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneFfmpegFailureClassifierTest.kt @@ -0,0 +1,61 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SceneFfmpegFailureClassifierTest { + @Test + fun `classifies an HTTP failure while reading the video source`() { + assertEquals( + SceneFfmpegFailure.SOURCE_READ, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "[https @ 0x1] HTTP error 403 Forbidden", + ), + ) + } + + @Test + fun `classifies a missing mapped stream`() { + assertEquals( + SceneFfmpegFailure.STREAM_MAPPING, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Stream map '0:4' matches no streams.", + ), + ) + } + + @Test + fun `classifies an input seek failure`() { + assertEquals( + SceneFfmpegFailure.SEEK, + classifySceneFfmpegFailure( + failStackTrace = "Could not seek to position 12.500", + logs = null, + ), + ) + } + + @Test + fun `classifies a temporary output write failure`() { + assertEquals( + SceneFfmpegFailure.OUTPUT_WRITE, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Could not write header for output file #0: Invalid argument", + ), + ) + } + + @Test + fun `keeps unrecognised FFmpeg output generic`() { + assertEquals( + SceneFfmpegFailure.UNKNOWN, + classifySceneFfmpegFailure( + failStackTrace = null, + logs = "Conversion failed.", + ), + ) + } +} diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt new file mode 100644 index 0000000000..47dda9acef --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneSentenceAudioServiceTest.kt @@ -0,0 +1,738 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import android.graphics.Bitmap +import chimahon.anki.AnkiMediaSource +import chimahon.anki.AnkiSentenceAudioDiagnostic +import chimahon.anki.AnkiSentenceAudioFailure +import chimahon.anki.AnkiSentenceAudioInputSource +import chimahon.anki.AnkiSentenceAudioPlayableFallback +import chimahon.anki.AnkiSentenceAudioPreparation +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import java.io.File + +class SceneSentenceAudioServiceTest { + @TempDir + lateinit var tempDirectory: File + + @Test + fun `exports safe audio when the video stream is HDR`() = runBlocking { + val executor = RecordingExecutor() + val request = request() + + try { + val result = service(executor).prepare(request) + val audio = (result as? AnkiSentenceAudioPreparation.Ready)?.source as? AnkiMediaSource.Bytes + + assertNotNull(audio) + assertEquals("m4a", audio!!.extension) + assertEquals(1, executor.ffmpegCalls) + assertEquals(listOf("a:0"), executor.ffprobeSelectors) + } finally { + request.close() + } + } + + @Test + fun `reports unavailable source when no sentence audio input was captured`() = runBlocking { + val request = request(sentenceAudioInput = null) + + try { + val result = service(RecordingExecutor()).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `records unavailable source when no sentence audio input was captured`() = runBlocking { + val diagnostics = RecordingDiagnosticLogger() + val request = request(sentenceAudioInput = null) + + try { + service(RecordingExecutor(), diagnosticLogger = diagnostics).prepare(request) + + assertEquals( + listOf( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = AnkiSentenceAudioFailure.SOURCE_UNAVAILABLE, + ), + ), + diagnostics.events, + ) + } finally { + request.close() + } + } + + @Test + fun `reports no audio in the original export input after unrestricted discovery`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `exports audio from MPV playable input when original source has no audio`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("a:0", "a", "a", "1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `exports audio from MPV playable input when original FFprobe fails`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Failed, + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("a:0", "1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `exports audio from MPV playable input when original source cannot be acquired`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ) + + try { + val result = service( + executor = executor, + inputAcquirer = SceneInputAcquirer { input -> + if (input.origin == SceneVideoInputOrigin.ORIGINAL_VIDEO) { + null + } else { + leaseFor(input) + } + }, + ).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("1"), executor.ffprobeSelectors) + assertEquals(listOf(playable.value), executor.ffmpegInputs) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `reports MPV playable input when fallback also has no audio`() = runBlocking { + val original = audioInput(value = "https://media.example/original.m3u8") + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = original, + sentenceAudioFallbackInput = playable, + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=video"), + "a" to SceneCommandResult.Success(), + "1" to SceneCommandResult.Success("codec_type=video"), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports original probe failure and missing playable fallback`() = runBlocking { + val request = request( + sentenceAudioFallbackStatus = AnkiSentenceAudioPlayableFallback.MISSING, + ) + + try { + val result = service( + RecordingExecutor(audioProbeResults = mapOf("a:0" to SceneCommandResult.Failed)), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + playableFallback = AnkiSentenceAudioPlayableFallback.MISSING, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `does not retry external audio after its probe fails`() = runBlocking { + val external = audioInput( + audioStreamIndex = 0, + origin = SceneVideoInputOrigin.EXTERNAL_AUDIO, + ) + val playable = audioInput( + value = "https://media.example/playable.m3u8", + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + val request = request( + sentenceAudioInput = external, + sentenceAudioFallbackInput = playable, + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Failed, + "1" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `normalizes a wrong MPV stream index when exactly one audio stream is readable`() = runBlocking { + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + codec_name=aac + [/STREAM] + """.trimIndent(), + ), + ), + ) + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service(executor).prepare(request) + + assertNotNull((result as? AnkiSentenceAudioPreparation.Ready)?.source) + assertEquals(listOf("0", "a"), executor.ffprobeSelectors) + assertEquals(listOf("0:1"), executor.ffmpegAudioMaps) + } finally { + request.close() + } + } + + @Test + fun `reports codec restriction when discovery finds audio in MPV playable input`() = runBlocking { + val request = request( + audioInput( + audioStreamIndex = 4, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ), + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "4" to SceneCommandResult.Success(), + "a" to SceneCommandResult.Success(), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + codec_name=unsupported_audio + [/STREAM] + """.trimIndent(), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_CODEC_RESTRICTED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports no audio in MPV selected external audio input`() = runBlocking { + val request = request( + audioInput( + audioStreamIndex = 0, + origin = SceneVideoInputOrigin.EXTERNAL_AUDIO, + ), + ) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success(), + ), + unrestrictedAudioProbeResult = SceneCommandResult.Success(), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_STREAMS_NOT_FOUND, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_EXTERNAL_AUDIO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports protected selected audio stream without remapping it`() = runBlocking { + val executor = RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success( + "index=0\ncodec_type=audio\nside_data_type=Encryption info", + ), + ), + ) + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service(executor).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.AUDIO_STREAM_PROTECTED, + ), + result, + ) + assertEquals(listOf("0"), executor.ffprobeSelectors) + assertEquals(0, executor.ffmpegCalls) + } finally { + request.close() + } + } + + @Test + fun `refuses to replace a wrong MPV stream index when multiple audio streams exist`() = runBlocking { + val request = request(audioInput(audioStreamIndex = 0)) + + try { + val result = service( + RecordingExecutor( + audioProbeResults = mapOf( + "0" to SceneCommandResult.Success("index=0\ncodec_type=video"), + "a" to SceneCommandResult.Success( + """ + [STREAM] + index=1 + codec_type=audio + [/STREAM] + [STREAM] + index=2 + codec_type=audio + [/STREAM] + """.trimIndent(), + ), + ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + AnkiSentenceAudioFailure.TRACK_MAPPING_UNAVAILABLE, + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports probe failure when FFprobe fails`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(audioProbeResults = mapOf("a:0" to SceneCommandResult.Failed)), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.AUDIO_PROBE_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports extraction failure when FFmpeg fails`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(ffmpegResult = SceneCommandResult.Failed), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.EXTRACTION_FAILED), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports missing output when FFmpeg succeeds without creating sentence audio`() = runBlocking { + val request = request() + + try { + val result = service( + RecordingExecutor(writeAudioOutput = false), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_OUTPUT_MISSING, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.ORIGINAL_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports source read failure when FFmpeg cannot read MPV playable source`() = runBlocking { + val request = request( + audioInput(origin = SceneVideoInputOrigin.PLAYABLE_VIDEO), + ) + + try { + val result = service( + RecordingExecutor( + ffmpegResult = SceneCommandResult.FfmpegFailed(SceneFfmpegFailure.SOURCE_READ), + ), + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable( + failure = AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED, + diagnostic = AnkiSentenceAudioDiagnostic( + inputSource = AnkiSentenceAudioInputSource.MPV_PLAYABLE_VIDEO, + ), + ), + result, + ) + } finally { + request.close() + } + } + + @Test + fun `reports extraction timeout when FFmpeg does not finish in time`() = runBlocking { + val diagnostics = RecordingDiagnosticLogger() + val request = request() + + try { + val result = service( + RecordingExecutor(ffmpegDelayMillis = 50), + timeoutMillis = 1, + diagnosticLogger = diagnostics, + ).prepare(request) + + assertEquals( + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT), + result, + ) + assertEquals( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = request.sentenceAudioInput, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + failure = AnkiSentenceAudioFailure.EXTRACTION_TIMED_OUT, + ), + diagnostics.events.last(), + ) + } finally { + request.close() + } + } + + private fun service( + executor: RecordingExecutor, + timeoutMillis: Long = 60_000L, + inputAcquirer: SceneInputAcquirer = SceneInputAcquirer { input -> leaseFor(input) }, + diagnosticLogger: SentenceAudioDiagnosticLogger = NoOpSentenceAudioDiagnosticLogger, + ): FrozenSceneSentenceAudioService { + return FrozenSceneSentenceAudioService( + cacheDirectory = tempDirectory, + inputAcquirer = inputAcquirer, + commandExecutor = executor, + timeoutMillis = timeoutMillis, + diagnosticLogger = diagnosticLogger, + ) + } + + private fun leaseFor(input: SceneVideoInputSpec): SceneInputLease { + return object : SceneInputLease { + override val ffmpegValue = input.value + override val tlsCaFile = "/files/cacert.pem" + + override fun close() = Unit + } + } + + private fun request( + sentenceAudioInput: SceneVideoInputSpec? = audioInput(), + sentenceAudioFallbackInput: SceneVideoInputSpec? = null, + sentenceAudioFallbackStatus: AnkiSentenceAudioPlayableFallback? = null, + ): SceneCaptureRequest { + val video = SceneVideoInputSpec( + value = "https://media.example/video.mkv", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = emptyList(), + ) + val bitmap = mockk(relaxed = true) + every { bitmap.isRecycled } returns false + return SceneCaptureRequest( + videoInput = video, + sentenceAudioInput = sentenceAudioInput, + sentenceAudioFallbackInput = sentenceAudioFallbackInput, + sentenceAudioFallbackStatus = sentenceAudioFallbackStatus, + resolvedTiming = SceneResolvedTiming( + animationRange = SceneTimeRange(1.25, 4.25), + audioRange = SceneTimeRange(1.25, 4.25), + ), + stillFallback = OwnedBitmap(bitmap), + ) + } + + private fun audioInput( + value: String = "https://media.example/audio.m4a", + audioStreamIndex: Int? = null, + origin: SceneVideoInputOrigin = SceneVideoInputOrigin.ORIGINAL_VIDEO, + ): SceneVideoInputSpec { + return SceneVideoInputSpec( + value = value, + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = emptyList(), + audioStreamIndex = audioStreamIndex, + origin = origin, + ) + } + + private class RecordingExecutor( + private val audioProbeResults: Map = mapOf( + "a:0" to SceneCommandResult.Success("codec_type=audio\ncodec_name=aac"), + ), + private val unrestrictedAudioProbeResult: SceneCommandResult? = null, + private val ffmpegResult: SceneCommandResult = SceneCommandResult.Success(), + private val ffmpegDelayMillis: Long = 0, + private val writeAudioOutput: Boolean = true, + ) : SceneCommandExecutor { + var ffmpegCalls = 0 + val ffprobeSelectors = mutableListOf() + val unrestrictedFfprobeSelectors = mutableListOf() + val ffmpegAudioMaps = mutableListOf() + val ffmpegInputs = mutableListOf() + + override suspend fun executeFfmpeg( + arguments: Array, + onNativeFinished: () -> Unit, + ): SceneCommandResult { + return try { + ffmpegCalls++ + ffmpegInputs += arguments[arguments.indexOf("-i") + 1] + ffmpegAudioMaps += arguments[arguments.indexOf("-map") + 1] + if (ffmpegDelayMillis > 0) delay(ffmpegDelayMillis) + if (ffmpegResult is SceneCommandResult.Success && writeAudioOutput) { + val output = File(arguments.last()) + output.writeBytes(byteArrayOf(1, 2, 3)) + } + ffmpegResult + } finally { + onNativeFinished() + } + } + + override suspend fun executeFfprobe( + arguments: Array, + onNativeFinished: () -> Unit, + ): SceneCommandResult { + return try { + val selector = arguments[arguments.indexOf("-select_streams") + 1] + ffprobeSelectors += selector + val restrictDecoders = "-codec_whitelist" in arguments + if (!restrictDecoders) { + unrestrictedFfprobeSelectors += selector + return unrestrictedAudioProbeResult + ?: error("Unexpected unrestricted stream selector: $selector") + } + when (selector) { + "v:0" -> SceneCommandResult.Success( + "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\nbits_per_raw_sample=10", + ) + else -> audioProbeResults[selector] + ?: error("Unexpected stream selector: $selector") + } + } finally { + onNativeFinished() + } + } + } + + private class RecordingDiagnosticLogger : SentenceAudioDiagnosticLogger { + val events = mutableListOf() + + override fun record(event: SentenceAudioDiagnosticEvent) { + events += event + } + } +} diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt index 243daccfde..c797c486eb 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SceneVideoInputTest.kt @@ -61,6 +61,82 @@ class SceneVideoInputTest { cases.forEach { assertNull(resolve(it)) } } + @Test + fun `resolved inputs retain whether export uses original playable or external audio`() { + val original = requireNotNull( + resolve(snapshot("/video/original.mkv", playableValue = "/video/playable.mkv")), + ) + val playable = requireNotNull( + resolve(snapshot("", playableValue = "/video/playable.mkv")), + ) + val externalAudio = requireNotNull( + resolve( + snapshot( + "/audio/episode.m4a", + playableValue = "/audio/episode.m4a", + isExternalAudio = true, + ), + ), + ) + + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, original.origin) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playable.origin) + assertEquals(SceneVideoInputOrigin.EXTERNAL_AUDIO, externalAudio.origin) + } + + @Test + fun `playable resolution keeps MPV source separate from an original video source`() { + val input = snapshot( + value = "https://media.example/original.m3u8", + playableValue = "https://media.example/playable.m3u8", + ) + + val original = requireNotNull(resolve(input)) + val playable = requireNotNull(SceneVideoInputResolver.resolvePlayable(input)) + + assertEquals("https://media.example/original.m3u8", original.value) + assertEquals(SceneVideoInputOrigin.ORIGINAL_VIDEO, original.origin) + assertEquals("https://media.example/playable.m3u8", playable.value) + assertEquals(SceneVideoInputOrigin.PLAYABLE_VIDEO, playable.origin) + } + + @Test + fun `playable fallback resolution distinguishes usable missing same and unsafe sources`() { + val originalValue = "https://media.example/original.m3u8" + val original = requireNotNull(resolve(snapshot(originalValue))) + + val available = SceneVideoInputResolver.resolvePlayableFallback( + snapshot( + value = originalValue, + playableValue = "https://media.example/playable.m3u8", + ), + original, + ) + val missing = SceneVideoInputResolver.resolvePlayableFallback( + snapshot(value = originalValue, playableValue = ""), + original, + ) + val same = SceneVideoInputResolver.resolvePlayableFallback( + snapshot(value = originalValue, playableValue = originalValue), + original, + ) + val unsafe = SceneVideoInputResolver.resolvePlayableFallback( + snapshot( + value = originalValue, + playableValue = "https://media.example/playable.m3u8?token=secret", + ), + original, + ) + + assertEquals( + "https://media.example/playable.m3u8", + (available as ScenePlayableFallbackResolution.Available).input.value, + ) + assertEquals(ScenePlayableFallbackResolution.Missing, missing) + assertEquals(ScenePlayableFallbackResolution.SameAsOriginal, same) + assertEquals(ScenePlayableFallbackResolution.Unavailable, unsafe) + } + @Test fun `AVIF command has the single bounded native recipe`() { val input = supportedInput() @@ -92,10 +168,6 @@ class SceneVideoInputTest { assertTrue( arguments.containsAll( listOf( - "-tls_verify", - "1", - "-ca_file", - "/files/cacert.pem", "-protocol_whitelist", "http,https,tls,tcp,crypto", "-rw_timeout", @@ -125,6 +197,7 @@ class SceneVideoInputTest { ), SceneFfmpegArguments.videoProbe(input, input.value, caFile), SceneFfmpegArguments.audioProbe(input, input.value, caFile), + SceneFfmpegArguments.allAudioProbe(input, input.value, caFile), SceneFfmpegArguments.sentenceAudio(input, input.value, range, "/cache/audio.m4a", caFile), ) @@ -138,6 +211,23 @@ class SceneVideoInputTest { } } + @Test + fun `audio discovery probe omits the decoder whitelist and only reads stream metadata`() { + val input = supportedInput() + val arguments = SceneFfmpegArguments.audioDiscoveryProbe( + input = input, + acquiredInputValue = input.value, + tlsCaFile = "/files/cacert.pem", + ).toList() + + assertFalse("-codec_whitelist" in arguments) + assertEquals("a", arguments[arguments.indexOf("-select_streams") + 1]) + assertEquals( + "stream=index,codec_type,codec_name:stream_side_data", + arguments[arguments.indexOf("-show_entries") + 1], + ) + } + @Test fun `sentence audio maps the frozen selected stream`() { val input = supportedInput().copy(videoStreamIndex = 2, audioStreamIndex = 3) @@ -179,16 +269,19 @@ class SceneVideoInputTest { private fun snapshot( value: String, + playableValue: String? = value, headers: List> = emptyList(), ffmpegStreamArgs: List> = emptyList(), seekable: Boolean = true, + isExternalAudio: Boolean = false, ) = SceneVideoInputSnapshot( originalVideoValue = value, - playableValue = value, + playableValue = playableValue, headers = headers, ffmpegStreamArgs = ffmpegStreamArgs, ffmpegVideoArgs = emptyList(), seekable = seekable, + isExternalAudio = isExternalAudio, ) private companion object { diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt new file mode 100644 index 0000000000..944ff97b6d --- /dev/null +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/player/scene/SentenceAudioDiagnosticLogTest.kt @@ -0,0 +1,102 @@ +package eu.kanade.tachiyomi.ui.player.scene + +import chimahon.anki.AnkiSentenceAudioFailure +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class SentenceAudioDiagnosticLogTest { + @Test + fun `records a redacted native failure with its sentence audio stage`() { + val input = SceneVideoInputSpec( + value = "https://media.example/episode.m3u8?token=secret", + kind = SceneVideoInputKind.REMOTE_HTTP, + headers = listOf("Authorization" to "Bearer secret"), + audioStreamIndex = 1, + origin = SceneVideoInputOrigin.PLAYABLE_VIDEO, + ) + + val text = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = input, + fallback = SentenceAudioDiagnosticFallback.ATTEMPTED, + failure = AnkiSentenceAudioFailure.EXTRACTION_SOURCE_READ_FAILED, + result = SceneCommandResult.FfmpegFailed( + failure = SceneFfmpegFailure.SOURCE_READ, + nativeDiagnostics = SceneNativeFailureDiagnostics( + returnCode = 1, + failStackTrace = null, + logs = "HTTP error 403 for https://media.example/episode.m3u8?token=secret\n" + + "Authorization: Bearer secret", + ), + ), + ), + ) + + assertTrue(text.contains("stage=AUDIO_EXTRACTION")) + assertTrue(text.contains("input_source=PLAYABLE_VIDEO")) + assertTrue(text.contains("input_kind=REMOTE_HTTP")) + assertTrue(text.contains("audio_stream_index=1")) + assertTrue(text.contains("fallback=ATTEMPTED")) + assertTrue(text.contains("failure=EXTRACTION_SOURCE_READ_FAILED")) + assertTrue(text.contains("return_code=1")) + assertTrue(text.contains("HTTP error 403")) + assertFalse(text.contains(input.value)) + assertFalse(text.contains("token=secret")) + assertFalse(text.contains("Bearer secret")) + } + + @Test + fun `keeps the newest entries within its byte budget`() { + var bytes = ByteArray(0) + + repeat(8) { + bytes = SentenceAudioDiagnosticJournal.retain( + existing = bytes, + entry = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.SELECTED_AUDIO_PROBE, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ).encodeToByteArray(), + maxBytes = 512, + ) + } + bytes = SentenceAudioDiagnosticJournal.retain( + existing = bytes, + entry = SentenceAudioDiagnosticJournal.render( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.AUDIO_EXTRACTION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ).encodeToByteArray(), + maxBytes = 512, + ) + + assertTrue(bytes.size <= 512) + assertTrue(bytes.decodeToString().contains("stage=AUDIO_EXTRACTION")) + } + + @Test + fun `reports a file writer failure instead of discarding it`() { + var writeFailure: Throwable? = null + val logger = StorageFolderSentenceAudioDiagnosticLogger( + directory = { null }, + onWriteFailure = { writeFailure = it }, + ) + + logger.record( + SentenceAudioDiagnosticEvent( + stage = SentenceAudioDiagnosticStage.REQUEST_VALIDATION, + input = null, + fallback = SentenceAudioDiagnosticFallback.NOT_APPLICABLE, + ), + ) + + assertNotNull(writeFailure) + } +} diff --git a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt index 6aa3801feb..7ee1a7b3e9 100644 --- a/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt +++ b/app/src/test/kotlin/eu/kanade/tachiyomi/ui/reader/setting/ReaderOcrSourceTest.kt @@ -37,8 +37,8 @@ class ReaderOcrSourceTest { @Test fun `local source is only offered when available`() { - assertEquals(ReaderOcrSource.entries, ReaderOcrSource.availableSources(localOcrAvailable = true)) - assertFalse(ReaderOcrSource.LOCAL in ReaderOcrSource.availableSources(localOcrAvailable = false)) + assertEquals(ReaderOcrSource.entries, ReaderOcrSource.availableSources(localOcrAvailable = true, mokuroAvailable = true)) + assertFalse(ReaderOcrSource.LOCAL in ReaderOcrSource.availableSources(localOcrAvailable = false, mokuroAvailable = true)) } @Test diff --git a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt index d9b64fd478..8bfdc65b99 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiCardCreator.kt @@ -384,16 +384,28 @@ object AnkiCardCreator { ownedPreparedFiles += screenshotPreparation.animation } + var sentenceAudioGenerationFailure: AnkiSentenceAudioFailure? = null + var sentenceAudioGenerationDiagnostic: AnkiSentenceAudioDiagnostic? = null + var sentenceAudioStorageFailed = false val sentenceAudioSource = if (hasSentenceAudioMarker) { - val lazySource = try { - mediaRequest?.sentenceAudioProvider?.prepare() - } catch (e: CancellationException) { - throw e - } catch (e: Exception) { - android.util.Log.w(TAG, "addToAnki: failed to prepare sentence audio", e) + val provider = mediaRequest?.sentenceAudioProvider + val lazyPreparation = if (provider != null) { + try { + provider.prepare() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + android.util.Log.w(TAG, "addToAnki: failed to prepare sentence audio", e) + AnkiSentenceAudioPreparation.Unavailable(AnkiSentenceAudioFailure.UNKNOWN) + } + } else { null } - lazySource ?: sentenceAudioBytes?.let { bytes -> + val lazySource = (lazyPreparation as? AnkiSentenceAudioPreparation.Ready)?.source + val lazyUnavailable = lazyPreparation as? AnkiSentenceAudioPreparation.Unavailable + val lazyFailure = lazyUnavailable?.failure + val lazyDiagnostic = lazyUnavailable?.diagnostic + val eagerSource = sentenceAudioBytes?.let { bytes -> val filename = generateSentenceAudioFilename(bytes, sentenceAudioExtension) AnkiMediaSource.Bytes( data = bytes, @@ -401,6 +413,11 @@ object AnkiCardCreator { extension = AnkiMediaNaming.safeExtension(filename, "m4a"), ) } + if (provider != null && lazySource == null && eagerSource == null) { + sentenceAudioGenerationFailure = lazyFailure ?: AnkiSentenceAudioFailure.UNKNOWN + sentenceAudioGenerationDiagnostic = lazyDiagnostic + } + lazySource ?: eagerSource } else { null } @@ -472,6 +489,9 @@ object AnkiCardCreator { sentenceAudioSource, "sentence audio", ) + if (sentenceAudioSource != null && sentenceAudioFilename == null) { + sentenceAudioStorageFailed = true + } val fieldsWithDictionaryMedia = resolveDictionaryMediaPlaceholders( fieldsWithPlaceholders, exportMedia, @@ -515,7 +535,19 @@ object AnkiCardCreator { titleId = titleId, ) if (syncOnCreate) bridge.triggerSync() - AnkiResult.Success(noteId, screenshotResult.warnings) + AnkiResult.Success( + noteId, + screenshotResult.warnings + listOfNotNull( + sentenceAudioGenerationFailure?.let { failure -> + AnkiMediaWarning.SentenceAudioGenerationFailed( + failure, + sentenceAudioGenerationDiagnostic, + ) + }, + AnkiMediaWarning.SentenceAudioStorageFailed + .takeIf { sentenceAudioStorageFailed }, + ), + ) } } } catch (e: CancellationException) { diff --git a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt index 8f97351d72..2ede7313a3 100644 --- a/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt +++ b/chimahon/src/main/java/chimahon/anki/AnkiMedia.kt @@ -44,10 +44,69 @@ sealed interface AnkiMediaSource { ) : AnkiMediaSource } +enum class AnkiSentenceAudioFailure { + TRACK_MAPPING_UNAVAILABLE, + SOURCE_UNAVAILABLE, + TIMING_UNAVAILABLE, + AUDIO_PROBE_FAILED, + AUDIO_STREAMS_NOT_FOUND, + AUDIO_CODEC_RESTRICTED, + AUDIO_STREAM_INDEX_UNAVAILABLE, + AUDIO_STREAM_NOT_AUDIO, + AUDIO_STREAM_PROTECTED, + AUDIO_STREAM_UNREADABLE, + EXTRACTION_FAILED, + EXTRACTION_OUTPUT_MISSING, + EXTRACTION_OUTPUT_READ_FAILED, + EXTRACTION_STREAM_MAPPING_FAILED, + EXTRACTION_SOURCE_READ_FAILED, + EXTRACTION_SEEK_FAILED, + EXTRACTION_OUTPUT_WRITE_FAILED, + EXTRACTION_TIMED_OUT, + UNKNOWN, +} + +enum class AnkiSentenceAudioInputSource { + ORIGINAL_VIDEO, + MPV_PLAYABLE_VIDEO, + MPV_EXTERNAL_AUDIO, +} + +/** + * Safe, frozen reason why the original video could not be retried through MPV's playable source. + * It deliberately does not expose a path, URL, header, or player property. + */ +enum class AnkiSentenceAudioPlayableFallback { + MISSING, + SAME_AS_ORIGINAL, + UNAVAILABLE, +} + +data class AnkiSentenceAudioDiagnostic( + val inputSource: AnkiSentenceAudioInputSource, + val playableFallback: AnkiSentenceAudioPlayableFallback? = null, +) + +sealed interface AnkiSentenceAudioPreparation { + data class Ready( + val source: AnkiMediaSource, + ) : AnkiSentenceAudioPreparation + + data class Unavailable( + val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, + ) : AnkiSentenceAudioPreparation +} + sealed interface AnkiMediaWarning { data object SceneGenerationFailed : AnkiMediaWarning data object AnimatedStorageFailed : AnkiMediaWarning data object StillStorageFailed : AnkiMediaWarning + data class SentenceAudioGenerationFailed( + val failure: AnkiSentenceAudioFailure, + val diagnostic: AnkiSentenceAudioDiagnostic? = null, + ) : AnkiMediaWarning + data object SentenceAudioStorageFailed : AnkiMediaWarning } sealed interface AnkiScreenshotPreparation { @@ -70,7 +129,7 @@ fun interface LazyAnkiScreenshotProvider { } fun interface LazyAnkiMediaProvider { - suspend fun prepare(): AnkiMediaSource? + suspend fun prepare(): AnkiSentenceAudioPreparation } data class AnkiMediaRequest( diff --git a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml index eff6a5eda9..18fae336bf 100644 --- a/i18n-kmk/src/commonMain/moko-resources/base/strings.xml +++ b/i18n-kmk/src/commonMain/moko-resources/base/strings.xml @@ -259,6 +259,39 @@ The animated scene could not be created. Chimahon used the captured still screenshot when available. AnkiDroid could not store the animated scene. Chimahon used the captured still screenshot when available. The screenshot could not be stored. The card was created without it. + Sentence audio could not be exported from this video source. The card was created without it. + The selected audio track could not be identified safely. Sentence audio was not added to avoid choosing the wrong track. + The selected video source could not be read for sentence audio. The card was created without it. + The subtitle timing was unavailable. Sentence audio was not added. + [Audio Probe Failed] The selected audio stream could not be inspected. The card was created without sentence audio. + [Audio Probe Failed (Original)] FFprobe could not inspect audio in the original video source. The card was created without sentence audio. + [Audio Probe Failed (Playable)] FFprobe could not inspect audio in MPV's playable video source. The card was created without sentence audio. + [Audio Streams Not Found] FFmpeg could not find an audio stream in the source used for export. The card was created without sentence audio. + [Audio Streams Not Found (Original)] FFmpeg could not find an audio stream in the original video source used for export. MPV may be playing a different source. The card was created without sentence audio. + [Audio Streams Not Found (Playable Missing)] FFmpeg could not find an audio stream in the original video source, and MPV did not provide a separate playable source to try. The card was created without sentence audio. + [Audio Streams Not Found (Playable Same)] FFmpeg could not find an audio stream in the original video source, and MPV's playable source was the same source. The card was created without sentence audio. + [Audio Streams Not Found (Playable Unsafe)] FFmpeg could not find an audio stream in the original video source, and MPV's separate playable source was not safe to use for export. The card was created without sentence audio. + [Audio Streams Not Found (Playable)] FFmpeg could not find an audio stream in MPV's playable video source. The card was created without sentence audio. + [Audio Streams Not Found (External)] MPV selected external audio, but FFmpeg could not find an audio stream in that source. The card was created without sentence audio. + [Codec Restricted] FFmpeg found audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (Original)] FFmpeg found audio in the original video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (Playable)] FFmpeg found audio in MPV's playable video source, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Codec Restricted (External)] MPV selected external audio, but its codec is outside the safe export allow-list. The card was created without sentence audio. + [Stream Index Unavailable] The audio track selected in the player was not found in this video. The card was created without sentence audio. + [Track Not Audio] The track selected as audio did not resolve to an audio stream in this video. The card was created without sentence audio. + [Stream Protected] The selected audio stream is protected and cannot be exported. The card was created without sentence audio. + [Stream Unreadable] No readable audio stream was found in this video. The card was created without sentence audio. + [Audio Extraction Failed] FFmpeg found an audio stream but could not extract sentence audio from this video. The card was created without it. + [Extraction Output Missing] FFmpeg completed but did not create a usable sentence-audio file. The card was created without it. + [Extraction Output Read Failed] FFmpeg created sentence audio, but Chimahon could not read its temporary file. The card was created without it. + [Stream Mapping Failed] FFmpeg could not map the selected audio track for sentence export. The card was created without it. + [Source Read Failed] FFmpeg could not read the audio data from the video source during sentence export. The card was created without it. + [Source Read Failed (Original)] FFmpeg could not read audio data from the original video source during sentence export. The card was created without it. + [Source Read Failed (Playable)] FFmpeg could not read audio data from MPV's playable video source during sentence export. The card was created without it. + [Seek Failed] FFmpeg could not seek the video source to the subtitle time for sentence export. The card was created without it. + [Output Write Failed] FFmpeg could not write the temporary sentence audio file. The card was created without it. + [Extraction Timed Out] Sentence audio extraction timed out. The card was created without it. + [Audio Storage Failed] AnkiDroid could not store the sentence audio. The card was created without it. Media Learning Sync