Improve RSVP thought flow and retire legacy grouping

README.md

@@ -89,7 +89,7 @@ The reader, tokenizer, persistence layer, and RSVP engine all work toward the sa
 
 ## Built With
 
-- Kotlin 2.4.10
+- Kotlin 2.4.20
 - Jetpack Compose
 - AndroidX Navigation
 - Room

app/src/main/java/com/kairo/reader/core/model/RsvpFrame.kt

@@ -19,4 +19,8 @@ data class RsvpFrame(
     // fragments such as "ha-ha-ha". A null end means the complete final token.
     val displayOriginalStartCharacterOffset: Int = 0,
     val displayOriginalEndCharacterOffset: Int? = null,
+    // Shared thought boundaries stay in source coordinates across grouping and split words.
+    val phraseStartTokenIndex: Int? = null,
+    val phraseEndTokenIndexExclusive: Int? = null,
+    val endsPhrase: Boolean = false,
 )

app/src/main/java/com/kairo/reader/core/model/TokenUtils.kt

@@ -303,7 +303,7 @@ private fun splitLongWordToken(
         token.copy(
             text = text,
             // Phrase chunking across subword splits is blocked via isSubwordChunk in
-            // isPhraseChunkCandidate; isClauseBoundary must stay truthful because the timing
+            // the scored segmenter; isClauseBoundary must stay truthful because the timing
             // model reads it (clause holds would otherwise fire on every chunk of a long word).
             isClauseBoundary = if (isLast) token.isClauseBoundary else false,
             isDialogue = token.isDialogue,

app/src/main/java/com/kairo/reader/core/rsvp/RsvpEngine.kt

@@ -88,7 +88,7 @@ class ComprehensionRsvpEngine : RsvpEngine {
             tokens = tokens,
             startIndex = startIndex,
             config = config,
-            options = RsvpGenerationOptions.LEGACY,
+            options = RsvpGenerationOptions.DEFAULT,
         )
 
     override fun generateFrames(
@@ -110,7 +110,7 @@ private data class RsvpGenerationContext(
     val expanded: List<ExpandedToken>,
     val config: RsvpConfig,
     val options: RsvpGenerationOptions,
-    val atomStream: RsvpAtomStream?,
+    val atomStream: RsvpAtomStream,
     val analysis: RsvpTokenAnalysis,
     val frames: MutableList<RsvpFrame>,
     val state: ContextState,
@@ -136,18 +136,13 @@ private fun generateFramesWithNormalizedConfig(
             expanded = expanded,
             config = config,
             options = options,
-            atomStream =
-            if (options.usesScoredSegmentation(config)) {
-                RsvpAtomStream.build(
-                    expandedTokens = expanded,
-                    languagePolicy = options.languagePolicy,
-                    useDialogueDetection = config.useDialogueDetection,
-                    useParentheticalAside = config.useParentheticalAside,
-                )
-            } else {
-                null
-            },
-            analysis = analyzeExpandedTokens(expanded, config),
+            atomStream = RsvpAtomStream.build(
+                expandedTokens = expanded,
+                languagePolicy = options.languagePolicy,
+                useDialogueDetection = config.useDialogueDetection,
+                useParentheticalAside = config.useParentheticalAside,
+            ),
+            analysis = analyzeExpandedTokens(expanded, config, options.languagePolicy),
             frames = mutableListOf(),
             state = createContextState(tokens, expanded[cursor].originalIndex),
             rhythm = createRhythmState(config),
@@ -396,17 +391,20 @@ private fun RsvpGenerationContext.appendReadingFrame(cursor: Int): Int? {
     val wordCursor = findFirstWordCursor(expanded, cursor)
     if (wordCursor >= expanded.size) return null
     val frameStartCursor = cursor
-    val scoredSelection = selectScoredFrame(cursor)
+    val selection = selectFrame(cursor)
     val contextBefore = state.snapshot()
     val (frameTokens, frameOriginalIndex, nextCursor) =
         buildUnit(
             expandedTokens = expanded,
             startCursor = cursor,
             config = config,
             state = state,
-            selectedWordCursors = scoredSelection?.selectedWordCursors,
+            selectedWordCursors = selection.selectedWordCursors,
+            phraseEndTokenIndexExclusive = analysis.thoughtCues[wordCursor]?.endTokenIndexExclusive,
         )
 
+    val unitCues = (wordCursor until nextCursor).mapNotNull { analysis.thoughtCues[it] }
+
     val durationMs =
         computeUnitDurationMs(
             RsvpUnitTimingInput(
@@ -429,9 +427,10 @@ private fun RsvpGenerationContext.appendReadingFrame(cursor: Int): Int? {
                 (frameStartCursor until nextCursor).any { it in analysis.pairedEmDashIndices },
                 afterPairedEmDash = followsPairedEmDash(wordCursor),
                 rhythmBoundaryStrengthMilli =
-                scoredSelection?.boundaryStrengthBeforeMilli ?: 0,
+                selection.boundaryStrengthBeforeMilli,
                 explicitSpeakerTag =
-                scoredSelection?.dialogueRole == RsvpDialogueRole.SPEAKER_TAG,
+                selection.dialogueRole == RsvpDialogueRole.SPEAKER_TAG,
+                thoughtCues = unitCues,
             ),
         )
 
@@ -456,22 +455,21 @@ private fun RsvpGenerationContext.appendReadingFrame(cursor: Int): Int? {
             expanded[frameStartCursor].sourceCharacterStart,
             displayOriginalEndCharacterOffset =
             expanded.getOrNull(nextCursor - 1)?.sourceCharacterEndExclusive,
+            phraseStartTokenIndex = unitCues.firstOrNull()?.startTokenIndex,
+            phraseEndTokenIndexExclusive = unitCues.lastOrNull()?.endTokenIndexExclusive,
+            endsPhrase = unitCues.lastOrNull()?.isLastWord == true,
         )
 
     return consumeContextPunctuation(nextCursor)
 }
 
-private fun RsvpGenerationContext.selectScoredFrame(cursor: Int): RsvpSegmentationDecision? {
-    if (!options.usesScoredSegmentation(config)) return null
-    val atoms = atomStream ?: return null
-    return RsvpDpSegmenter
-        .selectWordCount(
-            atomStream = atoms,
-            startCursor = cursor,
-            config = config,
-            languagePolicy = options.languagePolicy,
-        )
-}
+private fun RsvpGenerationContext.selectFrame(cursor: Int): RsvpSegmentationDecision =
+    RsvpDpSegmenter.selectWordCount(
+        atomStream = atomStream,
+        startCursor = cursor,
+        config = config,
+        languagePolicy = options.languagePolicy,
+    )
 
 /**
  * Whether the boundary punctuation directly before this word is the closing (or opening) dash of

app/src/main/java/com/kairo/reader/core/rsvp/RsvpEstimatedReadingPace.kt

@@ -23,7 +23,7 @@ object RsvpEstimatedReadingPace {
         sessionTempoMsPerWord: Long? = null,
         fallbackEstimatedWpm: Int = 0,
         languageTag: String? = null,
-        paceOptions: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.LEGACY,
+        paceOptions: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.DEFAULT,
     ): Int {
         val effectiveConfig =
             sessionTempoMsPerWord
@@ -96,7 +96,7 @@ object RsvpEstimatedReadingPace {
         sessionTempoMsPerWord: Long?,
         fallbackEstimatedWpm: Int = 0,
         languageTag: String? = null,
-        paceOptions: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.LEGACY,
+        paceOptions: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.DEFAULT,
     ): Int {
         val effectiveTempoMsPerWord =
             sessionTempoMsPerWord

app/src/main/java/com/kairo/reader/core/rsvp/RsvpGenerationOptions.kt

@@ -2,7 +2,6 @@ package com.kairo.reader.core.rsvp
 
 import com.kairo.reader.core.language.LanguageFamily
 import com.kairo.reader.core.language.LanguageFamilyClassifier
-import com.kairo.reader.core.model.RsvpConfig
 
 enum class RsvpLanguagePolicy {
     ENGLISH,
@@ -24,94 +23,25 @@ enum class RsvpLanguagePolicy {
     }
 }
 
-enum class RsvpSegmentationStrategy {
-    LEGACY_GREEDY,
-    SCORED_DP_V2,
-}
-
-data class RsvpGenerationOptions(
-    val languagePolicy: RsvpLanguagePolicy = RsvpLanguagePolicy.UNKNOWN,
-    val segmentationStrategy: RsvpSegmentationStrategy = RsvpSegmentationStrategy.LEGACY_GREEDY,
-) {
-    fun asPaceEstimationOptions(): RsvpPaceEstimationOptions =
-        if (languagePolicy == RsvpLanguagePolicy.ENGLISH) {
-            RsvpPaceEstimationOptions(segmentationStrategy = segmentationStrategy)
-        } else {
-            // The estimator currently uses an English sample. Until there are representative
-            // samples for each language family, do not apply a non-English policy to that text.
-            RsvpPaceEstimationOptions.LEGACY
-        }
+/** Language changes the scoring evidence, never the segmentation implementation. */
+data class RsvpGenerationOptions(val languagePolicy: RsvpLanguagePolicy = RsvpLanguagePolicy.UNKNOWN,) {
+    // The estimator currently has one English sample. Do not apply another language's
+    // rules to it; add representative samples before expanding estimation policies.
+    fun asPaceEstimationOptions(): RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.DEFAULT
 
     companion object {
-        val LEGACY = RsvpGenerationOptions()
+        val DEFAULT = RsvpGenerationOptions()
+
+        fun fromLanguageTag(languageTag: String?): RsvpGenerationOptions =
+            RsvpGenerationOptions(RsvpLanguagePolicy.fromLanguageTag(languageTag))
     }
 }
 
-internal fun RsvpGenerationOptions.usesScoredSegmentation(config: RsvpConfig): Boolean =
-    segmentationStrategy == RsvpSegmentationStrategy.SCORED_DP_V2 &&
-        when (languagePolicy) {
-            RsvpLanguagePolicy.ENGLISH ->
-                config.maxWordsPerUnit in SUPPORTED_ENGLISH_SCORED_WORD_COUNTS
-            RsvpLanguagePolicy.DEFAULT_NON_ENGLISH,
-            RsvpLanguagePolicy.CJK,
-            RsvpLanguagePolicy.RTL ->
-                config.maxWordsPerUnit in SUPPORTED_NON_ENGLISH_SCORED_WORD_COUNTS
-            RsvpLanguagePolicy.UNKNOWN -> false
-        }
-
-data class RsvpPaceEstimationOptions(
-    val sampleLanguagePolicy: RsvpLanguagePolicy = RsvpLanguagePolicy.ENGLISH,
-    val segmentationStrategy: RsvpSegmentationStrategy = RsvpSegmentationStrategy.LEGACY_GREEDY,
-) {
+data class RsvpPaceEstimationOptions(val sampleLanguagePolicy: RsvpLanguagePolicy = RsvpLanguagePolicy.ENGLISH,) {
     fun asGenerationOptions(): RsvpGenerationOptions =
-        if (sampleLanguagePolicy == RsvpLanguagePolicy.ENGLISH) {
-            RsvpGenerationOptions(
-                languagePolicy = sampleLanguagePolicy,
-                segmentationStrategy = segmentationStrategy,
-            )
-        } else {
-            RsvpGenerationOptions.LEGACY
-        }
+        RsvpGenerationOptions(languagePolicy = RsvpLanguagePolicy.ENGLISH)
 
     companion object {
-        val LEGACY = RsvpPaceEstimationOptions()
-    }
-}
-
-object RsvpSegmentationRolloutResolver {
-    fun resolve(
-        languageTag: String?,
-        config: RsvpConfig,
-        isDebugBuild: Boolean,
-    ): RsvpGenerationOptions =
-        resolve(
-            languagePolicy = RsvpLanguagePolicy.fromLanguageTag(languageTag),
-            config = config,
-            isDebugBuild = isDebugBuild,
-        )
-
-    fun resolve(
-        languagePolicy: RsvpLanguagePolicy,
-        config: RsvpConfig,
-        isDebugBuild: Boolean,
-    ): RsvpGenerationOptions {
-        val scoredOptions =
-            RsvpGenerationOptions(
-                languagePolicy = languagePolicy,
-                segmentationStrategy = RsvpSegmentationStrategy.SCORED_DP_V2,
-            )
-        val strategy =
-            if (isDebugBuild && scoredOptions.usesScoredSegmentation(config)) {
-                RsvpSegmentationStrategy.SCORED_DP_V2
-            } else {
-                RsvpSegmentationStrategy.LEGACY_GREEDY
-            }
-        return RsvpGenerationOptions(
-            languagePolicy = languagePolicy,
-            segmentationStrategy = strategy,
-        )
+        val DEFAULT = RsvpPaceEstimationOptions()
     }
 }
-
-private val SUPPORTED_ENGLISH_SCORED_WORD_COUNTS = 1..3
-private val SUPPORTED_NON_ENGLISH_SCORED_WORD_COUNTS = 1..2

app/src/main/java/com/kairo/reader/core/rsvp/RsvpPaceEstimator.kt

@@ -21,7 +21,7 @@ object RsvpPaceEstimator {
 
     fun estimateWpm(
         config: RsvpConfig,
-        options: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.LEGACY,
+        options: RsvpPaceEstimationOptions = RsvpPaceEstimationOptions.DEFAULT,
     ): Int {
         val steadyConfig =
             config.copy(

app/src/main/java/com/kairo/reader/core/rsvp/analysis/RsvpPhraseAnalysis.kt

@@ -6,6 +6,7 @@ import com.kairo.reader.core.model.RsvpConfig
 import com.kairo.reader.core.model.Token
 import com.kairo.reader.core.model.TokenType
 import com.kairo.reader.core.model.isSentenceEndingPunctuation
+import com.kairo.reader.core.rsvp.RsvpLanguagePolicy
 import com.kairo.reader.core.rsvp.engine.CLAUSE_ANTICIPATORY_CONTOUR
 import com.kairo.reader.core.rsvp.engine.CLAUSE_PRE_BOUNDARY_CONTOUR
 import com.kairo.reader.core.rsvp.engine.CLAUSE_RESTART_CONTOUR
@@ -26,6 +27,7 @@ internal data class RsvpTokenAnalysis(
     /** Expanded indices of em/en-dash tokens that open or close a paired aside within a sentence. */
     val pairedEmDashIndices: Set<Int>,
     val phraseContours: Map<Int, PhraseContour>,
+    val thoughtCues: Map<Int, RsvpThoughtCue> = emptyMap(),
 ) {
     companion object {
         val EMPTY =
@@ -42,6 +44,7 @@ internal data class RsvpTokenAnalysis(
 internal fun analyzeExpandedTokens(
     expanded: List<ExpandedToken>,
     config: RsvpConfig,
+    languagePolicy: RsvpLanguagePolicy = RsvpLanguagePolicy.UNKNOWN,
 ): RsvpTokenAnalysis {
     if (expanded.isEmpty()) return RsvpTokenAnalysis.EMPTY
 
@@ -50,6 +53,7 @@ internal fun analyzeExpandedTokens(
     val asides = HashSet<Int>()
     val pairedDashes = HashSet<Int>()
     val contours = HashMap<Int, PhraseContour>()
+    val thoughtCues = RsvpThoughtPlan.analyze(expanded, config, languagePolicy)
     val breathGroup = ArrayList<ExpandedToken>()
     var previousWord: Token? = null
     var emDashAsideCloseIndex = -1
@@ -92,7 +96,7 @@ internal fun analyzeExpandedTokens(
                 }
             }
         if (config.useFocalStress) {
-            addFocalWord(breathGroup, focal)
+            addFocalWord(breathGroup, focal, thoughtCues)
         }
         breathGroup.clear()
         applyRestartContour(tier = tier, afterIndex = boundaryIndex)
@@ -102,6 +106,9 @@ internal fun analyzeExpandedTokens(
         val token = entry.token
         when (token.type) {
             TokenType.WORD -> {
+                if (startsNewThought(entry, breathGroup, thoughtCues)) {
+                    applyBoundaryEffects(RsvpPunctuationTier.CLAUSE_BREAK, index - 1)
+                }
                 if (config.useParentheticalAside && emDashAsideCloseIndex > index) {
                     asides += entry.expandedIndex
                 }
@@ -110,7 +117,7 @@ internal fun analyzeExpandedTokens(
             }
             TokenType.PARAGRAPH_BREAK, TokenType.PAGE_BREAK -> {
                 if (config.useFocalStress) {
-                    addFocalWord(breathGroup, focal)
+                    addFocalWord(breathGroup, focal, thoughtCues)
                 }
                 breathGroup.clear()
                 previousWord = null
@@ -134,9 +141,7 @@ internal fun analyzeExpandedTokens(
                         prevWord = previousWord,
                         nextToken = nextTokenAfter(expanded, index),
                     )
-                if (tier == RsvpPunctuationTier.SENTENCE_END ||
-                    tier == RsvpPunctuationTier.CLAUSE_BREAK
-                ) {
+                if (tier.isThoughtBoundary()) {
                     applyBoundaryEffects(tier = tier, boundaryIndex = index)
                 }
                 if (index >= emDashAsideCloseIndex) {
@@ -146,7 +151,7 @@ internal fun analyzeExpandedTokens(
         }
     }
     if (config.useFocalStress) {
-        addFocalWord(breathGroup, focal)
+        addFocalWord(breathGroup, focal, thoughtCues)
     }
 
     return RsvpTokenAnalysis(
@@ -155,14 +160,33 @@ internal fun analyzeExpandedTokens(
         emDashAsideIndices = if (config.useParentheticalAside) asides else emptySet(),
         pairedEmDashIndices = pairedDashes,
         phraseContours = contours,
+        thoughtCues = thoughtCues,
     )
 }
 
+private fun RsvpPunctuationTier.isThoughtBoundary(): Boolean =
+    this == RsvpPunctuationTier.SENTENCE_END || this == RsvpPunctuationTier.CLAUSE_BREAK
+
+private fun startsNewThought(
+    entry: ExpandedToken,
+    group: List<ExpandedToken>,
+    cues: Map<Int, RsvpThoughtCue>,
+): Boolean =
+    group.isNotEmpty() &&
+        cues[entry.expandedIndex]?.startTokenIndex == entry.originalIndex &&
+        group.last().originalIndex != entry.originalIndex
+
 private fun addFocalWord(
     group: List<ExpandedToken>,
     focal: MutableSet<Int>,
+    thoughtCues: Map<Int, RsvpThoughtCue>,
 ) {
     if (group.isEmpty()) return
+    val protected = group.filter { thoughtCues[it.expandedIndex]?.protectedEmphasis == true }
+    if (protected.isNotEmpty()) {
+        protected.forEach { focal += it.expandedIndex }
+        return
+    }
     if (group.size == 1) {
         focal += group.first().expandedIndex
         return