From f3a3fe78ea23a3dc65cd0aa385dfa9708c0e016d Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 11:55:39 +0300 Subject: [PATCH 1/8] MS-1608 Add tracking mode feature flag definition --- .../models/ExperimentalProjectConfiguration.kt | 12 ++++++++++++ .../ExperimentalProjectConfigurationTest.kt | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt index a832cff387..eda1bdb72d 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt @@ -31,6 +31,17 @@ data class ExperimentalProjectConfiguration( ?.coerceIn(FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MIN, FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MAX) ?: FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_DEFAULT + /** + * Tracks the face anywhere in the preview instead of asking the subject to fill a fixed + * on-screen cutout, and picks the dominant face when several people are visible. + */ + val faceTrackingCaptureEnabled: Boolean + get() = customConfig + ?.get(FACE_TRACKING_CAPTURE_ENABLED) + ?.jsonPrimitive + ?.booleanOrNull + .let { it == true } + val recordsDbMigrationFromRealmEnabled: Boolean get() = customConfig ?.get(RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED) @@ -239,6 +250,7 @@ data class ExperimentalProjectConfiguration( companion object { internal const val DISABLE_SUBJECT_POOL_VALIDATION = "disableSubjectPoolValidation" internal const val FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS = "faceAutoCaptureImagingDurationMillis" + internal const val FACE_TRACKING_CAPTURE_ENABLED = "faceTrackingCaptureEnabled" internal const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED = "recordsDbMigrationFromRealmEnabled" const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_MAX_RETRIES = "recordsDbMigrationFromRealmMaxRetries" diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt index e95818ef80..0baf05f651 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt @@ -7,6 +7,7 @@ import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration. import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MAX import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MIN +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_CAPTURE_ENABLED import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.MFID_LIGHTING_CONDITIONS_ASSESSMENT_ENABLED @@ -59,6 +60,22 @@ internal class ExperimentalProjectConfigurationTest { } } + @Test + fun `check face tracking capture flag correctly`() { + mapOf( + // Value not present - the fixed cutout behaviour stays the default + emptyMap() to false, + // Value not boolean + mapOf(FACE_TRACKING_CAPTURE_ENABLED to JsonPrimitive(1)) to false, + // Value present and FALSE + mapOf(FACE_TRACKING_CAPTURE_ENABLED to JsonPrimitive(false)) to false, + // Value present and TRUE + mapOf(FACE_TRACKING_CAPTURE_ENABLED to JsonPrimitive(true)) to true, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).faceTrackingCaptureEnabled).isEqualTo(result) + } + } + @Test fun `check face auto capture imaging duration flag correctly`() { mapOf( From 9edaf3e22f45fcbe73cd97e453a58dee9180dd1b Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 12:20:59 +0300 Subject: [PATCH 2/8] MS-1608 Add an option to select specific face during detection --- .../basebiosdk/detection/FaceDetector.kt | 21 +++- .../infra/rocv1/detection/RocV1Detector.kt | 74 ++++++++---- .../infra/rocv3/detection/RocV3Detector.kt | 92 ++++++++++++--- .../simface/detection/SimFaceDetector.kt | 14 ++- .../simface/detection/SimFaceDetectorTest.kt | 108 ++++++++++++++++++ 5 files changed, 265 insertions(+), 44 deletions(-) diff --git a/face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/FaceDetector.kt b/face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/FaceDetector.kt index 60da3e7102..8397bf3cb4 100644 --- a/face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/FaceDetector.kt +++ b/face/infra/base-bio-sdk/src/main/java/com/simprints/face/infra/basebiosdk/detection/FaceDetector.kt @@ -1,8 +1,21 @@ package com.simprints.face.infra.basebiosdk.detection import android.graphics.Bitmap +import android.graphics.Rect import com.simprints.core.ExcludedFromGeneratedTestCoverageReports +/** + * Chooses which of the faces in an image a template should be extracted for. + * + * Receives every face the detector found, as bounding boxes in source-image pixels and in the + * order the SDK reported them, and returns the index to extract or null to extract none. + * + * This exists so the selection policy stays with the caller - the SDK has no way of knowing which + * person in the frame is the subject - while the expensive template extraction still runs only + * once, for the face that was chosen. + */ +typealias FaceSelector = (faces: List) -> Int? + @ExcludedFromGeneratedTestCoverageReports("No need to test the interface") interface FaceDetector { /** @@ -12,11 +25,16 @@ interface FaceDetector { * @param estimateAgeAndGender whether to also request age/gender estimation. This is extra native * processing on top of face detection/template extraction, so it should only be requested for * the final selected capture, not on every live-preview frame. - * @return Face object or null if no face is detected + * @param selectFace decides which face to extract when the image holds more than one. Defaults + * to the first face the SDK reports, which is all that can be done without a caller policy. + * Extraction runs for the selected face alone, so the returned [Face] always describes exactly + * one person. + * @return Face object, or null if no face was detected or [selectFace] chose none */ fun analyze( bitmap: Bitmap, estimateAgeAndGender: Boolean = false, + selectFace: FaceSelector? = null, ): Face? /** @@ -28,5 +46,6 @@ interface FaceDetector { fun spoofCheck( bitmap: Bitmap, configuredMaxSize: Int, + selectFace: FaceSelector? = null, ): SpoofCheckResult } diff --git a/face/infra/roc-v1/src/main/java/com/simprints/face/infra/rocv1/detection/RocV1Detector.kt b/face/infra/roc-v1/src/main/java/com/simprints/face/infra/rocv1/detection/RocV1Detector.kt index 3a88f2b04a..6d3a8cc3ea 100644 --- a/face/infra/roc-v1/src/main/java/com/simprints/face/infra/rocv1/detection/RocV1Detector.kt +++ b/face/infra/roc-v1/src/main/java/com/simprints/face/infra/rocv1/detection/RocV1Detector.kt @@ -5,6 +5,7 @@ import android.graphics.Rect import com.simprints.core.ExcludedFromGeneratedTestCoverageReports import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector +import com.simprints.face.infra.basebiosdk.detection.FaceSelector import com.simprints.face.infra.basebiosdk.detection.SpoofCheckResult import io.rankone.rocsdk.embedded.SWIGTYPE_p_float import io.rankone.rocsdk.embedded.SWIGTYPE_p_unsigned_char @@ -20,11 +21,6 @@ import javax.inject.Inject reason = "This class uses roc class that has native functions and can't be mocked", ) class RocV1Detector @Inject constructor() : FaceDetector { - companion object { - const val RANK_ONE_TEMPLATE_FORMAT_1_23 = "RANK_ONE_1_23" - } - - private val maxFaces = 1 private val falseDetectionRate = 0.1f private val relativeMinSize = 0.2f private val absoluteMinSize = 36L @@ -35,7 +31,7 @@ class RocV1Detector @Inject constructor() : FaceDetector { reason = "This class uses roc class that has native functions and can't be mocked", ) data class ROCFace( - var face: roc_detection, + var face: roc_detection?, var template: SWIGTYPE_p_unsigned_char, var yaw: SWIGTYPE_p_float, var quality: SWIGTYPE_p_float, @@ -43,7 +39,7 @@ class RocV1Detector @Inject constructor() : FaceDetector { var gender: roc_embedded_gender?, ) { fun cleanup() { - face.delete() + face?.delete() roc.delete_uint8_t_array(template) roc.delete_float(yaw) roc.delete_float(quality) @@ -55,11 +51,13 @@ class RocV1Detector @Inject constructor() : FaceDetector { override fun spoofCheck( bitmap: Bitmap, configuredMaxSize: Int, + selectFace: FaceSelector?, ) = SpoofCheckResult(0f, SpoofCheckResult.SkipReason.NOT_AVAILABLE) override fun analyze( bitmap: Bitmap, estimateAgeAndGender: Boolean, + selectFace: FaceSelector?, ): Face? { val rocColorImage = roc_image() val rocGrayImage = roc_image() @@ -78,7 +76,7 @@ class RocV1Detector @Inject constructor() : FaceDetector { roc.roc_free_image(rocColorImage) - return analyze(rocGrayImage, bitmap.width, bitmap.height, estimateAgeAndGender) + return analyze(rocGrayImage, bitmap.width, bitmap.height, estimateAgeAndGender, selectFace) } /** @@ -89,9 +87,10 @@ class RocV1Detector @Inject constructor() : FaceDetector { imageWidth: Int, imageHeight: Int, estimateAgeAndGender: Boolean, + selectFace: FaceSelector?, ): Face? { val rocFace = ROCFace( - roc_detection(), + null, roc.new_uint8_t_array(roc.ROC_FAST_FV_SIZE.toInt()), roc.new_float(), roc.new_float(), @@ -99,9 +98,10 @@ class RocV1Detector @Inject constructor() : FaceDetector { if (estimateAgeAndGender) roc_embedded_gender() else null, ) - val faceDetected = getRocTemplateFromImage(rocImage, rocFace) + getRocTemplateFromImage(rocImage, rocFace, selectFace) + val detection = rocFace.face - if (!faceDetected) { + if (detection == null) { roc.roc_free_image(rocImage) rocFace.cleanup() return null @@ -116,14 +116,9 @@ class RocV1Detector @Inject constructor() : FaceDetector { val face = Face( imageWidth, imageHeight, - Rect( - (rocFace.face.x - rocFace.face.width / 2).toInt(), - (rocFace.face.y - rocFace.face.height / 2).toInt(), - (rocFace.face.x + rocFace.face.width / 2).toInt(), - (rocFace.face.y + rocFace.face.height / 2).toInt(), - ), + detection.boundingRect(), yawValue, - rocFace.face.rotation, + detection.rotation, qualityValue, roc.cdata(roc.roc_cast(rocFace.template), roc.ROC_FAST_FV_SIZE.toInt()), RANK_ONE_TEMPLATE_FORMAT_1_23, @@ -146,7 +141,8 @@ class RocV1Detector @Inject constructor() : FaceDetector { private fun getRocTemplateFromImage( image: roc_image, rocFace: ROCFace, - ): Boolean { + selectFace: FaceSelector?, + ) { val adaptiveMinimumSize = roc.new_size_t() roc.roc_ensure( roc.roc_adaptive_minimum_size( @@ -158,7 +154,9 @@ class RocV1Detector @Inject constructor() : FaceDetector { ), ) + val maxFaces = maxFacesFor(selectFace) val n = roc.new_size_t() + val detections = roc.new_roc_detection_array(maxFaces) roc.roc_ensure( roc.roc_embedded_error_to_string( @@ -168,16 +166,29 @@ class RocV1Detector @Inject constructor() : FaceDetector { maxFaces, falseDetectionRate, n, - rocFace.face, + detections, ), ), ) - if (roc.size_t_value(n) != 1L) { + val numFaces = roc.size_t_value(n).toInt().coerceIn(0, maxFaces) + val candidates = (0 until numFaces).map { roc.roc_detection_array_getitem(detections, it) } + val selectedIndex = when { + candidates.isEmpty() -> null + selectFace == null -> 0 + else -> selectFace(candidates.map { it.boundingRect() }) + } + + val detection = selectedIndex?.let { candidates.getOrNull(it) } + candidates.forEach { if (it !== detection) it.delete() } + roc.delete_roc_detection_array(detections) + + if (detection == null) { roc.delete_size_t(adaptiveMinimumSize) roc.delete_size_t(n) - return false + return } + rocFace.face = detection val landmarks = roc.new_roc_embedded_landmark_array(68) val rightEye = roc_embedded_landmark() @@ -187,7 +198,7 @@ class RocV1Detector @Inject constructor() : FaceDetector { roc.roc_embedded_error_to_string( roc.roc_embedded_landmark_face( image, - rocFace.face, + detection, landmarks, rightEye, leftEye, @@ -202,7 +213,7 @@ class RocV1Detector @Inject constructor() : FaceDetector { roc.roc_embedded_error_to_string( roc.roc_embedded_represent_face( image, - rocFace.face, + detection, rightEye, leftEye, chin, @@ -224,7 +235,20 @@ class RocV1Detector @Inject constructor() : FaceDetector { roc.delete_size_t(adaptiveMinimumSize) roc.delete_size_t(n) roc.delete_roc_embedded_landmark_array(landmarks) + } + + private fun roc_detection.boundingRect() = Rect( + (x - width / 2).toInt(), + (y - height / 2).toInt(), + (x + width / 2).toInt(), + (y + height / 2).toInt(), + ) - return true + // Detecting several faces is only useful when the caller can choose between them. + private fun maxFacesFor(selectFace: FaceSelector?) = if (selectFace == null) 1 else MAX_FACES + + companion object { + const val RANK_ONE_TEMPLATE_FORMAT_1_23 = "RANK_ONE_1_23" + const val MAX_FACES = 3 } } diff --git a/face/infra/roc-v3/src/main/java/com/simprints/face/infra/rocv3/detection/RocV3Detector.kt b/face/infra/roc-v3/src/main/java/com/simprints/face/infra/rocv3/detection/RocV3Detector.kt index 04db820d8e..af23251475 100644 --- a/face/infra/roc-v3/src/main/java/com/simprints/face/infra/rocv3/detection/RocV3Detector.kt +++ b/face/infra/roc-v3/src/main/java/com/simprints/face/infra/rocv3/detection/RocV3Detector.kt @@ -12,11 +12,13 @@ import androidx.core.graphics.scale import com.simprints.core.ExcludedFromGeneratedTestCoverageReports import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector +import com.simprints.face.infra.basebiosdk.detection.FaceSelector import com.simprints.face.infra.basebiosdk.detection.SpoofCheckResult import java.nio.ByteBuffer import javax.inject.Inject import javax.inject.Singleton import kotlin.math.abs +import kotlin.math.roundToInt import ai.roc.rocsdk.embedded.roc as roc3 @ExcludedFromGeneratedTestCoverageReports( @@ -27,6 +29,7 @@ class RocV3Detector @Inject constructor() : FaceDetector { override fun analyze( bitmap: Bitmap, estimateAgeAndGender: Boolean, + selectFace: FaceSelector?, ): Face? { val rocColorImage = roc_image() val rocGrayImage = roc_image() @@ -39,7 +42,7 @@ class RocV3Detector @Inject constructor() : FaceDetector { rocColorImage, ) roc3.roc_bgr2gray(rocColorImage, rocGrayImage) - return detectFace(rocColorImage, rocGrayImage, bitmap.width, bitmap.height, estimateAgeAndGender) + return detectFace(rocColorImage, rocGrayImage, bitmap.width, bitmap.height, estimateAgeAndGender, selectFace) } /* @@ -54,14 +57,28 @@ class RocV3Detector @Inject constructor() : FaceDetector { width: Int, height: Int, estimateAgeAndGender: Boolean, + selectFace: FaceSelector?, ): Face? { - val detection = roc_detection() + val maxFaces = maxFacesFor(selectFace) + val detections = roc3.new_roc_detection_array(maxFaces) val template = roc3.new_uint8_t_array(roc3.ROC_FACE_FAST_FV_SIZE.toInt()) val yaw = roc3.new_float() val quality = roc3.new_float() val age = if (estimateAgeAndGender) roc3.new_float() else null val gender = if (estimateAgeAndGender) roc_embedded_gender() else null - val face = if (isFaceDetected(coloredImage, detection)) { + + val numFaces = detectFaces(coloredImage, detections, maxFaces).toInt().coerceIn(0, maxFaces) + // SWIG copies each element out of the array, so all of these are owned by Java and are + // deleted below whether they end up being used or not. + val candidates = (0 until numFaces).map { roc3.roc_detection_array_getitem(detections, it) } + val selectedIndex = when { + candidates.isEmpty() -> null + selectFace == null -> 0 + else -> selectFace(candidates.map { it.boundingRect() }) + } + val detection = selectedIndex?.let { candidates.getOrNull(it) } + + val face = if (detection != null) { generateFaceTemplateFromImage( coloredImage, grayImage, @@ -103,7 +120,8 @@ class RocV3Detector @Inject constructor() : FaceDetector { age?.let { roc3.delete_float(it) } gender?.delete() roc3.delete_uint8_t_array(template) - detection.delete() + candidates.forEach { it.delete() } + roc3.delete_roc_detection_array(detections) return face } @@ -153,10 +171,17 @@ class RocV3Detector @Inject constructor() : FaceDetector { ) } - private fun isFaceDetected( + /** + * Fills [detections] with up to [maxFaces] faces and returns how many were found. + * + * [detections] must be an array of at least [maxFaces] elements - the native call writes + * straight into it. + */ + private fun detectFaces( image: roc_image, - detection: roc_detection, - ): Boolean { + detections: roc_detection, + maxFaces: Int, + ): Long { val adaptiveMinimumSize = roc3.new_size_t() roc3.roc_adaptive_minimum_size( @@ -171,18 +196,25 @@ class RocV3Detector @Inject constructor() : FaceDetector { roc3.roc_embedded_detect_faces_accurate( image, roc3.size_t_value(adaptiveMinimumSize), - MAX_FACE_DETECTION, + maxFaces, FALSE_DETECTION_RATE, n, - detection, + detections, ) val numFaces = roc3.size_t_value(n) roc3.delete_size_t(n) roc3.delete_size_t(adaptiveMinimumSize) - return numFaces == 1L + return numFaces } + private fun Rect.scaledBy(factor: Float) = Rect( + (left * factor).roundToInt(), + (top * factor).roundToInt(), + (right * factor).roundToInt(), + (bottom * factor).roundToInt(), + ) + private fun roc_detection.boundingRect() = Rect( (x - width / 2).toInt(), (y - height / 2).toInt(), @@ -199,6 +231,7 @@ class RocV3Detector @Inject constructor() : FaceDetector { override fun spoofCheck( bitmap: Bitmap, configuredMaxSize: Int, + selectFace: FaceSelector?, ): SpoofCheckResult { if (minOf(bitmap.width, bitmap.height) < SPOOF_MIN_SIZE) { // As per documentation - smallest dimension must be at least 720px @@ -206,8 +239,12 @@ class RocV3Detector @Inject constructor() : FaceDetector { } // Scaling image down to lower the chance that IOD is outside of requirements. The check also runs faster on smaller images. - val scaledBitmap = if (maxOf(bitmap.width, bitmap.height) > configuredMaxSize) { - val scale = configuredMaxSize.toFloat() / maxOf(bitmap.width, bitmap.height).toFloat() + val scale = if (maxOf(bitmap.width, bitmap.height) > configuredMaxSize) { + configuredMaxSize.toFloat() / maxOf(bitmap.width, bitmap.height).toFloat() + } else { + 1f + } + val scaledBitmap = if (scale < 1f) { bitmap.scale((bitmap.width * scale).toInt(), (bitmap.height * scale).toInt(), false) } else { bitmap @@ -225,13 +262,28 @@ class RocV3Detector @Inject constructor() : FaceDetector { ) roc3.roc_bgr2gray(rocColorImage, rocGrayImage) - val detection = roc_detection() - val faceDetected = isFaceDetected(rocColorImage, detection) + val maxFaces = maxFacesFor(selectFace) + val detections = roc3.new_roc_detection_array(maxFaces) + val numFaces = detectFaces(rocColorImage, detections, maxFaces).toInt().coerceIn(0, maxFaces) + + // The score has to describe the same person the template was taken from, so the caller's + // selector picks the face to measure here too. + // SWIG copies each element out of the array, so all of these are owned by Java and are + // deleted below whether they end up being used or not. + val candidates = (0 until numFaces).map { roc3.roc_detection_array_getitem(detections, it) } + val selectedIndex = when { + candidates.isEmpty() -> null + selectFace == null -> 0 + else -> selectFace(candidates.map { it.boundingRect().scaledBy(1f / scale) }) + } + val detection = selectedIndex?.let { candidates.getOrNull(it) } + + val faceDetected = detection != null var iod = 0f var finalScore = 0f - if (faceDetected) { + if (detection != null) { val rightEye = roc_landmark() val leftEye = roc_landmark() val chin = roc_landmark() @@ -296,8 +348,11 @@ class RocV3Detector @Inject constructor() : FaceDetector { } roc3.roc_free_image(rocGrayImage) roc3.roc_free_image(rocColorImage) - detection.delete() + candidates.forEach { it.delete() } + roc3.delete_roc_detection_array(detections) byteBuffer.clear() + // The scaled copy is this method's own; the bitmap the caller passed in is not + if (scaledBitmap !== bitmap) scaledBitmap.recycle() return when { !faceDetected -> SpoofCheckResult(finalScore, SpoofCheckResult.SkipReason.NOT_AVAILABLE) @@ -307,9 +362,12 @@ class RocV3Detector @Inject constructor() : FaceDetector { } } + // Detecting several faces is only useful when the caller can choose between them. + private fun maxFacesFor(selectFace: FaceSelector?) = if (selectFace == null) 1 else MAX_FACE_DETECTION + companion object { const val RANK_ONE_TEMPLATE_FORMAT_3_1 = "RANK_ONE_3_1" - const val MAX_FACE_DETECTION = 1 + const val MAX_FACE_DETECTION = 3 const val FALSE_DETECTION_RATE = 0.1f const val RELATIVE_MIN_SIZE = 0.2f const val ABSOLUTE_MIN_SIZE = 36L diff --git a/face/infra/simface/src/main/java/com/simprints/face/infra/simface/detection/SimFaceDetector.kt b/face/infra/simface/src/main/java/com/simprints/face/infra/simface/detection/SimFaceDetector.kt index 8085c2a434..fc52bca792 100644 --- a/face/infra/simface/src/main/java/com/simprints/face/infra/simface/detection/SimFaceDetector.kt +++ b/face/infra/simface/src/main/java/com/simprints/face/infra/simface/detection/SimFaceDetector.kt @@ -4,6 +4,7 @@ import android.graphics.Bitmap import com.simprints.biometrics.simface.SimFace import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector +import com.simprints.face.infra.basebiosdk.detection.FaceSelector import com.simprints.face.infra.basebiosdk.detection.SpoofCheckResult import kotlinx.coroutines.runBlocking import javax.inject.Inject @@ -18,10 +19,20 @@ class SimFaceDetector @Inject constructor( override fun analyze( bitmap: Bitmap, estimateAgeAndGender: Boolean, + selectFace: FaceSelector?, ): Face? = runBlocking { // Load a bitmap image for processing val faces = simFace.detectFaceBlocking(bitmap) - val face = faces.getOrNull(0) ?: return@runBlocking null + + // Detection is cheap here, so every face is offered to the caller and only the chosen + // one goes through the costly alignment and embedding below. + val selectedIndex = when { + faces.isEmpty() -> return@runBlocking null + selectFace == null -> 0 + else -> selectFace(faces.map { it.absoluteBoundingBox }) ?: return@runBlocking null + } + + val face = faces.getOrNull(selectedIndex) ?: return@runBlocking null // Skip the obviously bad images, but leave the rest to be determined by the caller if (face.quality < BAD_FACE_THRESHOLD) return@runBlocking null @@ -43,6 +54,7 @@ class SimFaceDetector @Inject constructor( override fun spoofCheck( bitmap: Bitmap, configuredMaxSize: Int, + selectFace: FaceSelector?, ) = SpoofCheckResult(0f, SpoofCheckResult.SkipReason.NOT_AVAILABLE) companion object { diff --git a/face/infra/simface/src/test/java/com/simprints/face/infra/simface/detection/SimFaceDetectorTest.kt b/face/infra/simface/src/test/java/com/simprints/face/infra/simface/detection/SimFaceDetectorTest.kt index 49184c45d8..c0b4c1b014 100644 --- a/face/infra/simface/src/test/java/com/simprints/face/infra/simface/detection/SimFaceDetectorTest.kt +++ b/face/infra/simface/src/test/java/com/simprints/face/infra/simface/detection/SimFaceDetectorTest.kt @@ -1,9 +1,11 @@ package com.simprints.face.infra.simface.detection import android.graphics.Bitmap +import android.graphics.Rect import com.google.common.truth.Truth.* import com.simprints.biometrics.simface.SimFace import com.simprints.biometrics.simface.data.FaceDetection +import com.simprints.face.infra.basebiosdk.detection.SpoofCheckResult import io.mockk.* import io.mockk.impl.annotations.MockK import kotlinx.coroutines.test.runTest @@ -56,4 +58,110 @@ class SimFaceDetectorTest { verify { simFace.getEmbedding(any()) } } + + @Test + fun `extracts the first face when no selector is supplied`() = runTest { + val first = face(quality = 0.8f) + detects(first, face(quality = 0.9f)) + + detector.analyze(image) + + verify(exactly = 1) { first.alignedFaceImage(image) } + } + + @Test + fun `extracts the face the selector chose`() = runTest { + val chosen = face(quality = 0.9f) + detects(face(quality = 0.8f), chosen, face(quality = 0.7f)) + + detector.analyze(image, selectFace = { 1 }) + + verify(exactly = 1) { chosen.alignedFaceImage(image) } + } + + @Test + fun `aligns and embeds the selected face alone`() = runTest { + val chosen = face(quality = 0.9f) + val ignored = listOf(face(quality = 0.8f), face(quality = 0.7f)) + detects(ignored[0], chosen, ignored[1]) + + detector.analyze(image, selectFace = { 1 }) + + ignored.forEach { verify(exactly = 0) { it.alignedFaceImage(any()) } } + verify(exactly = 1) { simFace.getEmbedding(any()) } + } + + @Test + fun `never consults the selector when no faces were detected`() = runTest { + coEvery { simFace.detectFaceBlocking(any()) } returns emptyList() + var consulted = false + + detector.analyze(image, selectFace = { + consulted = true + 0 + }) + + assertThat(consulted).isFalse() + } + + @Test + fun `extracts nothing when the selector chooses none`() = runTest { + detects(face(quality = 0.8f), face(quality = 0.9f)) + + assertThat(detector.analyze(image, selectFace = { null })).isNull() + verify(exactly = 0) { simFace.getEmbedding(any()) } + } + + @Test + fun `extracts nothing when the selector returns an out of range index`() = runTest { + detects(face(quality = 0.8f), face(quality = 0.9f)) + + assertThat(detector.analyze(image, selectFace = { 2 })).isNull() + verify(exactly = 0) { simFace.getEmbedding(any()) } + } + + @Test + fun `applies the quality gate to the selected face, not the first one`() = runTest { + detects(face(quality = 0.9f), face(quality = 0.0f)) + + assertThat(detector.analyze(image, selectFace = { 1 })).isNull() + } + + @Test + fun `spoof check is not available on simface`() = runTest { + val result = detector.spoofCheck(image, configuredMaxSize = 1000) + + assertThat(result.skipReason).isEqualTo(SpoofCheckResult.SkipReason.NOT_AVAILABLE) + assertThat(result.score).isEqualTo(0f) + } + + @Test + fun `reports no age or gender, which simface does not estimate`() = runTest { + detects(face(quality = 0.8f)) + + val face = detector.analyze(image, estimateAgeAndGender = true) + + assertThat(face?.age).isNull() + assertThat(face?.gender).isNull() + } + + private fun detects(vararg faces: FaceDetection) { + coEvery { simFace.detectFaceBlocking(any()) } returns faces.toList() + } + + private fun face( + quality: Float, + yaw: Float = 0f, + roll: Float = 0f, + boundingBox: Rect = box(), + ) = mockk(relaxed = true) { + every { this@mockk.quality } returns quality + every { this@mockk.yaw } returns yaw + every { this@mockk.roll } returns roll + every { absoluteBoundingBox } returns boundingBox + every { alignedFaceImage(any()) } returns image + } + + /** A distinct bounding box - only its identity matters, so it needs no real coordinates. */ + private fun box() = mockk() } From 808b180e26690d883302f97878285894b2bb856d Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 12:25:51 +0300 Subject: [PATCH 3/8] MS-1608 Add a tracking capture mode --- .../livefeedback/LiveFeedbackFragment.kt | 99 ++-- .../screens/livefeedback/LiveFeedbackState.kt | 27 + .../livefeedback/LiveFeedbackViewModel.kt | 184 ++++++- .../livefeedback/views/FaceTrackingOverlay.kt | 339 +++++++++++++ .../views/TrackedProgressPainter.kt | 117 +++++ .../usecases/CropToFaceSquareUseCase.kt | 55 ++ .../usecases/IsFaceTrackingEnabledUseCase.kt | 10 + .../usecases/SelectDominantFaceUseCase.kt | 50 ++ .../res/drawable/feedback_scrim_bottom.xml | 12 + .../layout-land/fragment_live_feedback.xml | 25 +- .../res/layout/fragment_live_feedback.xml | 54 +- .../livefeedback/LiveFeedbackViewModelTest.kt | 475 ++++++++++++++++-- .../usecases/CropToFaceSquareUseCaseTest.kt | 71 +++ .../usecases/SelectDominantFaceUseCaseTest.kt | 75 +++ .../core/tools/extensions/RectF.ext.kt | 22 + 15 files changed, 1483 insertions(+), 132 deletions(-) create mode 100644 face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt create mode 100644 face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/TrackedProgressPainter.kt create mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt create mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt create mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCase.kt create mode 100644 face/capture/src/main/res/drawable/feedback_scrim_bottom.xml create mode 100644 face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt create mode 100644 face/capture/src/test/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCaseTest.kt diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt index 674f314ee8..c0c9d0e566 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt @@ -4,7 +4,6 @@ import android.Manifest import android.content.Intent import android.os.Bundle import android.provider.Settings -import android.util.Size import android.view.View import androidx.activity.result.contract.ActivityResultContracts import androidx.annotation.StringRes @@ -43,8 +42,8 @@ import com.simprints.infra.uibase.viewbinding.viewBinding import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.distinctUntilChangedBy +import kotlinx.coroutines.flow.filter import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import javax.inject.Inject @@ -63,8 +62,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) private val vm: LiveFeedbackViewModel by viewModels() private val binding by viewBinding(FragmentLiveFeedbackBinding::bind) - private lateinit var screenSize: Size - @Inject lateinit var cameraFrameProvider: CameraFrameProvider @@ -81,7 +78,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) get() = ContextCompat.getColor(requireContext(), IDR.color.simprints_green_light) private val defaultCaptureProgressColor: Int get() = ContextCompat.getColor(requireContext(), IDR.color.simprints_blue_grey_light) - private val validationProgressColor: Int get() = ContextCompat.getColor(requireContext(), IDR.color.simprints_orange) @@ -103,7 +99,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } private fun initFragment() { - screenSize = with(resources.displayMetrics) { Size(widthPixels, widthPixels) } bindViewModel() bindPermissionActions() setUpFrameProcessing() @@ -159,19 +154,22 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } /** Initialize CameraX, and prepare to bind the camera use cases */ - private fun setUpCamera() = viewLifecycleOwner.lifecycleScope.launch { + private fun setUpCamera(state: LiveFeedbackState) = viewLifecycleOwner.lifecycleScope.launch { if (cameraFrameProvider.isInitialised()) { return@launch } + val isTracking = state.isFaceTrackingEnabled + // Wait for the views to be properly laid out binding.faceCaptureCamera.awaitLayout() - binding.captureOverlay.awaitLayout() + if (!isTracking) binding.captureOverlay.awaitLayout() cameraFrameProvider.initialiseCamera( lifecycleOwner = viewLifecycleOwner, cameraPreviewView = binding.faceCaptureCamera, - target = binding.captureOverlay.circleRect.toRect(), + // Tracking analyses the whole preview; otherwise only the fixed cutout is analysed + target = if (isTracking) null else binding.captureOverlay.circleRect.toRect(), ) Simber.i("Camera setup finished", tag = FACE_CAPTURE) } @@ -208,11 +206,11 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) launch { vm.state.collect(::render) } launch { vm.state - .map { it.permissionStatus } - .distinctUntilChanged() - .collect { permissionStatus -> - if (permissionStatus == PermissionStatus.Granted) { - setUpCamera() + .filter { it.stateInitialised } + .distinctUntilChangedBy { it.permissionStatus } + .collect { state -> + if (state.permissionStatus == PermissionStatus.Granted) { + setUpCamera(state) toggleCaptureButtonIfAutoCapture(true) } } @@ -268,48 +266,70 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } private fun render(state: LiveFeedbackState) { + // Which mode's views exist at all is only known once the configuration has been read, so + // both sets stay hidden until then rather than one being drawn and swapped for the other. + // Everything below is mode-independent and renders from the first state, so a slow config + // read still leaves a screen that can explain itself. + if (state.stateInitialised) { + applyCaptureMode(state.isFaceTrackingEnabled) + // The two modes own entirely separate views, so this is the only place they are told apart + if (state.isFaceTrackingEnabled) renderTrackingOverlay(state) else renderCutoutOverlay(state) + } + if (state.permissionStatus != PermissionStatus.Granted) { renderNoPermission() return } - renderProgress(state.progress) when (state.phase) { LiveFeedbackState.Phase.NOT_STARTED -> { - renderOverlay(overlayWhite = false, explanationVisible = false) + renderControls(explanationVisible = false) renderFeedbackOnButton(state) } LiveFeedbackState.Phase.CAPTURING -> { - renderOverlay(overlayWhite = true, explanationVisible = true) + renderControls(explanationVisible = true) renderFeedbackOnButton(state) } LiveFeedbackState.Phase.VALIDATING -> { - renderOverlay(overlayWhite = true, explanationVisible = true) + renderControls(explanationVisible = true) renderValidating() } LiveFeedbackState.Phase.VALIDATION_FAILED -> { - renderOverlay(overlayWhite = true, explanationVisible = true) + renderControls(explanationVisible = true) renderValidationFailed() } LiveFeedbackState.Phase.FINISHED -> onCaptureFinished(state.result) } } - private fun renderProgress(progress: Progress) = with(binding.captureProgress) { - value = progress.value - progressColor = when (progress.tint) { - Progress.Tint.DEFAULT -> defaultCaptureProgressColor - Progress.Tint.VALID -> validCaptureProgressColor - Progress.Tint.VALIDATION -> validationProgressColor - } - isInvisible = !progress.visible + /** + * Shows the views belonging to the active mode and hides the other mode's entirely, so nothing + * from the experimental path can be drawn while the flag is off. + */ + private fun applyCaptureMode(isFaceTrackingEnabled: Boolean) = with(binding) { + captureOverlay.isVisible = !isFaceTrackingEnabled + captureProgress.isVisible = !isFaceTrackingEnabled + faceTrackingOverlay.isVisible = isFaceTrackingEnabled + captureControlsScrim.isVisible = isFaceTrackingEnabled } - private fun renderOverlay( - overlayWhite: Boolean, - explanationVisible: Boolean, - ) = with(binding) { - if (overlayWhite) { + private fun renderTrackingOverlay(state: LiveFeedbackState) = with(binding) { + val hasPermission = state.permissionStatus == PermissionStatus.Granted + faceTrackingOverlay.update( + target = state.targetBox.takeIf { hasPermission }, + progress = if (hasPermission) state.progress else Progress.HIDDEN, + // Only useful while the operator is still framing the subject + showAimGuide = hasPermission && + ( + state.phase == LiveFeedbackState.Phase.NOT_STARTED || + state.phase == LiveFeedbackState.Phase.CAPTURING + ), + ) + } + + private fun renderCutoutOverlay(state: LiveFeedbackState) = with(binding) { + val dimPreview = state.permissionStatus == PermissionStatus.Granted && state.phase != LiveFeedbackState.Phase.NOT_STARTED + if (dimPreview) { captureOverlay.drawWhiteTarget() captureFeedbackTxtExplanation.setTextColor(ContextCompat.getColor(requireContext(), IDR.color.simprints_blue_grey)) } else { @@ -317,6 +337,16 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) captureFeedbackTxtExplanation.setTextColor(ContextCompat.getColor(requireContext(), IDR.color.simprints_text_white)) } + captureProgress.value = state.progress.value + captureProgress.progressColor = when (state.progress.tint) { + Progress.Tint.DEFAULT -> defaultCaptureProgressColor + Progress.Tint.VALID -> validCaptureProgressColor + Progress.Tint.VALIDATION -> validationProgressColor + } + captureProgress.isInvisible = !state.progress.visible + } + + private fun renderControls(explanationVisible: Boolean) = with(binding) { captureFeedbackTxtExplanation.isVisible = explanationVisible captureFeedbackBtn.isVisible = true captureFeedbackPermissionButton.isGone = true @@ -394,7 +424,6 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) } private fun renderValidating() = with(binding) { - captureOverlay.drawWhiteTarget() captureFeedbackBtn.setText(IDR.string.face_capture_title_validating) captureFeedbackBtn.setCheckedWithLeftDrawable(false) setManualCaptureButtonClickable(false) @@ -428,7 +457,7 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) private fun renderNoPermission() { binding.apply { - renderOverlay(overlayWhite = false, explanationVisible = true) + renderControls(explanationVisible = true) captureFeedbackTxtExplanation.setText(IDR.string.face_capture_permission_denied) captureFeedbackBtn.isGone = true captureFeedbackPermissionButton.isVisible = true diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt index 7f39d6f5ef..78a7dfdffa 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt @@ -1,5 +1,6 @@ package com.simprints.face.capture.screens.livefeedback +import android.graphics.RectF import com.simprints.core.domain.permission.PermissionStatus import com.simprints.face.capture.models.FaceDetection @@ -10,9 +11,12 @@ internal data class LiveFeedbackState( val phase: Phase, val feedback: Feedback, val isAutoCapture: Boolean, + val isFaceTrackingEnabled: Boolean, val permissionStatus: PermissionStatus, val progress: Progress, + val targetBox: FaceTargetBox? = null, // Tracking mode only: the square drawn around the subject. Always null for the cutout. val result: List = emptyList(), + val stateInitialised: Boolean = false, // Prevents camera init until the config values are available ) { /** Overall capture phase / state machine. */ enum class Phase { NOT_STARTED, CAPTURING, VALIDATING, VALIDATION_FAILED, FINISHED } @@ -28,6 +32,7 @@ internal data class LiveFeedbackState( phase = Phase.NOT_STARTED, feedback = Feedback.NONE, isAutoCapture = isAutoCapture, + isFaceTrackingEnabled = false, permissionStatus = PermissionStatus.Denied, progress = Progress.HIDDEN, ) @@ -56,3 +61,25 @@ internal fun FaceDetection.Status.toFeedback(): LiveFeedbackState.Feedback = whe FaceDetection.Status.TOOFAR -> LiveFeedbackState.Feedback.TOO_FAR FaceDetection.Status.BAD_QUALITY -> LiveFeedbackState.Feedback.BAD_QUALITY } + +/** + * The square drawn around the tracked face. + */ +internal data class FaceTargetBox( + val rect: RectF, + val tint: Tint, +) { + enum class Tint { INVALID, WARNING, VALID } +} + +/** + * Colour of the tracking square: + * - red when the face cannot be used at all, + * - yellow when it is usable but pose or quality is off, + * - green when every parameter matches. + */ +internal fun FaceDetection.Status.toTargetTint(): FaceTargetBox.Tint = when (this) { + FaceDetection.Status.VALID, FaceDetection.Status.VALID_CAPTURING -> FaceTargetBox.Tint.VALID + FaceDetection.Status.OFFYAW, FaceDetection.Status.OFFROLL, FaceDetection.Status.BAD_QUALITY -> FaceTargetBox.Tint.WARNING + FaceDetection.Status.TOOFAR, FaceDetection.Status.TOOCLOSE, FaceDetection.Status.NOFACE -> FaceTargetBox.Tint.INVALID +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt index 37da44e450..7c5c209a23 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt @@ -1,21 +1,28 @@ package com.simprints.face.capture.screens.livefeedback import android.graphics.Bitmap +import android.graphics.Rect import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import com.simprints.core.DispatcherBG import com.simprints.core.domain.permission.PermissionStatus import com.simprints.core.tools.extensions.area +import com.simprints.core.tools.extensions.normalisedIn +import com.simprints.core.tools.extensions.scaledTo import com.simprints.core.tools.time.TimeHelper import com.simprints.face.capture.models.FaceDetection import com.simprints.face.capture.models.FaceTarget import com.simprints.face.capture.models.SymmetricTarget import com.simprints.face.capture.screens.CaptureAttemptTracker +import com.simprints.face.capture.usecases.CropToFaceSquareUseCase import com.simprints.face.capture.usecases.GetSpoofCheckConfigurationUseCase +import com.simprints.face.capture.usecases.IsFaceTrackingEnabledUseCase import com.simprints.face.capture.usecases.IsUsingAutoCaptureUseCase +import com.simprints.face.capture.usecases.SelectDominantFaceUseCase import com.simprints.face.capture.usecases.SimpleCaptureEventReporter import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector +import com.simprints.face.infra.basebiosdk.detection.FaceSelector import com.simprints.face.infra.biosdkresolver.ResolveFaceBioSdkUseCase import com.simprints.infra.config.store.ConfigRepository import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_DEFAULT @@ -40,6 +47,8 @@ import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.util.concurrent.atomic.AtomicBoolean import javax.inject.Inject +import kotlin.math.max +import kotlin.math.min import kotlin.time.Duration.Companion.milliseconds import kotlin.time.measureTimedValue @@ -50,8 +59,11 @@ internal class LiveFeedbackViewModel @Inject constructor( private val eventReporter: SimpleCaptureEventReporter, private val timeHelper: TimeHelper, private val isUsingAutoCaptureUseCase: IsUsingAutoCaptureUseCase, + private val isFaceTrackingEnabledUseCase: IsFaceTrackingEnabledUseCase, private val getSpoofCheckConfiguration: GetSpoofCheckConfigurationUseCase, private val captureAttemptTracker: CaptureAttemptTracker, + private val cropToFaceSquare: CropToFaceSquareUseCase, + private val selectDominantFace: SelectDominantFaceUseCase, @param:DispatcherBG private val bgDispatcher: CoroutineDispatcher, ) : ViewModel() { private var samplesToCapture: Int = 1 @@ -87,6 +99,9 @@ internal class LiveFeedbackViewModel @Inject constructor( private var validationStartTime: Long = 0 var isAutoCaptureHeldOff = true private set + + private var isFaceTrackingEnabled = false + private var autoCaptureImagingTimeoutJob: Job? = null private var autoCaptureImagingDurationMillis: Long = FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_DEFAULT private lateinit var faceDetector: FaceDetector @@ -100,15 +115,20 @@ internal class LiveFeedbackViewModel @Inject constructor( feedback: LiveFeedbackState.Feedback = state.value.feedback, permissionStatus: PermissionStatus = state.value.permissionStatus, detectionForTint: FaceDetection? = null, + targetBox: FaceTargetBox? = state.value.targetBox, result: List = state.value.result, + stateInitialised: Boolean? = null, ) { state.update { currentState -> currentState.copy( + stateInitialised = stateInitialised ?: currentState.stateInitialised, phase = phase, feedback = feedback, isAutoCapture = isAutoCapture, + isFaceTrackingEnabled = isFaceTrackingEnabled, permissionStatus = permissionStatus, progress = computeProgress(phase, detectionForTint), + targetBox = targetBox, result = result, ) } @@ -116,12 +136,13 @@ internal class LiveFeedbackViewModel @Inject constructor( suspend fun initAutoCapture() { val config = configRepository.getProjectConfiguration() + isFaceTrackingEnabled = isFaceTrackingEnabledUseCase(config) isAutoCapture = isUsingAutoCaptureUseCase(config) if (isAutoCapture) { // Await until capture button is pressed holdOffAutoCapture() } - emit() // Reset UI state with correct auto-capture value + emit(stateInitialised = true) // Reset UI state with correct auto-capture value } fun onScreenResumed(permissionStatus: PermissionStatus) { @@ -173,7 +194,12 @@ internal class LiveFeedbackViewModel @Inject constructor( return // too late - imaging has already started } isAutoCaptureHeldOff = true - emit(phase = LiveFeedbackState.Phase.NOT_STARTED, feedback = LiveFeedbackState.Feedback.NONE) // reset view + // reset view + emit( + phase = LiveFeedbackState.Phase.NOT_STARTED, + feedback = LiveFeedbackState.Feedback.NONE, + targetBox = null, + ) } } @@ -204,6 +230,9 @@ internal class LiveFeedbackViewModel @Inject constructor( /** * Processes the image. Called on the CameraX analyzer executor (off the main thread). + * + * [frame] covers the whole visible preview: the face is detected anywhere within it, and the + * square around the detection is what gets cropped for the rest of the pipeline. */ fun process( originalBitmap: Bitmap, @@ -228,7 +257,18 @@ internal class LiveFeedbackViewModel @Inject constructor( } val captureStartTime = timeHelper.now() - val potentialFace = faceDetector.analyze(croppedBitmap) + val potentialFace = if (isFaceTrackingEnabled) { + // Detection covers the whole preview, so the subject is picked out of whatever is in + // frame and only that face has a template extracted for it + faceDetector.analyze(croppedBitmap) { faces -> + selectDominantFace(faces, croppedBitmap.width, croppedBitmap.height) + } + } else { + faceDetector.analyze(croppedBitmap) + } + val trackedSquare = trackedSquareFor(potentialFace, croppedBitmap) + val frameWidth = croppedBitmap.width + val frameHeight = croppedBitmap.height val faceDetection = getFaceDetectionFromPotentialFace(originalBitmap, croppedBitmap, potentialFace) faceDetection.detectionStartTime = captureStartTime @@ -261,14 +301,14 @@ internal class LiveFeedbackViewModel @Inject constructor( } when (newPhase) { - LiveFeedbackState.Phase.NOT_STARTED -> updateFallbackCaptureIfValid(faceDetection) + LiveFeedbackState.Phase.NOT_STARTED -> updateFallbackCaptureIfValid(faceDetection, trackedSquare) LiveFeedbackState.Phase.CAPTURING -> { if (isAutoCapture) { if (isQualifying(faceDetection)) { - updateUserCapturesWith(faceDetection) + updateUserCapturesWith(cropBitmapToTrackedSquare(faceDetection, trackedSquare)) } } else { - userCaptures.add(faceDetection) + userCaptures.add(cropBitmapToTrackedSquare(faceDetection, trackedSquare)) if (userCaptures.size == samplesToCapture) { finishCapture(captureAttemptTracker.attemptNumber) } @@ -279,7 +319,14 @@ internal class LiveFeedbackViewModel @Inject constructor( } } - emit(phase = newPhase, feedback = feedback, detectionForTint = faceDetection) + emit( + phase = newPhase, + feedback = feedback, + detectionForTint = faceDetection, + targetBox = trackedSquare?.let { + FaceTargetBox(it.normalisedIn(frameWidth, frameHeight), faceDetection.status.toTargetTint()) + }, + ) } private fun computeProgress( @@ -381,7 +428,14 @@ internal class LiveFeedbackViewModel @Inject constructor( val duration = measureTimedValue { for ((index, bitmap) in userCaptures.map { it.original }.withIndex()) { - val result = faceDetector.spoofCheck(bitmap, spoofCheckConfig.maxBitmapSize) + // Only face tracking allows more than one person in frame, so only it needs a + // policy for picking between them - cutout capture stays on the SDK's own choice + val selectFace: FaceSelector? = if (isFaceTrackingEnabled) { + { faces -> selectDominantFace(faces, bitmap.width, bitmap.height) } + } else { + null + } + val result = faceDetector.spoofCheck(bitmap, spoofCheckConfig.maxBitmapSize, selectFace) Simber.i("Spoof result: $result", tag = FACE_CAPTURE) userCaptures[index].spoofCheckResult = result } @@ -416,7 +470,25 @@ internal class LiveFeedbackViewModel @Inject constructor( Simber.i("Captures tracked in ${duration.duration}, waiting for ${delay}ms", tag = FACE_CAPTURE) if (delay > 0) delay(delay.milliseconds) - emit(phase = LiveFeedbackState.Phase.NOT_STARTED, feedback = LiveFeedbackState.Feedback.NONE, result = emptyList()) + emit( + phase = LiveFeedbackState.Phase.NOT_STARTED, + feedback = LiveFeedbackState.Feedback.NONE, + targetBox = null, + result = emptyList(), + ) + } + + /** + * Face tracking only: the square to crop and draw around the subject, in frame pixels. Null + * for cutout capture, where the frame handed in is already cropped to the on-screen target. + */ + private fun trackedSquareFor( + potentialFace: Face?, + frame: Bitmap, + ): Rect? { + if (!isFaceTrackingEnabled || potentialFace == null) return null + val faceBox = potentialFace.relativeBoundingBox.scaledTo(frame.width, frame.height) + return cropToFaceSquare.squareFor(faceBox, frame.width, frame.height).takeUnless { it.isEmpty } } private fun getFaceDetectionFromPotentialFace( @@ -442,40 +514,86 @@ internal class LiveFeedbackViewModel @Inject constructor( original: Bitmap, bitmap: Bitmap, potentialFace: Face, + ): FaceDetection = FaceDetection( + original = original, + bitmap = bitmap, + face = potentialFace, + status = if (isFaceTrackingEnabled) trackedFaceStatus(potentialFace, bitmap) else cutoutFaceStatus(potentialFace), + detectionStartTime = timeHelper.now(), + detectionEndTime = timeHelper.now(), + ) + + /** + * Cropping costs an allocation and a rescale on the analyzer thread, so it is + * done here rather than for every analysed frame - the overwhelming majority are discarded + * straight after their status is read. + */ + private fun cropBitmapToTrackedSquare( + faceDetection: FaceDetection, + trackedSquare: Rect?, ): FaceDetection { + if (trackedSquare == null) return faceDetection + val crop = cropToFaceSquare(faceDetection.bitmap, trackedSquare) + // The use case hands the frame straight back when the square is unusable + if (crop === faceDetection.bitmap) return faceDetection + faceDetection.bitmap.recycle() + return faceDetection.copy(bitmap = crop) + } + + /** Cutout capture judges distance by how much of the fixed on-screen target the face fills. */ + private fun cutoutFaceStatus(potentialFace: Face): FaceDetection.Status { val areaOccupied = potentialFace.relativeBoundingBox.area() - val status = when { + return when { areaOccupied < faceTarget.areaRange.start -> FaceDetection.Status.TOOFAR areaOccupied > faceTarget.areaRange.endInclusive -> FaceDetection.Status.TOOCLOSE - potentialFace.yaw !in faceTarget.yawTarget -> FaceDetection.Status.OFFYAW - potentialFace.roll !in faceTarget.rollTarget -> FaceDetection.Status.OFFROLL - potentialFace.quality < qualityThreshold -> FaceDetection.Status.BAD_QUALITY - phase == LiveFeedbackState.Phase.CAPTURING -> FaceDetection.Status.VALID_CAPTURING - else -> FaceDetection.Status.VALID + else -> poseAndQualityStatus(potentialFace) } + } - return FaceDetection( - original = original, - bitmap = bitmap, - face = potentialFace, - status = status, - detectionStartTime = timeHelper.now(), - detectionEndTime = timeHelper.now(), - ) + /** + * Face tracking judges distance by the face's own pixel size, since it can sit anywhere in the + * preview. Those are the pixels the SDK actually saw, which is what governs whether a usable + * template can be extracted from it. + */ + private fun trackedFaceStatus( + potentialFace: Face, + frame: Bitmap, + ): FaceDetection.Status { + val faceBox = potentialFace.relativeBoundingBox.scaledTo(frame.width, frame.height) + val detectedSide = max(faceBox.width(), faceBox.height()) + val frameSide = min(frame.width, frame.height) + return when { + detectedSide < frameSide * MIN_FACE_FRAME_RATIO -> FaceDetection.Status.TOOFAR + detectedSide > frameSide * MAX_FACE_FRAME_RATIO -> FaceDetection.Status.TOOCLOSE + else -> poseAndQualityStatus(potentialFace) + } + } + + /** Everything the two capture modes judge the same way, once distance is settled. */ + private fun poseAndQualityStatus(potentialFace: Face): FaceDetection.Status = when { + potentialFace.yaw !in faceTarget.yawTarget -> FaceDetection.Status.OFFYAW + potentialFace.roll !in faceTarget.rollTarget -> FaceDetection.Status.OFFROLL + potentialFace.quality < qualityThreshold -> FaceDetection.Status.BAD_QUALITY + phase == LiveFeedbackState.Phase.CAPTURING -> FaceDetection.Status.VALID_CAPTURING + else -> FaceDetection.Status.VALID } /** * While the user has not started the capture flow, we save fallback images. If the capture doesn't * get any good images, at least one good image will be saved */ - private fun updateFallbackCaptureIfValid(faceDetection: FaceDetection) { + private fun updateFallbackCaptureIfValid( + faceDetection: FaceDetection, + trackedSquare: Rect?, + ) { val fallbackQuality = fallbackCapture?.face?.quality ?: -1f // To ensure that detection is better with defaults val detectionQuality = faceDetection.face?.quality ?: 0f if (faceDetection.hasValidStatus() && detectionQuality >= fallbackQuality) { Simber.i("Fallback capture updated", tag = FACE_CAPTURE) - fallbackCapture = faceDetection.apply { isFallback = true } - createFirstFallbackCaptureEvent(faceDetection) + val kept = cropBitmapToTrackedSquare(faceDetection, trackedSquare).apply { isFallback = true } + fallbackCapture = kept + createFirstFallbackCaptureEvent(kept) } } @@ -527,6 +645,20 @@ internal class LiveFeedbackViewModel @Inject constructor( companion object { private const val VALID_ROLL_DELTA = 15f private const val VALID_YAW_DELTA = 30f + + /** + * How big the face has to be for face tracking, as a fraction of the preview's shorter + * edge. Chosen so both capture modes ask for a face of roughly the same size. + * + * The cutout wants the face to fill 20-50% of its target's area, which is 45-71% of that + * target's side; the target is in turn 90% of the preview's shorter edge, so the same face + * measures about 40-64% of the whole preview. + * + * Expressed as proportions rather than pixels because the analyser resolution follows the + * preview size - a fixed pixel band would mean a different thing on every device. + */ + private const val MIN_FACE_FRAME_RATIO = 0.40f + private const val MAX_FACE_FRAME_RATIO = 0.65f } enum class PermissionAction { diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt new file mode 100644 index 0000000000..d946401f9b --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt @@ -0,0 +1,339 @@ +package com.simprints.face.capture.screens.livefeedback.views + +import android.content.Context +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Path +import android.graphics.RectF +import android.util.AttributeSet +import android.view.View +import android.view.animation.AnimationUtils +import androidx.annotation.ColorInt +import androidx.core.content.ContextCompat +import com.simprints.core.tools.extensions.dpToPx +import com.simprints.face.capture.screens.livefeedback.FaceTargetBox +import com.simprints.face.capture.screens.livefeedback.Progress +import com.simprints.infra.uibase.annotations.ExcludedFromGeneratedTestCoverageReports +import kotlin.math.abs +import kotlin.math.exp +import kotlin.math.min +import com.simprints.infra.resources.R as IDR + +/** + * Draws the live face-tracking feedback over the whole camera preview: a colour-coded square + * around the detected face, plus the capture progress traced along that square's contour so it + * follows the face instead of a fixed screen cutout. + * + * Nothing is dimmed - the preview stays fully visible and the square colour carries the state. + * + * ### Smoothing + * Detections only arrive as fast as the SDK can analyse a frame, which is far slower than the + * display refreshes, so drawing each one directly makes the square step from position to position. + * Instead the square glides toward the latest detection, redrawing itself between detections until + * it catches up. How hard it is smoothed depends on how far behind it is, so that a still face + * gets a calm square and a moving one gets a square that stays on it - see [timeConstantFor]. + * + * This is presentation only. The view never reports geometry back, so the crop handed to the rest + * of the pipeline still comes from the unsmoothed detection and never lags behind the face. The + * tint is likewise applied immediately rather than glided, so colour never describes a position + * the face has already left. + */ +@ExcludedFromGeneratedTestCoverageReports("UI code") +internal class FaceTrackingOverlay @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0, +) : View(context, attrs, defStyleAttr) { + private val boxStrokeWidth = 3f.dpToPx(context) + private val boxCornerRadius = 8f.dpToPx(context) + private val progressStrokeWidth = 8f.dpToPx(context) + + /** Clears the box outline so the two strokes sit side by side instead of overlapping. */ + private val progressOutset = (boxStrokeWidth + progressStrokeWidth) / 2f + 4f.dpToPx(context) + + /** Below this the remaining distance is not worth another frame, so the glide ends. */ + private val settleThreshold = 0.5f.dpToPx(context) + + /** Up to here the square is assumed to be chasing detector noise rather than a moving face. */ + private val jitterDistance = 2f.dpToPx(context) + + /** From here on the face is genuinely moving and the square should keep up with it. */ + private val motionDistance = 16f.dpToPx(context) + + private val boxPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + strokeWidth = boxStrokeWidth + strokeCap = Paint.Cap.ROUND + strokeJoin = Paint.Join.ROUND + } + + private val progressPainter = TrackedProgressPainter( + progressColor = Color.WHITE, + strokeWidth = progressStrokeWidth, + dashWidth = 3f.dpToPx(context), + dashSpace = 1f.dpToPx(context), + ) + + private val aimGuidePaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + strokeWidth = 2f.dpToPx(context) + strokeCap = Paint.Cap.ROUND + strokeJoin = Paint.Join.ROUND + color = AIM_GUIDE_COLOR + } + + /** Corner brackets marking where the subject's face should go. Rebuilt only on resize. */ + private val aimGuidePath = Path() + + /** Latest detection in view pixels: where the square is heading. */ + private val targetRect = RectF() + + /** Where the square actually is this frame. */ + private val displayedRect = RectF() + private var hasDisplayedRect = false + private var lastDrawTimeMs = 0L + + private val drawRect = RectF() + private val progressBounds = RectF() + + private val invalidColor by lazy { color(IDR.color.simprints_red) } + private val warningColor by lazy { color(IDR.color.simprints_yellow) } + private val validColor by lazy { color(IDR.color.simprints_green_light) } + private val defaultProgressColor by lazy { color(IDR.color.simprints_blue_grey_light) } + private val validProgressColor by lazy { color(IDR.color.simprints_green_light) } + private val validationProgressColor by lazy { color(IDR.color.simprints_orange) } + + private var target: FaceTargetBox? = null + private var progress: Progress = Progress.HIDDEN + private var showAimGuide: Boolean = false + + init { + setBackgroundColor(Color.TRANSPARENT) + } + + /** + * Renders one frame of feedback. A null [target] means no face is currently tracked. + * + * [showAimGuide] draws the corner brackets that tell the operator where to put the subject. + */ + fun update( + target: FaceTargetBox?, + progress: Progress, + showAimGuide: Boolean, + ) { + if (this.target == target && this.progress == progress && this.showAimGuide == showAimGuide) return + this.showAimGuide = showAimGuide + + if (target == null) { + // Nothing to glide from once the face is gone; the next one appears where it is + hasDisplayedRect = false + } else if (this.target?.rect != target.rect) { + // Measure the first step of the glide from now rather than from the last frame drawn, + // which may have been a whole detection ago + lastDrawTimeMs = AnimationUtils.currentAnimationTimeMillis() + } + + this.target = target + this.progress = progress + invalidate() + } + + override fun onSizeChanged( + w: Int, + h: Int, + oldw: Int, + oldh: Int, + ) { + super.onSizeChanged(w, h, oldw, oldh) + // View pixels mean something different now, so start again rather than glide across + hasDisplayedRect = false + rebuildAimGuide() + } + + override fun onDraw(canvas: Canvas) { + super.onDraw(canvas) + if (width == 0 || height == 0) return + + if (showAimGuide) canvas.drawPath(aimGuidePath, aimGuidePaint) + + val box = target + val isGliding = box != null && advanceTowards(box.rect) + + if (box != null) { + boxPaint.color = colorFor(box.tint) + drawRect.set(displayedRect) + // Inset so the outline is never half-clipped against the edge of the preview + drawRect.inset(boxStrokeWidth / 2f, boxStrokeWidth / 2f) + canvas.drawRoundRect(drawRect, boxCornerRadius, boxCornerRadius, boxPaint) + } + + if (progress.visible) { + progressPainter.progressColor = colorFor(progress.tint) + progressPainter.setValue(progress.value) + progressPainter.setContour( + progressContourAround(displayedRect.takeIf { box != null }), + boxCornerRadius + progressOutset, + ) + progressPainter.draw(canvas) + } + + if (isGliding) postInvalidateOnAnimation() + } + + /** + * Lays out the aim guide: four corner brackets around a centred square, which read as "put the + * face here" without walling off the rest of the preview the way a solid frame would. The + * guide is advisory - a face outside it is still captured if it is clearly the dominant one. + */ + private fun rebuildAimGuide() { + aimGuidePath.rewind() + if (width == 0 || height == 0) return + + val half = min(width, height) * AIM_GUIDE_SIZE_RATIO / 2f + val left = width / 2f - half + val top = height / 2f - half + val right = width / 2f + half + val bottom = height / 2f + half + val arm = half * 2f * AIM_GUIDE_ARM_RATIO + + with(aimGuidePath) { + moveTo(left, top + arm) + lineTo(left, top) + lineTo(left + arm, top) + + moveTo(right - arm, top) + lineTo(right, top) + lineTo(right, top + arm) + + moveTo(right, bottom - arm) + lineTo(right, bottom) + lineTo(right - arm, bottom) + + moveTo(left + arm, bottom) + lineTo(left, bottom) + lineTo(left, bottom - arm) + } + } + + /** + * Moves [displayedRect] one time-proportional step toward the normalised [normalisedTarget]. + * + * @return true while the square is still short of its target and needs another frame + */ + private fun advanceTowards(normalisedTarget: RectF): Boolean { + targetRect.set( + normalisedTarget.left * width, + normalisedTarget.top * height, + normalisedTarget.right * width, + normalisedTarget.bottom * height, + ) + + if (!hasDisplayedRect) { + displayedRect.set(targetRect) + hasDisplayedRect = true + lastDrawTimeMs = AnimationUtils.currentAnimationTimeMillis() + return false + } + + val distance = distanceToTarget() + if (distance < settleThreshold) { + displayedRect.set(targetRect) + lastDrawTimeMs = AnimationUtils.currentAnimationTimeMillis() + return false + } + + val now = AnimationUtils.currentAnimationTimeMillis() + // Driven by elapsed time so the glide takes the same wall-clock time on any refresh rate. + // Capped so a spell with no frames drawn resolves over a few frames instead of snapping. + val deltaMs = (now - lastDrawTimeMs).coerceIn(0L, MAX_FRAME_DELTA_MS) + lastDrawTimeMs = now + + val progressTowardsTarget = 1f - exp(-deltaMs.toFloat() / timeConstantFor(distance)) + displayedRect.set( + lerp(displayedRect.left, targetRect.left, progressTowardsTarget), + lerp(displayedRect.top, targetRect.top, progressTowardsTarget), + lerp(displayedRect.right, targetRect.right, progressTowardsTarget), + lerp(displayedRect.bottom, targetRect.bottom, progressTowardsTarget), + ) + + return true + } + + /** How far the drawn square still has to travel, as its furthest-off edge. */ + private fun distanceToTarget(): Float = maxOf( + abs(displayedRect.left - targetRect.left), + abs(displayedRect.top - targetRect.top), + abs(displayedRect.right - targetRect.right), + abs(displayedRect.bottom - targetRect.bottom), + ) + + /** + * Picks how hard to smooth, based on how far behind the square currently is. + */ + private fun timeConstantFor(distance: Float): Float { + val towardsMotion = ((distance - jitterDistance) / (motionDistance - jitterDistance)).coerceIn(0f, 1f) + return lerp(GLIDE_TIME_CONSTANT_STILL_MS, GLIDE_TIME_CONSTANT_MOVING_MS, towardsMotion) + } + + private fun lerp( + from: Float, + to: Float, + fraction: Float, + ) = from + (to - from) * fraction + + /** + * The progress traces the square itself, just outside its outline. Without a tracked face it + * falls back to a centred square so progress stays visible between detections. + */ + private fun progressContourAround(box: RectF?): RectF { + if (box == null) { + val half = min(width, height) * FALLBACK_PROGRESS_SIZE_RATIO / 2f + progressBounds.set(width / 2f - half, height / 2f - half, width / 2f + half, height / 2f + half) + } else { + progressBounds.set(box) + progressBounds.inset(-progressOutset, -progressOutset) + } + return progressBounds + } + + @ColorInt + private fun colorFor(tint: FaceTargetBox.Tint): Int = when (tint) { + FaceTargetBox.Tint.INVALID -> invalidColor + FaceTargetBox.Tint.WARNING -> warningColor + FaceTargetBox.Tint.VALID -> validColor + } + + @ColorInt + private fun colorFor(tint: Progress.Tint): Int = when (tint) { + Progress.Tint.DEFAULT -> defaultProgressColor + Progress.Tint.VALID -> validProgressColor + Progress.Tint.VALIDATION -> validationProgressColor + } + + @ColorInt + private fun color(resId: Int) = ContextCompat.getColor(context, resId) + + companion object { + private const val FALLBACK_PROGRESS_SIZE_RATIO = 0.7f + + /** Side of the aim guide as a fraction of the preview's shorter edge. */ + private const val AIM_GUIDE_SIZE_RATIO = 0.62f + + /** Length of each bracket arm as a fraction of the guide's side. */ + private const val AIM_GUIDE_ARM_RATIO = 0.18f + + /** Faint enough to read as guidance rather than compete with the tracking square. */ + private const val AIM_GUIDE_COLOR = 0x80FFFFFF.toInt() + + /** + * Time to cover ~63% of the remaining distance. Raise the first to calm a shimmering + * square at rest; lower the second if it still trails a moving face. + */ + private const val GLIDE_TIME_CONSTANT_STILL_MS = 250f + private const val GLIDE_TIME_CONSTANT_MOVING_MS = 15f + + /** Roughly four 60Hz frames, so a long gap between draws still resolves quickly. */ + private const val MAX_FRAME_DELTA_MS = 64L + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/TrackedProgressPainter.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/TrackedProgressPainter.kt new file mode 100644 index 0000000000..08c8aadff7 --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/TrackedProgressPainter.kt @@ -0,0 +1,117 @@ +package com.simprints.face.capture.screens.livefeedback.views + +import android.graphics.Canvas +import android.graphics.DashPathEffect +import android.graphics.Paint +import android.graphics.Path +import android.graphics.PathMeasure +import android.graphics.RectF +import com.simprints.infra.uibase.annotations.ExcludedFromGeneratedTestCoverageReports + +/** + * Paints capture progress along the contour of the square tracking a face, so the progress reads + * as that square filling in rather than as a separate widget. + * + * The cutout capture keeps its own [ProgressPainter], which draws an arc on a fixed circle. This + * one exists because a square that moves with the face needs a path it can trace, not an arc. + * + * The contour starts at the top centre and runs clockwise. + */ +@ExcludedFromGeneratedTestCoverageReports("UI code") +internal class TrackedProgressPainter( + progressColor: Int, + strokeWidth: Float, + dashWidth: Float, + dashSpace: Float, +) { + private val contour = Path() + private val progressPath = Path() + private val pathMeasure = PathMeasure() + private val bounds = RectF() + private val corner = RectF() + + private var cornerRadius = 0f + private var contourLength = 0f + private var value = 0f + private var isProgressPathStale = true + + private val progressPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply { + this.strokeWidth = strokeWidth + color = progressColor + style = Paint.Style.STROKE + pathEffect = DashPathEffect(floatArrayOf(dashWidth, dashSpace), dashSpace) + } + + var progressColor = progressColor + set(value) { + field = value + progressPaint.color = value + } + + fun draw(canvas: Canvas) { + if (contourLength <= 0f || value <= 0f) return + if (isProgressPathStale) rebuildProgressPath() + canvas.drawPath(progressPath, progressPaint) + } + + /** Positions the outline. [rect] is expected to be square, but any rect is handled. */ + fun setContour( + rect: RectF, + cornerRadius: Float, + ) { + if (bounds == rect && this.cornerRadius == cornerRadius) return + bounds.set(rect) + this.cornerRadius = cornerRadius + rebuildContour() + isProgressPathStale = true + } + + fun setValue(value: Float) { + val coerced = value.coerceIn(0f, 1f) + if (this.value == coerced) return + this.value = coerced + isProgressPathStale = true + } + + private fun rebuildContour() { + contour.rewind() + contourLength = 0f + if (bounds.isEmpty) return + + val radius = cornerRadius.coerceAtMost(minOf(bounds.width(), bounds.height()) / 2f) + val diameter = radius * 2f + with(contour) { + moveTo(bounds.centerX(), bounds.top) + lineTo(bounds.right - radius, bounds.top) + corner.set(bounds.right - diameter, bounds.top, bounds.right, bounds.top + diameter) + arcTo(corner, TOP, QUARTER_TURN, false) + lineTo(bounds.right, bounds.bottom - radius) + corner.set(bounds.right - diameter, bounds.bottom - diameter, bounds.right, bounds.bottom) + arcTo(corner, RIGHT, QUARTER_TURN, false) + lineTo(bounds.left + radius, bounds.bottom) + corner.set(bounds.left, bounds.bottom - diameter, bounds.left + diameter, bounds.bottom) + arcTo(corner, BOTTOM, QUARTER_TURN, false) + lineTo(bounds.left, bounds.top + radius) + corner.set(bounds.left, bounds.top, bounds.left + diameter, bounds.top + diameter) + arcTo(corner, LEFT, QUARTER_TURN, false) + lineTo(bounds.centerX(), bounds.top) + } + + pathMeasure.setPath(contour, false) + contourLength = pathMeasure.length + } + + private fun rebuildProgressPath() { + progressPath.rewind() + pathMeasure.getSegment(0f, contourLength * value, progressPath, true) + isProgressPathStale = false + } + + companion object { + private const val QUARTER_TURN = 90f + private const val TOP = 270f + private const val RIGHT = 0f + private const val BOTTOM = 90f + private const val LEFT = 180f + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt new file mode 100644 index 0000000000..e3c767ef2f --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt @@ -0,0 +1,55 @@ +package com.simprints.face.capture.usecases + +import android.graphics.Bitmap +import android.graphics.Rect +import android.graphics.RectF +import javax.inject.Inject +import kotlin.math.max +import kotlin.math.min +import kotlin.math.roundToInt + +/** + * Turns a face detection bounding box into the square region that is both drawn on the preview and + * fed to the rest of the capture pipeline. + * + * The square is centred on the detection and its side is the longer side of the detection, but it + * is always kept fully inside the frame: the side is first capped at the frame's shorter dimension + * and the square is then shifted inwards until it fits. + */ +internal class CropToFaceSquareUseCase @Inject constructor() { + /** + * @param faceBox detection bounding box in frame pixels + * @return the square to draw and crop, in frame pixels, or an empty rect if it cannot be built + */ + fun squareFor( + faceBox: RectF, + frameWidth: Int, + frameHeight: Int, + ): Rect { + if (frameWidth <= 0 || frameHeight <= 0) return Rect() + + val side = max(faceBox.width(), faceBox.height()) + .roundToInt() + .coerceAtMost(min(frameWidth, frameHeight)) + if (side <= 0) return Rect() + + val left = (faceBox.centerX() - side / 2f).roundToInt().coerceIn(0, frameWidth - side) + val top = (faceBox.centerY() - side / 2f).roundToInt().coerceIn(0, frameHeight - side) + + return Rect(left, top, left + side, top + side) + } + + /** + * Crops [frame] to [square]. Returns [frame] untouched if the square does not describe a + * usable sub-region, so callers always get a bitmap back. + */ + operator fun invoke( + frame: Bitmap, + square: Rect, + ): Bitmap { + if (square.isEmpty || square.width() > frame.width || square.height() > frame.height) { + return frame + } + return Bitmap.createBitmap(frame, square.left, square.top, square.width(), square.height()) + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt new file mode 100644 index 0000000000..8ad8a1a5fb --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt @@ -0,0 +1,10 @@ +package com.simprints.face.capture.usecases + +import com.simprints.infra.config.store.models.ProjectConfiguration +import com.simprints.infra.config.store.models.experimental +import javax.inject.Inject + +internal class IsFaceTrackingEnabledUseCase @Inject constructor() { + operator fun invoke(projectConfiguration: ProjectConfiguration): Boolean = + projectConfiguration.experimental().faceTrackingCaptureEnabled +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCase.kt new file mode 100644 index 0000000000..b2cebd1a85 --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCase.kt @@ -0,0 +1,50 @@ +package com.simprints.face.capture.usecases + +import android.graphics.Rect +import javax.inject.Inject +import kotlin.math.hypot +import kotlin.math.pow + +/** + * Decides which of the detected faces is the subject being captured. + * + * The operator aims the camera at the person they are enrolling, so the subject is the face that + * is both large and near the middle of the frame - which is what the aim guide drawn on the + * preview asks them to do. Faces are therefore scored on the area they occupy, weighted by how + * central they are, and the highest score wins. + */ +internal class SelectDominantFaceUseCase @Inject constructor() { + /** + * @param faces detected bounding boxes in frame pixels + * @return index of the subject within [faces], or null if there was nothing to choose from + */ + operator fun invoke( + faces: List, + frameWidth: Int, + frameHeight: Int, + ): Int? { + if (faces.isEmpty() || frameWidth <= 0 || frameHeight <= 0) return null + return faces.indices.maxByOrNull { score(faces[it], frameWidth, frameHeight) } + } + + private fun score( + face: Rect, + frameWidth: Int, + frameHeight: Int, + ): Float { + val area = face.width().toFloat() * face.height() / (frameWidth.toFloat() * frameHeight) + val centreX = frameWidth / 2f + val centreY = frameHeight / 2f + val offset = hypot(face.exactCenterX() - centreX, face.exactCenterY() - centreY) + val centrality = (1f - offset / hypot(centreX, centreY)).coerceIn(0f, 1f) + return area * centrality.pow(CENTRALITY_WEIGHT) + } + + companion object { + /** + * Above 1 this pulls the choice toward the middle of the frame rather than raw size, so a + * bystander who happens to be closer to the lens does not outrank the person being aimed at. + */ + private const val CENTRALITY_WEIGHT = 2f + } +} diff --git a/face/capture/src/main/res/drawable/feedback_scrim_bottom.xml b/face/capture/src/main/res/drawable/feedback_scrim_bottom.xml new file mode 100644 index 0000000000..5e28d77ad5 --- /dev/null +++ b/face/capture/src/main/res/drawable/feedback_scrim_bottom.xml @@ -0,0 +1,12 @@ + + + + + + diff --git a/face/capture/src/main/res/layout-land/fragment_live_feedback.xml b/face/capture/src/main/res/layout-land/fragment_live_feedback.xml index 3c684b0c22..f221a5274e 100644 --- a/face/capture/src/main/res/layout-land/fragment_live_feedback.xml +++ b/face/capture/src/main/res/layout-land/fragment_live_feedback.xml @@ -35,7 +35,9 @@ + android:layout_height="match_parent" + android:visibility="invisible" + tools:visibility="visible" /> + + + + + + android:keepScreenOn="true" > + + + - - + + + + + app:layout_constraintStart_toStartOf="parent" /> diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index 0d4d9ed26c..a0f0f58fb1 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -9,11 +9,15 @@ import com.simprints.core.domain.permission.PermissionStatus import com.simprints.core.tools.time.TimeHelper import com.simprints.core.tools.time.Timestamp import com.simprints.face.capture.screens.CaptureAttemptTracker +import com.simprints.face.capture.usecases.CropToFaceSquareUseCase import com.simprints.face.capture.usecases.GetSpoofCheckConfigurationUseCase +import com.simprints.face.capture.usecases.IsFaceTrackingEnabledUseCase import com.simprints.face.capture.usecases.IsUsingAutoCaptureUseCase +import com.simprints.face.capture.usecases.SelectDominantFaceUseCase import com.simprints.face.capture.usecases.SimpleCaptureEventReporter import com.simprints.face.infra.basebiosdk.detection.Face import com.simprints.face.infra.basebiosdk.detection.FaceDetector +import com.simprints.face.infra.basebiosdk.detection.FaceSelector import com.simprints.face.infra.basebiosdk.detection.SpoofCheckResult import com.simprints.face.infra.biosdkresolver.ResolveFaceBioSdkUseCase import com.simprints.infra.config.store.ConfigRepository @@ -24,6 +28,7 @@ import com.simprints.testtools.common.coroutines.TestCoroutineRule import io.mockk.* import io.mockk.impl.annotations.MockK import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.toList import kotlinx.coroutines.launch import kotlinx.coroutines.test.TestScope @@ -77,13 +82,25 @@ internal class LiveFeedbackViewModelTest { @MockK private lateinit var getSpoofCheckConfiguration: GetSpoofCheckConfigurationUseCase + @MockK + private lateinit var isFaceTrackingEnabled: IsFaceTrackingEnabledUseCase + private lateinit var viewModel: LiveFeedbackViewModel private val testCaptureAttemptTracker = CaptureAttemptTracker() + /** + * The square geometry is exercised for real so the status rules are genuinely tested; only the + * bitmap cropping itself is stubbed out, since [frame] is a mock. + */ + private val cropToFaceSquare = spyk(CropToFaceSquareUseCase()) + @Before fun setUp() { MockKAnnotations.init(this, relaxed = true) + every { frame.width } returns FRAME_SIZE_PX + every { frame.height } returns FRAME_SIZE_PX + every { cropToFaceSquare.invoke(any(), any()) } returns frame coEvery { resolveFaceBioSdkUseCase.invoke(any()) } returns mockk { every { detector } returns faceDetector } @@ -95,6 +112,8 @@ internal class LiveFeedbackViewModelTest { ?.qualityThreshold } returns QUALITY_THRESHOLD every { isUsingAutoCapture.invoke(any()) } returns false + // The flag is off by default, so the bulk of the suite covers the standard cutout capture + every { isFaceTrackingEnabled.invoke(any()) } returns false every { getSpoofCheckConfiguration.invoke(any(), any()) } returns SpoofCheckConfiguration.DISABLED every { timeHelper.now() } returnsMany (0..1000L).map { Timestamp(it) } @@ -106,8 +125,11 @@ internal class LiveFeedbackViewModelTest { eventReporter, timeHelper, isUsingAutoCapture, + isFaceTrackingEnabled, getSpoofCheckConfiguration, testCaptureAttemptTracker, + cropToFaceSquare, + SelectDominantFaceUseCase(), testCoroutineRule.testCoroutineDispatcher, ) } @@ -164,7 +186,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - valid face before start keeps NOT_STARTED, shows VALID feedback and stores fallback`() = runTest { - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -180,7 +202,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - starting capture moves to CAPTURING and valid frames become VALID_CAPTURING`() = runTest { - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) @@ -198,9 +220,9 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - invalid faces map to the correct feedback`() = runTest { - every { faceDetector.analyze(frame) } returnsMany listOf( - getFace(Rect(0, 0, 30, 30)), // too far - getFace(Rect(0, 0, 80, 80)), // too close + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame + getFace(Rect(0, 0, 110, 110)), // too close - the square cannot fit the frame getFace(yaw = 45f), // off yaw getFace(roll = 45f), // off roll getFace(quality = 0f), @@ -226,7 +248,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - bad quality faces considered valid after a valid fallback capture`() = runTest { - every { faceDetector.analyze(frame) } returnsMany listOf( + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( getFace(quality = 0f), getFace(), getFace(yaw = 45f), // to switch out the result @@ -250,7 +272,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - progress reflects captured sample ratio`() = runTest { - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) @@ -263,7 +285,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - capturing enough samples finishes and publishes sorted result`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame) } returns validFace + every { faceDetector.analyze(frame, any(), any()) } returns validFace val states = collectStates() viewModel.initAutoCapture() @@ -290,9 +312,9 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - frames arriving while finishing is still in progress are dropped, not appended`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame) } returns validFace + every { faceDetector.analyze(frame, any(), any()) } returns validFace // Simulate the camera delivering another frame while finishCapture() - every { faceDetector.analyze(frame, estimateAgeAndGender = true) } answers { + every { faceDetector.analyze(frame, true, any()) } answers { assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) viewModel.process(frame, frame) validFace @@ -304,13 +326,13 @@ internal class LiveFeedbackViewModelTest { viewModel.process(frame, frame) // reaches the requested sample count and triggers finishCapture() assertThat(viewModel.userCaptures).hasSize(1) - verify(exactly = 1) { faceDetector.analyze(frame) } + verify(exactly = 1) { faceDetector.analyze(frame, false, any()) } } @Test fun `auto - does not start until start capture is pressed`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -324,7 +346,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - held off capture does not start`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -338,7 +360,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - valid face after start begins CAPTURING and finishes after imaging duration`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() val states = collectStates() viewModel.initAutoCapture() @@ -362,13 +384,13 @@ internal class LiveFeedbackViewModelTest { fun `auto - frames arriving while finishing is still in progress are dropped, not appended`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true val validFace = getFace() - every { faceDetector.analyze(frame) } returns validFace + every { faceDetector.analyze(frame, any(), any()) } returns validFace var elapsedMs = 0L every { timeHelper.now() } answers { Timestamp(elapsedMs) } // Simulate a frame arriving while finishCapture() is still running - every { faceDetector.analyze(frame, estimateAgeAndGender = true) } answers { + every { faceDetector.analyze(frame, true, any()) } answers { assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) viewModel.process(frame, frame) validFace @@ -384,15 +406,15 @@ internal class LiveFeedbackViewModelTest { // Only the original sample was ever captured - the frame injected mid-finish was dropped. assertThat(viewModel.userCaptures).hasSize(1) - verify(exactly = 1) { faceDetector.analyze(frame) } + verify(exactly = 1) { faceDetector.analyze(frame, false, any()) } } @Test fun `auto - invalid faces map to the correct feedback`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returnsMany listOf( - getFace(Rect(0, 0, 30, 30)), // too far - getFace(Rect(0, 0, 80, 80)), // too close + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame + getFace(Rect(0, 0, 110, 110)), // too close - the square cannot fit the frame getFace(yaw = 45f), // off yaw getFace(roll = 45f), // off roll getFace(quality = 0f), // bad quality @@ -423,8 +445,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - returns correct amount of valid faces after finishing`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returnsMany listOf( - getFace(Rect(0, 0, 30, 30)), // too far + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame getFace(quality = 0.95f), // good getFace(quality = 0f), // bad quality getFace(quality = 0.9f), // good, but will be replaced @@ -449,8 +471,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof RECORDED finishes regardless of score`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) val states = collectStates() viewModel.initAutoCapture() @@ -468,8 +490,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED passing finishes capture`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.1f) + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.1f) viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -484,8 +506,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED failing goes through VALIDATION_FAILED and resets to NOT_STARTED`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) val states = collectStates() viewModel.initAutoCapture() @@ -504,8 +526,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED failing max attempts finishes capture`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -528,8 +550,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED failing retry reports an incrementing attempt number`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) val attemptNumbers = mutableListOf() coEvery { eventReporter.addCaptureEvents(any(), capture(attemptNumbers), any(), any(), any()) @@ -556,8 +578,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `frames are skipped while validating and progress uses the validation tint`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } answers { + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } answers { // A frame arriving mid-validation must not trigger another analysis. viewModel.process(frame, frame) assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.VALIDATING) @@ -571,12 +593,12 @@ internal class LiveFeedbackViewModelTest { viewModel.process(frame, frame) advanceUntilIdle() - verify(exactly = 1) { faceDetector.analyze(frame) } + verify(exactly = 1) { faceDetector.analyze(frame, false, any()) } } @Test fun `event saving - fallback capture event is saved only once across multiple valid pre-start frames`() = runTest { - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -591,7 +613,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - single sample capture saves one capture event and the fallback`() = runTest { - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -607,8 +629,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - captured samples are stored as non-fallback with one event per sample plus fallback`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame) } returns validFace - every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns null + every { faceDetector.analyze(frame, any(), any()) } returns validFace + every { faceDetector.analyze(any(), true, any()) } returns null viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) @@ -633,8 +655,8 @@ internal class LiveFeedbackViewModelTest { fun `event saving - enriches only the final accepted captures with age and gender`() = runTest { val validFace = getFace() val enrichedFace = getFace().copy(age = 34f, gender = Face.Gender(0.2f, 0.8f)) - every { faceDetector.analyze(frame) } returns validFace - every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns enrichedFace + every { faceDetector.analyze(frame, any(), any()) } returns validFace + every { faceDetector.analyze(any(), true, any()) } returns enrichedFace viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.RANK_ONE, 1) @@ -648,15 +670,15 @@ internal class LiveFeedbackViewModelTest { assertThat(first().face?.gender).isEqualTo(Face.Gender(0.2f, 0.8f)) } // Once for the captured sample and once for the fallback capture. - verify(exactly = 2) { faceDetector.analyze(any(), estimateAgeAndGender = true) } + verify(exactly = 2) { faceDetector.analyze(any(), true, any()) } } @Test fun `event saving - age and gender estimation runs during CAPTURING for every spoof-check retry`() = runTest { every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame) } returns getFace() - every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any()) } returns SpoofCheckResult(score = 0.9f) // always fails + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + every { faceDetector.analyze(any(), true, any()) } returns getFace() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) // always fails viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -668,7 +690,7 @@ internal class LiveFeedbackViewModelTest { advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) // Once for the captured sample and once for the fallback capture. - verify(exactly = 2) { faceDetector.analyze(any(), estimateAgeAndGender = true) } + verify(exactly = 2) { faceDetector.analyze(any(), true, any()) } // Attempt 2 reaches maxAttempts and finishes despite still failing spoof check. viewModel.process(frame, frame) @@ -678,15 +700,15 @@ internal class LiveFeedbackViewModelTest { assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.FINISHED) // Enrichment runs again for the second attempt's captures + fallback. - verify(exactly = 4) { faceDetector.analyze(any(), estimateAgeAndGender = true) } + verify(exactly = 4) { faceDetector.analyze(any(), true, any()) } } @Test fun `event saving - disabled spoof check never shows VALIDATING phase or the orange validation tint`() = runTest { // Default configuration from setUp() is SpoofCheckConfiguration.DISABLED. val validFace = getFace() - every { faceDetector.analyze(frame) } returns validFace - every { faceDetector.analyze(any(), estimateAgeAndGender = true) } returns validFace + every { faceDetector.analyze(frame, any(), any()) } returns validFace + every { faceDetector.analyze(any(), true, any()) } returns validFace val states = collectStates() viewModel.initAutoCapture() @@ -700,12 +722,12 @@ internal class LiveFeedbackViewModelTest { // Age/gender enrichment must never trigger assertThat(states.map { it.phase }).doesNotContain(LiveFeedbackState.Phase.VALIDATING) assertThat(states.map { it.progress.tint }).doesNotContain(Progress.Tint.VALIDATION) - coVerify(exactly = 0) { faceDetector.spoofCheck(any(), any()) } + coVerify(exactly = 0) { faceDetector.spoofCheck(any(), any(), any()) } } @Test fun `event saving - falls back to the fallback capture when no captured sample qualifies`() = runTest { - every { faceDetector.analyze(frame) } returnsMany listOf( + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( getFace(), // valid fallback frame before start null, // invalid captured sample (no face) ) @@ -728,7 +750,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - auto capture saves an event per stored sample plus the fallback`() = runTest { every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame) } returns getFace() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) @@ -742,14 +764,340 @@ internal class LiveFeedbackViewModelTest { coVerify(exactly = 1) { eventReporter.addFallbackCaptureEvent(any(), any()) } } + @Test + fun `target box tracks the detection as a square centred on it`() = runTest { + // 300x200px detection centred at (400, 400) in a 1000x1000 frame + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(250, 300, 550, 500)) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + val box = requireNotNull(viewModel.state.value.targetBox).rect + // Side is the longer detection side (300px), normalised against the 1000px frame + assertThat(box.width()).isWithin(TOLERANCE).of(0.3f) + assertThat(box.height()).isWithin(TOLERANCE).of(0.3f) + assertThat(box.centerX()).isWithin(TOLERANCE).of(0.4f) + assertThat(box.centerY()).isWithin(TOLERANCE).of(0.4f) + } + + @Test + fun `target box stays inside the frame when the face is against the edge`() = runTest { + // 300x300px detection whose centre sits on the left edge of the frame + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(-150, 300, 150, 600)) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + val box = requireNotNull(viewModel.state.value.targetBox).rect + assertThat(box.left).isAtLeast(0f) + assertThat(box.top).isAtLeast(0f) + assertThat(box.right).isAtMost(1f) + assertThat(box.bottom).isAtMost(1f) + assertThat(box.width()).isWithin(TOLERANCE).of(box.height()) + } + + @Test + fun `target box is tinted by the detection status`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + trackedFace(), // valid + trackedFace(yaw = 45f), // pose is off + trackedFace(quality = 0f), // quality is off + trackedFace(Rect(0, 0, 100, 100)), // below the minimum size + trackedFace(Rect(0, 0, 800, 800)), // above the maximum size + ) + val states = collectStates() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 5) + repeat(5) { viewModel.process(frame, frame) } + + assertThat(states.mapNotNull { it.targetBox?.tint }).containsExactly( + FaceTargetBox.Tint.VALID, + FaceTargetBox.Tint.WARNING, + FaceTargetBox.Tint.WARNING, + FaceTargetBox.Tint.INVALID, + FaceTargetBox.Tint.INVALID, + ) + } + + @Test + fun `target box is cleared when no face is detected`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf(trackedFace(), null) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) + viewModel.process(frame, frame) + assertThat(viewModel.state.value.targetBox).isNotNull() + + viewModel.process(frame, frame) + assertThat(viewModel.state.value.targetBox).isNull() + } + + @Test + fun `the square crop, not the full frame, is stored as the capture bitmap`() = runTest { + val squareCrop = mockk(relaxed = true) + every { cropToFaceSquare.invoke(any(), any()) } returns squareCrop + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + with(viewModel.userCaptures.first()) { + assertThat(bitmap).isSameInstanceAs(squareCrop) + assertThat(original).isSameInstanceAs(frame) + } + } + + @Test + fun `cutout capture leaves the spoof check on the SDK's own face choice`() = runTest { + every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + val selectorGiven = recordSpoofCheckSelector() + + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + advanceUntilIdle() + + // Only one person can be in a cutout, so there is nothing for a policy to choose between + // and the SDK is left to report its own best detection + assertThat(selectorGiven.single()).isNull() + } + + @Test + fun `face tracking gives the spoof check a face to pick, since the frame may hold several`() = runTest { + every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + val selectorGiven = recordSpoofCheckSelector() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + advanceUntilIdle() + + assertThat(selectorGiven.single()).isNotNull() + } + + @Test + fun `frames that are only looked at are never cropped`() = runTest { + // Too far to be kept, so its only job is to produce feedback + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(0, 0, 40, 40)) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) + // The square is still measured, for the overlay - only the pixel work is skipped + verify(exactly = 0) { cropToFaceSquare.invoke(any(), any()) } + } + + @Test + fun `the frame the crop was taken from is released once the capture is kept`() = runTest { + val analysed = mockk(relaxed = true) { + every { width } returns FRAME_SIZE_PX + every { height } returns FRAME_SIZE_PX + } + val squareCrop = mockk(relaxed = true) + every { cropToFaceSquare.invoke(analysed, any()) } returns squareCrop + every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, analysed) + + verify(exactly = 1) { analysed.recycle() } + verify(exactly = 0) { squareCrop.recycle() } + assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(squareCrop) + } + + @Test + fun `the analysed frame survives when the square turns out to be unusable`() = runTest { + // The use case hands the frame straight back rather than cropping + every { cropToFaceSquare.invoke(any(), any()) } returns frame + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + verify(exactly = 0) { frame.recycle() } + assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(frame) + } + + /** + * Drives the detector the way the real SDKs do: it offers [faces] to the selector the view + * model supplies and only "extracts" for the index that comes back, so these tests exercise + * the selection wiring rather than assuming it. + */ + @Test + fun `the capture mode is published in the state, not left for the UI to ask about`() = runTest { + // The UI renders one mode or the other off this flag, so it has to travel with the state + assertThat(viewModel.state.value.isFaceTrackingEnabled).isFalse() + + enableFaceTracking() + viewModel.initAutoCapture() + + assertThat(viewModel.state.value.isFaceTrackingEnabled).isTrue() + } + + @Test + fun `the capture mode is known by the first state the screen acts on`() = runTest { + enableFaceTracking() + val states = collectStates() + + viewModel.initAutoCapture() + + // The screen picks its mode before any frame is processed, so it never renders one mode + // and then switches to the other + assertThat(states.first().isFaceTrackingEnabled).isTrue() + assertThat(states.map { it.isFaceTrackingEnabled }.distinct()).containsExactly(true) + } + + @Test + fun `cutout - a tall face is judged by the area it fills, not by its longest side`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returns tallFace() + + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + // 45% by 75% is 34% of the area, which the cutout accepts + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) + } + + @Test + fun `tracking - the same tall face is rejected on its longest side`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returns tallFace() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + // The square is built from the longest side, and 75% of the frame is past the limit + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_CLOSE) + } + + @Test + fun `cutout - no target box is reported, since the cutout draws itself`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returns getFace() + + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) + assertThat(viewModel.state.value.targetBox).isNull() + } + + @Test + fun `cutout - the cutout crop is kept as the capture bitmap and no square is taken`() = runTest { + val analysed = mockk(relaxed = true) + every { faceDetector.analyze(analysed, any(), any()) } returns getFace() + + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, analysed) + + assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(analysed) + assertThat(viewModel.userCaptures.single().original).isSameInstanceAs(frame) + verify(exactly = 0) { cropToFaceSquare.squareFor(any(), any(), any()) } + } + + /** Records the selector handed to each spoof check, so the caller can assert on it. */ + private fun recordSpoofCheckSelector(): List { + val seen = mutableListOf() + coEvery { faceDetector.spoofCheck(any(), any(), any()) } answers { + seen += thirdArg() + SpoofCheckResult(score = 0.9f) + } + return seen + } + + /** Turns on the experimental whole-preview tracking behaviour for a single test. */ + private suspend fun enableFaceTracking() { + every { isFaceTrackingEnabled.invoke(any()) } returns true + } + + private fun detectorSees(vararg faces: Rect) { + every { faceDetector.analyze(frame, false, any()) } answers { + val selectFace = thirdArg<((List) -> Int?)?>() + val index = selectFace?.invoke(faces.toList()) ?: 0 + frameFace(faces[index]) + } + } + + @Test + fun `tracking - several faces in frame do not stop the capture`() = runTest { + enableFaceTracking() + detectorSees(Rect(100, 350, 400, 650), Rect(600, 350, 900, 650)) + + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + // One of them is chosen and the capture carries on rather than blocking on the ambiguity + assertThat(viewModel.userCaptures).hasSize(1) + assertThat(viewModel.state.value.targetBox).isNotNull() + } + + /** A face whose bounding box is expressed in the mocked frame's own pixels. */ + private fun frameFace(rect: Rect) = Face(FRAME_SIZE_PX, FRAME_SIZE_PX, rect, 0f, 0f, 1f, Random.nextBytes(20), "format") + + @Test + fun `the dominant face is the one handed to template extraction`() = runTest { + val bystander = Rect(50, 50, 150, 150) // small, off in a corner + val subject = Rect(250, 250, 750, 750) // large, centred and inside the valid size band + detectorSees(bystander, subject) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + // The stored capture describes the subject, not the bystander the SDK listed first + val stored = viewModel.userCaptures.single().face!! + assertThat(stored.relativeBoundingBox.width()).isWithin(TOLERANCE).of(0.5f) + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID_CAPTURING) + } + + /** + * Collects the states the screen acts on. The pre-initialisation state is dropped, matching + * the fragment, so these assertions describe rendered behaviour rather than start-up order. + */ private fun TestScope.collectStates(): List { val states = mutableListOf() backgroundScope.launch(testCoroutineRule.testCoroutineDispatcher) { - viewModel.state.toList(states) + viewModel.state.filter { it.stateInitialised }.toList(states) } return states } + /** + * A face sized for the cutout rules, which judge the share of the fixed target it fills. + * [rect] is in the source image's own pixels, so it is relative to the 100x100 source below. + */ private fun getFace( rect: Rect = Rect(0, 0, 60, 60), quality: Float = 1f, @@ -757,6 +1105,25 @@ internal class LiveFeedbackViewModelTest { roll: Float = 0f, ) = Face(100, 100, rect, yaw, roll, quality, Random.nextBytes(20), "format") + /** + * A face sized for the face tracking rules, which judge the face against the whole preview. + * Its source matches the mocked frame, so [rect] is read directly as frame pixels rather than + * being rescaled. The default fills half the frame, comfortably inside the valid band. + */ + /** + * A face taller than it is wide: 45% by 75% of the frame. The two capture modes disagree about + * it, because the cutout measures the area it fills while face tracking measures its longest + * side, and only the latter is over its limit. + */ + private fun tallFace() = trackedFace(Rect(275, 125, 725, 875)) + + private fun trackedFace( + rect: Rect = Rect(0, 0, 500, 500), + quality: Float = 1f, + yaw: Float = 0f, + roll: Float = 0f, + ) = Face(FRAME_SIZE_PX, FRAME_SIZE_PX, rect, yaw, roll, quality, Random.nextBytes(20), "format") + private fun spoofConfig(mode: FaceConfiguration.SpoofCheckMode = FaceConfiguration.SpoofCheckMode.RECORDED): SpoofCheckConfiguration = SpoofCheckConfiguration( mode = mode, @@ -769,6 +1136,8 @@ internal class LiveFeedbackViewModelTest { companion object { private const val QUALITY_THRESHOLD = 0.5f + private const val FRAME_SIZE_PX = 1000 + private const val TOLERANCE = 0.002f private const val AUTO_CAPTURE_IMAGING_DURATION_MS = 3000L } } diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt new file mode 100644 index 0000000000..8b7838d479 --- /dev/null +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt @@ -0,0 +1,71 @@ +package com.simprints.face.capture.usecases + +import android.graphics.Rect +import android.graphics.RectF +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +internal class CropToFaceSquareUseCaseTest { + private val useCase = CropToFaceSquareUseCase() + + @Test + fun `square takes the longer detection side and keeps the detection centre`() { + val square = useCase.squareFor(RectF(400f, 300f, 600f, 700f), 1000, 1000) + + assertThat(square).isEqualTo(Rect(300, 300, 700, 700)) + } + + @Test + fun `already square detections are returned unchanged`() { + val square = useCase.squareFor(RectF(100f, 200f, 400f, 500f), 1000, 1000) + + assertThat(square).isEqualTo(Rect(100, 200, 400, 500)) + } + + @Test + fun `square overhanging an edge is shifted inwards at full size`() { + val square = useCase.squareFor(RectF(-50f, 100f, 150f, 300f), 1000, 1000) + + assertThat(square).isEqualTo(Rect(0, 100, 200, 300)) + } + + @Test + fun `square overhanging two edges is shifted inwards on both axes`() { + val square = useCase.squareFor(RectF(900f, 900f, 1100f, 1100f), 1000, 1000) + + assertThat(square).isEqualTo(Rect(800, 800, 1000, 1000)) + } + + @Test + fun `square larger than the frame is capped at the shorter frame dimension`() { + val square = useCase.squareFor(RectF(-100f, -100f, 900f, 900f), 800, 600) + + assertThat(square.width()).isEqualTo(600) + assertThat(square.height()).isEqualTo(600) + assertThat(square).isEqualTo(Rect(100, 0, 700, 600)) + } + + @Test + fun `square is always fully inside a non-square frame`() { + val square = useCase.squareFor(RectF(700f, 10f, 900f, 210f), 800, 1600) + + assertThat(square.left).isAtLeast(0) + assertThat(square.top).isAtLeast(0) + assertThat(square.right).isAtMost(800) + assertThat(square.bottom).isAtMost(1600) + assertThat(square.width()).isEqualTo(square.height()) + } + + @Test + fun `empty detection yields an empty square`() { + assertThat(useCase.squareFor(RectF(100f, 100f, 100f, 100f), 1000, 1000).isEmpty).isTrue() + } + + @Test + fun `unmeasured frame yields an empty square`() { + assertThat(useCase.squareFor(RectF(0f, 0f, 200f, 200f), 0, 0).isEmpty).isTrue() + } +} diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCaseTest.kt new file mode 100644 index 0000000000..ec1294b187 --- /dev/null +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/SelectDominantFaceUseCaseTest.kt @@ -0,0 +1,75 @@ +package com.simprints.face.capture.usecases + +import android.graphics.Rect +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +internal class SelectDominantFaceUseCaseTest { + private val useCase = SelectDominantFaceUseCase() + + /** Centred square of [side] px, offset from the middle of the frame by [offsetX]/[offsetY]. */ + private fun face( + side: Int, + offsetX: Int = 0, + offsetY: Int = 0, + ): Rect { + val cx = FRAME / 2 + offsetX + val cy = FRAME / 2 + offsetY + return Rect(cx - side / 2, cy - side / 2, cx + side / 2, cy + side / 2) + } + + @Test + fun `no faces yields no selection`() { + assertThat(useCase(emptyList(), FRAME, FRAME)).isNull() + } + + @Test + fun `unmeasured frame yields no selection`() { + assertThat(useCase(listOf(face(300)), 0, 0)).isNull() + } + + @Test + fun `a lone face is the subject wherever it sits`() { + assertThat(useCase(listOf(face(300)), FRAME, FRAME)).isEqualTo(0) + assertThat(useCase(listOf(face(200, offsetX = 400, offsetY = 400)), FRAME, FRAME)).isEqualTo(0) + } + + @Test + fun `the clearly larger face wins`() { + val small = face(120, offsetX = 300) + val large = face(400) + + assertThat(useCase(listOf(small, large), FRAME, FRAME)).isEqualTo(1) + } + + @Test + fun `a centred face beats an equally sized one off to the side`() { + val offToTheSide = face(250, offsetX = 420) + val centred = face(250) + + assertThat(useCase(listOf(offToTheSide, centred), FRAME, FRAME)).isEqualTo(1) + } + + @Test + fun `a bystander does not outrank the subject being aimed at`() { + val subject = face(420) + val bystander = face(110, offsetX = 380, offsetY = -300) + + assertThat(useCase(listOf(subject, bystander), FRAME, FRAME)).isEqualTo(0) + } + + @Test + fun `two comparable faces still produce a winner rather than stalling`() { + val slightlyBigger = face(270, offsetX = -200) + val other = face(260, offsetX = 200) + + assertThat(useCase(listOf(slightlyBigger, other), FRAME, FRAME)).isEqualTo(0) + } + + private companion object { + const val FRAME = 1000 + } +} diff --git a/infra/core/src/main/java/com/simprints/core/tools/extensions/RectF.ext.kt b/infra/core/src/main/java/com/simprints/core/tools/extensions/RectF.ext.kt index 5168c2a5f2..13df51cb4a 100644 --- a/infra/core/src/main/java/com/simprints/core/tools/extensions/RectF.ext.kt +++ b/infra/core/src/main/java/com/simprints/core/tools/extensions/RectF.ext.kt @@ -1,6 +1,28 @@ package com.simprints.core.tools.extensions +import android.graphics.Rect import android.graphics.RectF import kotlin.math.abs fun RectF.area() = abs(height() * width()) + +/** + * Expands a bounding box expressed as fractions of the source image into source pixels. + */ +fun RectF.scaledTo( + width: Int, + height: Int, +) = RectF(left * width, top * height, right * width, bottom * height) + +/** + * Inverse of [scaledTo]: expresses a pixel rect as fractions of the source image. + */ +fun Rect.normalisedIn( + width: Int, + height: Int, +) = RectF( + left.toFloat() / width, + top.toFloat() / height, + right.toFloat() / width, + bottom.toFloat() / height, +) From ac9f09d890e3400b4264b65c2496c6bb288bd2a9 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 14:43:50 +0300 Subject: [PATCH 4/8] MS-1608 Adjust the cropped detection size and max ratio --- .../livefeedback/LiveFeedbackViewModel.kt | 26 ++++--- .../usecases/CropToFaceSquareUseCase.kt | 43 +++++++++-- .../livefeedback/LiveFeedbackViewModelTest.kt | 72 ++++++++++++++++--- .../usecases/CropToFaceSquareUseCaseTest.kt | 61 ++++++++++++++++ 4 files changed, 180 insertions(+), 22 deletions(-) diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt index 7c5c209a23..5929855b63 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt @@ -562,8 +562,10 @@ internal class LiveFeedbackViewModel @Inject constructor( val faceBox = potentialFace.relativeBoundingBox.scaledTo(frame.width, frame.height) val detectedSide = max(faceBox.width(), faceBox.height()) val frameSide = min(frame.width, frame.height) + // The square is the detection's longer side, so this is the size the stored crop would have + val smallestUsableSide = max(MIN_FACE_SIDE_PX.toFloat(), frameSide * MIN_FACE_FRAME_RATIO) return when { - detectedSide < frameSide * MIN_FACE_FRAME_RATIO -> FaceDetection.Status.TOOFAR + detectedSide < smallestUsableSide -> FaceDetection.Status.TOOFAR detectedSide > frameSide * MAX_FACE_FRAME_RATIO -> FaceDetection.Status.TOOCLOSE else -> poseAndQualityStatus(potentialFace) } @@ -648,17 +650,25 @@ internal class LiveFeedbackViewModel @Inject constructor( /** * How big the face has to be for face tracking, as a fraction of the preview's shorter - * edge. Chosen so both capture modes ask for a face of roughly the same size. - * - * The cutout wants the face to fill 20-50% of its target's area, which is 45-71% of that - * target's side; the target is in turn 90% of the preview's shorter edge, so the same face - * measures about 40-64% of the whole preview. + * edge. The lower bound asks for about as much face as the cutout does; the upper one is + * deliberately loose, since a square that follows the face only stops working once it no + * longer fits the preview. * * Expressed as proportions rather than pixels because the analyser resolution follows the * preview size - a fixed pixel band would mean a different thing on every device. */ - private const val MIN_FACE_FRAME_RATIO = 0.40f - private const val MAX_FACE_FRAME_RATIO = 0.65f + private const val MIN_FACE_FRAME_RATIO = 0.4f + private const val MAX_FACE_FRAME_RATIO = 0.9f + + /** + * Smallest face tracking will accept, in pixels of the square that gets cropped. Below this + * the face SDKs cannot be relied on to extract a template, so such a face is reported as + * too far however large it looks next to a low resolution preview. + * + * The upper bound belongs to [CropToFaceSquareUseCase.MAX_CROP_SIZE_PX], which shrinks the + * crop rather than rejecting it. + */ + private const val MIN_FACE_SIDE_PX = 150 } enum class PermissionAction { diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt index e3c767ef2f..7f74e4dcdb 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt @@ -3,6 +3,8 @@ package com.simprints.face.capture.usecases import android.graphics.Bitmap import android.graphics.Rect import android.graphics.RectF +import androidx.core.graphics.scale +import com.simprints.face.capture.usecases.CropToFaceSquareUseCase.Companion.MAX_CROP_SIZE_PX import javax.inject.Inject import kotlin.math.max import kotlin.math.min @@ -40,16 +42,49 @@ internal class CropToFaceSquareUseCase @Inject constructor() { } /** - * Crops [frame] to [square]. Returns [frame] untouched if the square does not describe a - * usable sub-region, so callers always get a bitmap back. + * Crops [frame] to [square], scaled down so that neither side exceeds [maxImageSizePx]. + * Returns [frame] untouched if the square does not describe a usable sub-region, so callers + * always get a bitmap back. [frame] stays the caller's to release either way. + * Crops [frame] to [square], capped at [MAX_CROP_SIZE_PX]. Returns [frame] untouched if the + * square does not describe a usable sub-region, so callers always get a bitmap back. */ operator fun invoke( frame: Bitmap, square: Rect, ): Bitmap { - if (square.isEmpty || square.width() > frame.width || square.height() > frame.height) { + // An in-bounds square that is offset off the edge would make createBitmap throw on the analyzer thread + if (square.isEmpty || !Rect(0, 0, frame.width, frame.height).contains(square)) { return frame } - return Bitmap.createBitmap(frame, square.left, square.top, square.width(), square.height()) + val crop = Bitmap.createBitmap(frame, square.left, square.top, square.width(), square.height()) + return scaledDownToCap(crop) + } + + /** + * Shrinks a crop that is larger than [MAX_CROP_SIZE_PX] in either dimension, keeping its + * proportions. Past that size the extra pixels buy nothing for template extraction while the + * image is still stored and uploaded, so they only cost storage and bandwidth. + */ + private fun scaledDownToCap(crop: Bitmap): Bitmap { + val longestSide = max(crop.width, crop.height) + if (longestSide <= MAX_CROP_SIZE_PX) return crop + + val scale = MAX_CROP_SIZE_PX.toFloat() / longestSide + val scaled = crop.scale( + width = (crop.width * scale).roundToInt().coerceAtLeast(1), + height = (crop.height * scale).roundToInt().coerceAtLeast(1), + ) + // createScaledBitmap can hand back the source itself when nothing needed doing + if (scaled !== crop) crop.recycle() + return scaled + } + + companion object { + /** + * Largest side the preserved crop may have, in pixels. The lower bound is enforced by the + * view model instead, which rejects a face too small to yield a template rather than + * upscaling it into one. + */ + const val MAX_CROP_SIZE_PX = 300 } } diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index a0f0f58fb1..bea8ad0602 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -807,7 +807,7 @@ internal class LiveFeedbackViewModelTest { trackedFace(yaw = 45f), // pose is off trackedFace(quality = 0f), // quality is off trackedFace(Rect(0, 0, 100, 100)), // below the minimum size - trackedFace(Rect(0, 0, 800, 800)), // above the maximum size + trackedFace(Rect(0, 0, 950, 950)), // above the maximum size ) val states = collectStates() @@ -979,7 +979,7 @@ internal class LiveFeedbackViewModelTest { viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) viewModel.process(frame, frame) - // 45% by 75% is 34% of the area, which the cutout accepts + // 45% by 95% is 43% of the area, which the cutout accepts assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) } @@ -992,10 +992,25 @@ internal class LiveFeedbackViewModelTest { viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) viewModel.process(frame, frame) - // The square is built from the longest side, and 75% of the frame is past the limit + // The square is built from the longest side, and 95% of the frame is past the limit assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_CLOSE) } + @Test + fun `tracking - a face filling most of the preview is still accepted`() = runTest { + // 88% of the frame: close enough that the cutout would have called it too close + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(60, 60, 940, 940)) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(frame, frame) + + // Tracking follows the face across the preview, so it only objects once the square + // stops fitting rather than as soon as the face grows past the cutout's target + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) + } + @Test fun `cutout - no target box is reported, since the cutout draws itself`() = runTest { every { faceDetector.analyze(frame, any(), any()) } returns getFace() @@ -1023,6 +1038,37 @@ internal class LiveFeedbackViewModelTest { verify(exactly = 0) { cropToFaceSquare.squareFor(any(), any(), any()) } } + @Test + fun `tracking - a face below the pixel floor is too far even when it fills the preview`() = runTest { + // 140px of face on a 300px preview is 47% of it, which the proportional rule would accept + val smallFrame = previewFrame(300) + every { faceDetector.analyze(smallFrame, any(), any()) } returns + Face(300, 300, Rect(80, 80, 220, 220), 0f, 0f, 1f, Random.nextBytes(20), "format") + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(smallFrame, smallFrame) + + // Too few pixels of face for any SDK to extract a template from, whatever it looks like + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) + } + + @Test + fun `tracking - a face above the pixel floor on the same preview is accepted`() = runTest { + val smallFrame = previewFrame(300) + every { faceDetector.analyze(smallFrame, any(), any()) } returns + Face(300, 300, Rect(70, 70, 230, 230), 0f, 0f, 1f, Random.nextBytes(20), "format") + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(smallFrame, smallFrame) + + // 160px clears the floor, so only the proportional rules have a say + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) + } + /** Records the selector handed to each spoof check, so the caller can assert on it. */ private fun recordSpoofCheckSelector(): List { val seen = mutableListOf() @@ -1033,6 +1079,12 @@ internal class LiveFeedbackViewModelTest { return seen } + /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ + private fun previewFrame(size: Int) = mockk(relaxed = true) { + every { width } returns size + every { height } returns size + } + /** Turns on the experimental whole-preview tracking behaviour for a single test. */ private suspend fun enableFaceTracking() { every { isFaceTrackingEnabled.invoke(any()) } returns true @@ -1106,17 +1158,17 @@ internal class LiveFeedbackViewModelTest { ) = Face(100, 100, rect, yaw, roll, quality, Random.nextBytes(20), "format") /** - * A face sized for the face tracking rules, which judge the face against the whole preview. - * Its source matches the mocked frame, so [rect] is read directly as frame pixels rather than - * being rescaled. The default fills half the frame, comfortably inside the valid band. - */ - /** - * A face taller than it is wide: 45% by 75% of the frame. The two capture modes disagree about + * A face taller than it is wide: 45% by 95% of the frame. The two capture modes disagree about * it, because the cutout measures the area it fills while face tracking measures its longest * side, and only the latter is over its limit. */ - private fun tallFace() = trackedFace(Rect(275, 125, 725, 875)) + private fun tallFace() = trackedFace(Rect(275, 25, 725, 975)) + /** + * A face sized for the face tracking rules, which judge the face against the whole preview. + * Its source matches the mocked frame, so [rect] is read directly as frame pixels rather than + * being rescaled. The default fills half the frame, comfortably inside the valid band. + */ private fun trackedFace( rect: Rect = Rect(0, 0, 500, 500), quality: Float = 1f, diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt index 8b7838d479..d5e572c5a0 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt @@ -1,5 +1,6 @@ package com.simprints.face.capture.usecases +import android.graphics.Bitmap import android.graphics.Rect import android.graphics.RectF import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -68,4 +69,64 @@ internal class CropToFaceSquareUseCaseTest { fun `unmeasured frame yields an empty square`() { assertThat(useCase.squareFor(RectF(0f, 0f, 200f, 200f), 0, 0).isEmpty).isTrue() } + + @Test + fun `an oversized crop is scaled down to the cap before it is kept`() { + val crop = useCase(frame(1000, 1000), Rect(100, 100, 700, 700)) + + // 600px of face is more than any template needs, and it is stored and uploaded as-is + assertThat(crop.width).isEqualTo(CropToFaceSquareUseCase.MAX_CROP_SIZE_PX) + assertThat(crop.height).isEqualTo(CropToFaceSquareUseCase.MAX_CROP_SIZE_PX) + } + + @Test + fun `a crop within the cap is kept at its own size`() { + val crop = useCase(frame(1000, 1000), Rect(0, 0, 250, 250)) + + assertThat(crop.width).isEqualTo(250) + assertThat(crop.height).isEqualTo(250) + } + + @Test + fun `a crop exactly at the cap is left alone`() { + val size = CropToFaceSquareUseCase.MAX_CROP_SIZE_PX + val crop = useCase(frame(1000, 1000), Rect(0, 0, size, size)) + + assertThat(crop.width).isEqualTo(size) + assertThat(crop.height).isEqualTo(size) + } + + @Test + fun `a square that fits but hangs off the edge is refused rather than cropped`() { + val source = frame(1000, 1000) + + // 300px wide and so small enough for the frame, but its right edge is past it + val crop = useCase(source, Rect(800, 100, 1100, 400)) + + assertThat(crop).isSameInstanceAs(source) + } + + @Test + fun `a square flush against the far edge is still cropped`() { + val crop = useCase(frame(1000, 1000), Rect(700, 700, 1000, 1000)) + + assertThat(crop.width).isEqualTo(300) + assertThat(crop.height).isEqualTo(300) + } + + @Test + fun `scaling down leaves the caller's frame and the returned crop usable`() { + val source = frame(1000, 1000) + + val crop = useCase(source, Rect(0, 0, 800, 800)) + + // Only the intermediate full-size crop is the use case's to release + assertThat(source.isRecycled).isFalse() + assertThat(crop.isRecycled).isFalse() + } + + private fun frame( + width: Int, + height: Int, + ) = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) } From 889c83ebc8c0c19b88e9b35c6bc8769c846ca423 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 15:12:06 +0300 Subject: [PATCH 5/8] MS-1608 Add additional configuration flags --- .../models/FaceTrackingConfiguration.kt | 26 ++++++ .../livefeedback/LiveFeedbackViewModel.kt | 33 +++----- .../usecases/CropToFaceSquareUseCase.kt | 31 +++---- .../GetFaceTrackingConfigurationUseCase.kt | 20 +++++ .../usecases/IsFaceTrackingEnabledUseCase.kt | 10 --- .../livefeedback/LiveFeedbackViewModelTest.kt | 55 ++++++++++--- .../usecases/CropToFaceSquareUseCaseTest.kt | 33 +++++--- ...GetFaceTrackingConfigurationUseCaseTest.kt | 81 +++++++++++++++++++ .../ExperimentalProjectConfiguration.kt | 30 +++++++ .../ExperimentalProjectConfigurationTest.kt | 37 +++++++++ 10 files changed, 279 insertions(+), 77 deletions(-) create mode 100644 face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt create mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt delete mode 100644 face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt create mode 100644 face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt diff --git a/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt b/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt new file mode 100644 index 0000000000..79baa2509b --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt @@ -0,0 +1,26 @@ +package com.simprints.face.capture.models + +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT + +/** + * How face tracking behaves for this project. + * + * The two sizes bracket the crop kept around the subject: anything smaller than [minFaceSizePx] is + * rejected as too far to yield a template, and anything larger than [maxImageSizePx] is scaled down + * before it is stored and uploaded. + */ +internal data class FaceTrackingConfiguration( + val enabled: Boolean, + val minFaceSizePx: Int, + val maxImageSizePx: Int, +) { + companion object { + /** What the standard cutout capture runs with, so the sizes are never read unset. */ + val DISABLED = FaceTrackingConfiguration( + enabled = false, + minFaceSizePx = FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT, + maxImageSizePx = FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT, + ) + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt index 5929855b63..7254c43474 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt @@ -12,11 +12,12 @@ import com.simprints.core.tools.extensions.scaledTo import com.simprints.core.tools.time.TimeHelper import com.simprints.face.capture.models.FaceDetection import com.simprints.face.capture.models.FaceTarget +import com.simprints.face.capture.models.FaceTrackingConfiguration import com.simprints.face.capture.models.SymmetricTarget import com.simprints.face.capture.screens.CaptureAttemptTracker import com.simprints.face.capture.usecases.CropToFaceSquareUseCase +import com.simprints.face.capture.usecases.GetFaceTrackingConfigurationUseCase import com.simprints.face.capture.usecases.GetSpoofCheckConfigurationUseCase -import com.simprints.face.capture.usecases.IsFaceTrackingEnabledUseCase import com.simprints.face.capture.usecases.IsUsingAutoCaptureUseCase import com.simprints.face.capture.usecases.SelectDominantFaceUseCase import com.simprints.face.capture.usecases.SimpleCaptureEventReporter @@ -59,7 +60,7 @@ internal class LiveFeedbackViewModel @Inject constructor( private val eventReporter: SimpleCaptureEventReporter, private val timeHelper: TimeHelper, private val isUsingAutoCaptureUseCase: IsUsingAutoCaptureUseCase, - private val isFaceTrackingEnabledUseCase: IsFaceTrackingEnabledUseCase, + private val getFaceTrackingConfiguration: GetFaceTrackingConfigurationUseCase, private val getSpoofCheckConfiguration: GetSpoofCheckConfigurationUseCase, private val captureAttemptTracker: CaptureAttemptTracker, private val cropToFaceSquare: CropToFaceSquareUseCase, @@ -100,7 +101,7 @@ internal class LiveFeedbackViewModel @Inject constructor( var isAutoCaptureHeldOff = true private set - private var isFaceTrackingEnabled = false + private var faceTracking = FaceTrackingConfiguration.DISABLED private var autoCaptureImagingTimeoutJob: Job? = null private var autoCaptureImagingDurationMillis: Long = FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_DEFAULT @@ -125,7 +126,7 @@ internal class LiveFeedbackViewModel @Inject constructor( phase = phase, feedback = feedback, isAutoCapture = isAutoCapture, - isFaceTrackingEnabled = isFaceTrackingEnabled, + isFaceTrackingEnabled = faceTracking.enabled, permissionStatus = permissionStatus, progress = computeProgress(phase, detectionForTint), targetBox = targetBox, @@ -136,7 +137,7 @@ internal class LiveFeedbackViewModel @Inject constructor( suspend fun initAutoCapture() { val config = configRepository.getProjectConfiguration() - isFaceTrackingEnabled = isFaceTrackingEnabledUseCase(config) + faceTracking = getFaceTrackingConfiguration(config) isAutoCapture = isUsingAutoCaptureUseCase(config) if (isAutoCapture) { // Await until capture button is pressed @@ -257,7 +258,7 @@ internal class LiveFeedbackViewModel @Inject constructor( } val captureStartTime = timeHelper.now() - val potentialFace = if (isFaceTrackingEnabled) { + val potentialFace = if (faceTracking.enabled) { // Detection covers the whole preview, so the subject is picked out of whatever is in // frame and only that face has a template extracted for it faceDetector.analyze(croppedBitmap) { faces -> @@ -430,7 +431,7 @@ internal class LiveFeedbackViewModel @Inject constructor( for ((index, bitmap) in userCaptures.map { it.original }.withIndex()) { // Only face tracking allows more than one person in frame, so only it needs a // policy for picking between them - cutout capture stays on the SDK's own choice - val selectFace: FaceSelector? = if (isFaceTrackingEnabled) { + val selectFace: FaceSelector? = if (faceTracking.enabled) { { faces -> selectDominantFace(faces, bitmap.width, bitmap.height) } } else { null @@ -486,7 +487,7 @@ internal class LiveFeedbackViewModel @Inject constructor( potentialFace: Face?, frame: Bitmap, ): Rect? { - if (!isFaceTrackingEnabled || potentialFace == null) return null + if (!faceTracking.enabled || potentialFace == null) return null val faceBox = potentialFace.relativeBoundingBox.scaledTo(frame.width, frame.height) return cropToFaceSquare.squareFor(faceBox, frame.width, frame.height).takeUnless { it.isEmpty } } @@ -518,7 +519,7 @@ internal class LiveFeedbackViewModel @Inject constructor( original = original, bitmap = bitmap, face = potentialFace, - status = if (isFaceTrackingEnabled) trackedFaceStatus(potentialFace, bitmap) else cutoutFaceStatus(potentialFace), + status = if (faceTracking.enabled) trackedFaceStatus(potentialFace, bitmap) else cutoutFaceStatus(potentialFace), detectionStartTime = timeHelper.now(), detectionEndTime = timeHelper.now(), ) @@ -533,7 +534,7 @@ internal class LiveFeedbackViewModel @Inject constructor( trackedSquare: Rect?, ): FaceDetection { if (trackedSquare == null) return faceDetection - val crop = cropToFaceSquare(faceDetection.bitmap, trackedSquare) + val crop = cropToFaceSquare(faceDetection.bitmap, trackedSquare, faceTracking.maxImageSizePx) // The use case hands the frame straight back when the square is unusable if (crop === faceDetection.bitmap) return faceDetection faceDetection.bitmap.recycle() @@ -563,7 +564,7 @@ internal class LiveFeedbackViewModel @Inject constructor( val detectedSide = max(faceBox.width(), faceBox.height()) val frameSide = min(frame.width, frame.height) // The square is the detection's longer side, so this is the size the stored crop would have - val smallestUsableSide = max(MIN_FACE_SIDE_PX.toFloat(), frameSide * MIN_FACE_FRAME_RATIO) + val smallestUsableSide = max(faceTracking.minFaceSizePx.toFloat(), frameSide * MIN_FACE_FRAME_RATIO) return when { detectedSide < smallestUsableSide -> FaceDetection.Status.TOOFAR detectedSide > frameSide * MAX_FACE_FRAME_RATIO -> FaceDetection.Status.TOOCLOSE @@ -659,16 +660,6 @@ internal class LiveFeedbackViewModel @Inject constructor( */ private const val MIN_FACE_FRAME_RATIO = 0.4f private const val MAX_FACE_FRAME_RATIO = 0.9f - - /** - * Smallest face tracking will accept, in pixels of the square that gets cropped. Below this - * the face SDKs cannot be relied on to extract a template, so such a face is reported as - * too far however large it looks next to a low resolution preview. - * - * The upper bound belongs to [CropToFaceSquareUseCase.MAX_CROP_SIZE_PX], which shrinks the - * crop rather than rejecting it. - */ - private const val MIN_FACE_SIDE_PX = 150 } enum class PermissionAction { diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt index 7f74e4dcdb..0ddb19871a 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCase.kt @@ -4,7 +4,6 @@ import android.graphics.Bitmap import android.graphics.Rect import android.graphics.RectF import androidx.core.graphics.scale -import com.simprints.face.capture.usecases.CropToFaceSquareUseCase.Companion.MAX_CROP_SIZE_PX import javax.inject.Inject import kotlin.math.max import kotlin.math.min @@ -44,32 +43,29 @@ internal class CropToFaceSquareUseCase @Inject constructor() { /** * Crops [frame] to [square], scaled down so that neither side exceeds [maxImageSizePx]. * Returns [frame] untouched if the square does not describe a usable sub-region, so callers - * always get a bitmap back. [frame] stays the caller's to release either way. - * Crops [frame] to [square], capped at [MAX_CROP_SIZE_PX]. Returns [frame] untouched if the - * square does not describe a usable sub-region, so callers always get a bitmap back. + * always get a bitmap back. */ operator fun invoke( frame: Bitmap, square: Rect, + maxImageSizePx: Int, ): Bitmap { // An in-bounds square that is offset off the edge would make createBitmap throw on the analyzer thread if (square.isEmpty || !Rect(0, 0, frame.width, frame.height).contains(square)) { return frame } val crop = Bitmap.createBitmap(frame, square.left, square.top, square.width(), square.height()) - return scaledDownToCap(crop) + return scaledDownToCap(crop, maxImageSizePx) } - /** - * Shrinks a crop that is larger than [MAX_CROP_SIZE_PX] in either dimension, keeping its - * proportions. Past that size the extra pixels buy nothing for template extraction while the - * image is still stored and uploaded, so they only cost storage and bandwidth. - */ - private fun scaledDownToCap(crop: Bitmap): Bitmap { + private fun scaledDownToCap( + crop: Bitmap, + maxImageSizePx: Int, + ): Bitmap { val longestSide = max(crop.width, crop.height) - if (longestSide <= MAX_CROP_SIZE_PX) return crop + if (longestSide <= maxImageSizePx) return crop - val scale = MAX_CROP_SIZE_PX.toFloat() / longestSide + val scale = maxImageSizePx.toFloat() / longestSide val scaled = crop.scale( width = (crop.width * scale).roundToInt().coerceAtLeast(1), height = (crop.height * scale).roundToInt().coerceAtLeast(1), @@ -78,13 +74,4 @@ internal class CropToFaceSquareUseCase @Inject constructor() { if (scaled !== crop) crop.recycle() return scaled } - - companion object { - /** - * Largest side the preserved crop may have, in pixels. The lower bound is enforced by the - * view model instead, which rejects a face too small to yield a template rather than - * upscaling it into one. - */ - const val MAX_CROP_SIZE_PX = 300 - } } diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt new file mode 100644 index 0000000000..9816bfe4b8 --- /dev/null +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt @@ -0,0 +1,20 @@ +package com.simprints.face.capture.usecases + +import com.simprints.face.capture.models.FaceTrackingConfiguration +import com.simprints.infra.config.store.models.ProjectConfiguration +import com.simprints.infra.config.store.models.experimental +import javax.inject.Inject + +internal class GetFaceTrackingConfigurationUseCase @Inject constructor() { + operator fun invoke(projectConfiguration: ProjectConfiguration): FaceTrackingConfiguration = + projectConfiguration.experimental().let { experimental -> + val minFaceSizePx = experimental.faceTrackingMinFaceSizePx + FaceTrackingConfiguration( + enabled = experimental.faceTrackingCaptureEnabled, + minFaceSizePx = minFaceSizePx, + // A cap under the floor would scale every accepted face back below it, leaving + // nothing capturable at all, so the two are kept in order however they are configured + maxImageSizePx = experimental.faceTrackingMaxImageSizePx.coerceAtLeast(minFaceSizePx), + ) + } +} diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt deleted file mode 100644 index 8ad8a1a5fb..0000000000 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/IsFaceTrackingEnabledUseCase.kt +++ /dev/null @@ -1,10 +0,0 @@ -package com.simprints.face.capture.usecases - -import com.simprints.infra.config.store.models.ProjectConfiguration -import com.simprints.infra.config.store.models.experimental -import javax.inject.Inject - -internal class IsFaceTrackingEnabledUseCase @Inject constructor() { - operator fun invoke(projectConfiguration: ProjectConfiguration): Boolean = - projectConfiguration.experimental().faceTrackingCaptureEnabled -} diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index bea8ad0602..c4b2cb91c0 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -8,10 +8,11 @@ import com.google.common.truth.Truth.* import com.simprints.core.domain.permission.PermissionStatus import com.simprints.core.tools.time.TimeHelper import com.simprints.core.tools.time.Timestamp +import com.simprints.face.capture.models.FaceTrackingConfiguration import com.simprints.face.capture.screens.CaptureAttemptTracker import com.simprints.face.capture.usecases.CropToFaceSquareUseCase +import com.simprints.face.capture.usecases.GetFaceTrackingConfigurationUseCase import com.simprints.face.capture.usecases.GetSpoofCheckConfigurationUseCase -import com.simprints.face.capture.usecases.IsFaceTrackingEnabledUseCase import com.simprints.face.capture.usecases.IsUsingAutoCaptureUseCase import com.simprints.face.capture.usecases.SelectDominantFaceUseCase import com.simprints.face.capture.usecases.SimpleCaptureEventReporter @@ -83,7 +84,7 @@ internal class LiveFeedbackViewModelTest { private lateinit var getSpoofCheckConfiguration: GetSpoofCheckConfigurationUseCase @MockK - private lateinit var isFaceTrackingEnabled: IsFaceTrackingEnabledUseCase + private lateinit var getFaceTrackingConfiguration: GetFaceTrackingConfigurationUseCase private lateinit var viewModel: LiveFeedbackViewModel @@ -100,7 +101,7 @@ internal class LiveFeedbackViewModelTest { MockKAnnotations.init(this, relaxed = true) every { frame.width } returns FRAME_SIZE_PX every { frame.height } returns FRAME_SIZE_PX - every { cropToFaceSquare.invoke(any(), any()) } returns frame + every { cropToFaceSquare.invoke(any(), any(), any()) } returns frame coEvery { resolveFaceBioSdkUseCase.invoke(any()) } returns mockk { every { detector } returns faceDetector } @@ -113,7 +114,7 @@ internal class LiveFeedbackViewModelTest { } returns QUALITY_THRESHOLD every { isUsingAutoCapture.invoke(any()) } returns false // The flag is off by default, so the bulk of the suite covers the standard cutout capture - every { isFaceTrackingEnabled.invoke(any()) } returns false + every { getFaceTrackingConfiguration.invoke(any()) } returns FaceTrackingConfiguration.DISABLED every { getSpoofCheckConfiguration.invoke(any(), any()) } returns SpoofCheckConfiguration.DISABLED every { timeHelper.now() } returnsMany (0..1000L).map { Timestamp(it) } @@ -125,7 +126,7 @@ internal class LiveFeedbackViewModelTest { eventReporter, timeHelper, isUsingAutoCapture, - isFaceTrackingEnabled, + getFaceTrackingConfiguration, getSpoofCheckConfiguration, testCaptureAttemptTracker, cropToFaceSquare, @@ -842,7 +843,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `the square crop, not the full frame, is stored as the capture bitmap`() = runTest { val squareCrop = mockk(relaxed = true) - every { cropToFaceSquare.invoke(any(), any()) } returns squareCrop + every { cropToFaceSquare.invoke(any(), any(), any()) } returns squareCrop every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() enableFaceTracking() @@ -902,7 +903,7 @@ internal class LiveFeedbackViewModelTest { assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) // The square is still measured, for the overlay - only the pixel work is skipped - verify(exactly = 0) { cropToFaceSquare.invoke(any(), any()) } + verify(exactly = 0) { cropToFaceSquare.invoke(any(), any(), any()) } } @Test @@ -912,7 +913,7 @@ internal class LiveFeedbackViewModelTest { every { height } returns FRAME_SIZE_PX } val squareCrop = mockk(relaxed = true) - every { cropToFaceSquare.invoke(analysed, any()) } returns squareCrop + every { cropToFaceSquare.invoke(analysed, any(), any()) } returns squareCrop every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace() enableFaceTracking() @@ -929,7 +930,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `the analysed frame survives when the square turns out to be unusable`() = runTest { // The use case hands the frame straight back rather than cropping - every { cropToFaceSquare.invoke(any(), any()) } returns frame + every { cropToFaceSquare.invoke(any(), any(), any()) } returns frame every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() enableFaceTracking() @@ -1079,6 +1080,34 @@ internal class LiveFeedbackViewModelTest { return seen } + @Test + fun `tracking - the configured minimum face size decides what is too far`() = runTest { + // 160px of face, which the built-in 150px floor would have accepted + val smallFrame = previewFrame(300) + every { faceDetector.analyze(smallFrame, any(), any()) } returns + Face(300, 300, Rect(70, 70, 230, 230), 0f, 0f, 1f, Random.nextBytes(20), "format") + + enableFaceTracking(minFaceSizePx = 200) + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(smallFrame, smallFrame) + + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) + } + + @Test + fun `tracking - the configured image cap is the one the crop is taken with`() = runTest { + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + + enableFaceTracking(maxImageSizePx = 512) + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + verify { cropToFaceSquare.invoke(any(), any(), 512) } + } + /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ private fun previewFrame(size: Int) = mockk(relaxed = true) { every { width } returns size @@ -1086,8 +1115,12 @@ internal class LiveFeedbackViewModelTest { } /** Turns on the experimental whole-preview tracking behaviour for a single test. */ - private suspend fun enableFaceTracking() { - every { isFaceTrackingEnabled.invoke(any()) } returns true + private fun enableFaceTracking( + minFaceSizePx: Int = FaceTrackingConfiguration.DISABLED.minFaceSizePx, + maxImageSizePx: Int = FaceTrackingConfiguration.DISABLED.maxImageSizePx, + ) { + every { getFaceTrackingConfiguration.invoke(any()) } returns + FaceTrackingConfiguration(enabled = true, minFaceSizePx = minFaceSizePx, maxImageSizePx = maxImageSizePx) } private fun detectorSees(vararg faces: Rect) { diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt index d5e572c5a0..f4409ee85b 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/CropToFaceSquareUseCaseTest.kt @@ -3,8 +3,8 @@ package com.simprints.face.capture.usecases import android.graphics.Bitmap import android.graphics.Rect import android.graphics.RectF -import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.google.common.truth.Truth.assertThat +import androidx.test.ext.junit.runners.* +import com.google.common.truth.Truth.* import org.junit.Test import org.junit.runner.RunWith @@ -72,16 +72,24 @@ internal class CropToFaceSquareUseCaseTest { @Test fun `an oversized crop is scaled down to the cap before it is kept`() { - val crop = useCase(frame(1000, 1000), Rect(100, 100, 700, 700)) + val crop = useCase(frame(1000, 1000), Rect(100, 100, 700, 700), maxImageSizePx = 300) // 600px of face is more than any template needs, and it is stored and uploaded as-is - assertThat(crop.width).isEqualTo(CropToFaceSquareUseCase.MAX_CROP_SIZE_PX) - assertThat(crop.height).isEqualTo(CropToFaceSquareUseCase.MAX_CROP_SIZE_PX) + assertThat(crop.width).isEqualTo(300) + assertThat(crop.height).isEqualTo(300) + } + + @Test + fun `the cap comes from the caller, so a project can ask for more detail`() { + val crop = useCase(frame(1000, 1000), Rect(100, 100, 700, 700), maxImageSizePx = 500) + + assertThat(crop.width).isEqualTo(500) + assertThat(crop.height).isEqualTo(500) } @Test fun `a crop within the cap is kept at its own size`() { - val crop = useCase(frame(1000, 1000), Rect(0, 0, 250, 250)) + val crop = useCase(frame(1000, 1000), Rect(0, 0, 250, 250), maxImageSizePx = 300) assertThat(crop.width).isEqualTo(250) assertThat(crop.height).isEqualTo(250) @@ -89,11 +97,10 @@ internal class CropToFaceSquareUseCaseTest { @Test fun `a crop exactly at the cap is left alone`() { - val size = CropToFaceSquareUseCase.MAX_CROP_SIZE_PX - val crop = useCase(frame(1000, 1000), Rect(0, 0, size, size)) + val crop = useCase(frame(1000, 1000), Rect(0, 0, 300, 300), maxImageSizePx = 300) - assertThat(crop.width).isEqualTo(size) - assertThat(crop.height).isEqualTo(size) + assertThat(crop.width).isEqualTo(300) + assertThat(crop.height).isEqualTo(300) } @Test @@ -101,14 +108,14 @@ internal class CropToFaceSquareUseCaseTest { val source = frame(1000, 1000) // 300px wide and so small enough for the frame, but its right edge is past it - val crop = useCase(source, Rect(800, 100, 1100, 400)) + val crop = useCase(source, Rect(800, 100, 1100, 400), maxImageSizePx = 300) assertThat(crop).isSameInstanceAs(source) } @Test fun `a square flush against the far edge is still cropped`() { - val crop = useCase(frame(1000, 1000), Rect(700, 700, 1000, 1000)) + val crop = useCase(frame(1000, 1000), Rect(700, 700, 1000, 1000), maxImageSizePx = 300) assertThat(crop.width).isEqualTo(300) assertThat(crop.height).isEqualTo(300) @@ -118,7 +125,7 @@ internal class CropToFaceSquareUseCaseTest { fun `scaling down leaves the caller's frame and the returned crop usable`() { val source = frame(1000, 1000) - val crop = useCase(source, Rect(0, 0, 800, 800)) + val crop = useCase(source, Rect(0, 0, 800, 800), maxImageSizePx = 300) // Only the intermediate full-size crop is the use case's to release assertThat(source.isRecycled).isFalse() diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt new file mode 100644 index 0000000000..7a4f96fecb --- /dev/null +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt @@ -0,0 +1,81 @@ +package com.simprints.face.capture.usecases + +import com.google.common.truth.Truth.assertThat +import com.simprints.infra.config.store.models.ProjectConfiguration +import io.mockk.MockKAnnotations +import io.mockk.every +import io.mockk.impl.annotations.MockK +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive +import org.junit.Before +import org.junit.Test + +internal class GetFaceTrackingConfigurationUseCaseTest { + @MockK + private lateinit var projectConfiguration: ProjectConfiguration + + private val useCase = GetFaceTrackingConfigurationUseCase() + + @Before + fun setUp() { + MockKAnnotations.init(this) + } + + /** + * Keys are spelled out rather than imported, since they are internal to the config store. That + * also pins the names the backend actually sends. + */ + private fun customConfig(vararg entries: Pair) { + every { projectConfiguration.custom } returns mapOf(*entries) + } + + @Test + fun `tracking is off and the sizes take their defaults when nothing is configured`() { + customConfig() + + with(useCase(projectConfiguration)) { + assertThat(enabled).isFalse() + assertThat(minFaceSizePx).isEqualTo(150) + assertThat(maxImageSizePx).isEqualTo(300) + } + } + + @Test + fun `configured sizes are carried through`() { + customConfig( + "faceTrackingCaptureEnabled" to JsonPrimitive(true), + "faceTrackingMinFaceSizePx" to JsonPrimitive(200), + "faceTrackingMaxImageSizePx" to JsonPrimitive(600), + ) + + with(useCase(projectConfiguration)) { + assertThat(enabled).isTrue() + assertThat(minFaceSizePx).isEqualTo(200) + assertThat(maxImageSizePx).isEqualTo(600) + } + } + + @Test + fun `a cap configured below the floor is raised to it rather than left unusable`() { + customConfig( + "faceTrackingMinFaceSizePx" to JsonPrimitive(400), + "faceTrackingMaxImageSizePx" to JsonPrimitive(200), + ) + + // Scaling an accepted face down to 200px would put every capture back under the floor + with(useCase(projectConfiguration)) { + assertThat(minFaceSizePx).isEqualTo(400) + assertThat(maxImageSizePx).isEqualTo(400) + } + } + + @Test + fun `the sizes are resolved even while tracking is off, so they are never read unset`() { + customConfig("faceTrackingMinFaceSizePx" to JsonPrimitive(250)) + + with(useCase(projectConfiguration)) { + assertThat(enabled).isFalse() + assertThat(minFaceSizePx).isEqualTo(250) + } + } +} diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt index eda1bdb72d..f020a8da38 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt @@ -42,6 +42,30 @@ data class ExperimentalProjectConfiguration( ?.booleanOrNull .let { it == true } + /** + * Smallest face, measured on the square that gets cropped around it, that face tracking will + * accept. Below this the face SDKs cannot be relied on to extract a template. + */ + val faceTrackingMinFaceSizePx: Int + get() = customConfig + ?.get(FACE_TRACKING_MIN_FACE_SIZE_PX) + ?.jsonPrimitive + ?.intOrNull + ?.coerceAtLeast(FACE_TRACKING_FACE_SIZE_PX_MIN) + ?: FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT + + /** + * Largest the cropped face is kept at before it is stored and uploaded. Anything bigger is + * scaled down, since the extra pixels cost storage and bandwidth without helping the template. + */ + val faceTrackingMaxImageSizePx: Int + get() = customConfig + ?.get(FACE_TRACKING_MAX_IMAGE_SIZE_PX) + ?.jsonPrimitive + ?.intOrNull + ?.coerceAtLeast(FACE_TRACKING_FACE_SIZE_PX_MIN) + ?: FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT + val recordsDbMigrationFromRealmEnabled: Boolean get() = customConfig ?.get(RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED) @@ -252,6 +276,12 @@ data class ExperimentalProjectConfiguration( internal const val FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS = "faceAutoCaptureImagingDurationMillis" internal const val FACE_TRACKING_CAPTURE_ENABLED = "faceTrackingCaptureEnabled" + internal const val FACE_TRACKING_MIN_FACE_SIZE_PX = "faceTrackingMinFaceSizePx" + const val FACE_TRACKING_FACE_SIZE_PX_MIN = 112 // Should at least be viable for SimFace + const val FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT = 150 + internal const val FACE_TRACKING_MAX_IMAGE_SIZE_PX = "faceTrackingMaxImageSizePx" + const val FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT = 300 + internal const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED = "recordsDbMigrationFromRealmEnabled" const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_MAX_RETRIES = "recordsDbMigrationFromRealmMaxRetries" internal const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_DEFAULT_MAX_RETRIES = 10 diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt index 0baf05f651..ec8ea8e238 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt @@ -8,6 +8,10 @@ import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration. import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MAX import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_AUTO_CAPTURE_IMAGING_DURATION_MILLIS_MIN import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_CAPTURE_ENABLED +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MAX_IMAGE_SIZE_PX +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MIN_FACE_SIZE_PX +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.MFID_LIGHTING_CONDITIONS_ASSESSMENT_ENABLED @@ -76,6 +80,39 @@ internal class ExperimentalProjectConfigurationTest { } } + @Test + fun `check face tracking minimum face size correctly`() { + mapOf( + // Value not present + emptyMap() to FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT, + // Value not int + mapOf(FACE_TRACKING_MIN_FACE_SIZE_PX to JsonPrimitive(true)) to FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT, + // Value present but not a usable size + mapOf(FACE_TRACKING_MIN_FACE_SIZE_PX to JsonPrimitive(0)) to 112, + mapOf(FACE_TRACKING_MIN_FACE_SIZE_PX to JsonPrimitive(-100)) to 112, + // Value present + mapOf(FACE_TRACKING_MIN_FACE_SIZE_PX to JsonPrimitive(200)) to 200, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).faceTrackingMinFaceSizePx).isEqualTo(result) + } + } + + @Test + fun `check face tracking maximum image size correctly`() { + mapOf( + // Value not present + emptyMap() to FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT, + // Value not int + mapOf(FACE_TRACKING_MAX_IMAGE_SIZE_PX to JsonPrimitive("300")) to FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT, + // Value present but not a usable size + mapOf(FACE_TRACKING_MAX_IMAGE_SIZE_PX to JsonPrimitive(0)) to 112, + // Value present + mapOf(FACE_TRACKING_MAX_IMAGE_SIZE_PX to JsonPrimitive(512)) to 512, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).faceTrackingMaxImageSizePx).isEqualTo(result) + } + } + @Test fun `check face auto capture imaging duration flag correctly`() { mapOf( From 138cb95cdeefdd6a7d86b71454a45582c2518222 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Mon, 21 Sep 2026 16:38:10 +0300 Subject: [PATCH 6/8] MS-1608 Add option to show the progress around capture button --- .../models/FaceTrackingConfiguration.kt | 5 +++ .../livefeedback/LiveFeedbackFragment.kt | 21 +++++++++++ .../screens/livefeedback/LiveFeedbackState.kt | 3 ++ .../livefeedback/LiveFeedbackViewModel.kt | 1 + .../livefeedback/views/FaceTrackingOverlay.kt | 32 +++++++++++++--- .../GetFaceTrackingConfigurationUseCase.kt | 1 + .../livefeedback/LiveFeedbackViewModelTest.kt | 37 ++++++++++++++++++- ...GetFaceTrackingConfigurationUseCaseTest.kt | 18 +++++++++ .../ExperimentalProjectConfiguration.kt | 12 ++++++ .../ExperimentalProjectConfigurationTest.kt | 19 ++++++++++ 10 files changed, 141 insertions(+), 8 deletions(-) diff --git a/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt b/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt index 79baa2509b..359e58f60f 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/models/FaceTrackingConfiguration.kt @@ -9,11 +9,15 @@ import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration. * The two sizes bracket the crop kept around the subject: anything smaller than [minFaceSizePx] is * rejected as too far to yield a template, and anything larger than [maxImageSizePx] is scaled down * before it is stored and uploaded. + * + * [progressAroundCaptureButton] moves the capture progress off the square that follows the face and + * onto the capture button, for projects that would rather have it stay put. */ internal data class FaceTrackingConfiguration( val enabled: Boolean, val minFaceSizePx: Int, val maxImageSizePx: Int, + val progressAroundCaptureButton: Boolean, ) { companion object { /** What the standard cutout capture runs with, so the sizes are never read unset. */ @@ -21,6 +25,7 @@ internal data class FaceTrackingConfiguration( enabled = false, minFaceSizePx = FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT, maxImageSizePx = FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT, + progressAroundCaptureButton = false, ) } } diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt index c0c9d0e566..73877fa43b 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackFragment.kt @@ -2,6 +2,7 @@ package com.simprints.face.capture.screens.livefeedback import android.Manifest import android.content.Intent +import android.graphics.RectF import android.os.Bundle import android.provider.Settings import android.view.View @@ -324,6 +325,26 @@ internal class LiveFeedbackFragment : Fragment(R.layout.fragment_live_feedback) state.phase == LiveFeedbackState.Phase.NOT_STARTED || state.phase == LiveFeedbackState.Phase.CAPTURING ), + progressAnchor = if (state.isProgressAroundCaptureButton) captureButtonBoundsInOverlay() else null, + ) + } + + /** + * The capture button's outline in the overlay's own coordinates, so progress can be drawn + * around it. Null until both views are laid out, which leaves the progress on the tracked face + * for the frame or two before that happens. + * + * A [com.google.android.material.button.MaterialButton] draws its background inset from its + * own bounds, so those insets are taken off here - otherwise the ring would sit further from + * the chip vertically than horizontally. + */ + private fun captureButtonBoundsInOverlay(): RectF? = with(binding) { + if (captureFeedbackBtn.width == 0 || captureFeedbackBtn.height == 0) return null + RectF( + (captureFeedbackBtn.left - faceTrackingOverlay.left).toFloat(), + (captureFeedbackBtn.top - faceTrackingOverlay.top + captureFeedbackBtn.insetTop).toFloat(), + (captureFeedbackBtn.right - faceTrackingOverlay.left).toFloat(), + (captureFeedbackBtn.bottom - faceTrackingOverlay.top - captureFeedbackBtn.insetBottom).toFloat(), ) } diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt index 78a7dfdffa..08f166a072 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackState.kt @@ -12,6 +12,8 @@ internal data class LiveFeedbackState( val feedback: Feedback, val isAutoCapture: Boolean, val isFaceTrackingEnabled: Boolean, + // Tracking mode only: keeps the progress on the capture button instead of on the tracked face + val isProgressAroundCaptureButton: Boolean, val permissionStatus: PermissionStatus, val progress: Progress, val targetBox: FaceTargetBox? = null, // Tracking mode only: the square drawn around the subject. Always null for the cutout. @@ -33,6 +35,7 @@ internal data class LiveFeedbackState( feedback = Feedback.NONE, isAutoCapture = isAutoCapture, isFaceTrackingEnabled = false, + isProgressAroundCaptureButton = false, permissionStatus = PermissionStatus.Denied, progress = Progress.HIDDEN, ) diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt index 7254c43474..326e685b8d 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt @@ -127,6 +127,7 @@ internal class LiveFeedbackViewModel @Inject constructor( feedback = feedback, isAutoCapture = isAutoCapture, isFaceTrackingEnabled = faceTracking.enabled, + isProgressAroundCaptureButton = faceTracking.progressAroundCaptureButton, permissionStatus = permissionStatus, progress = computeProgress(phase, detectionForTint), targetBox = targetBox, diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt index d946401f9b..c055dc3e43 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/views/FaceTrackingOverlay.kt @@ -23,7 +23,8 @@ import com.simprints.infra.resources.R as IDR /** * Draws the live face-tracking feedback over the whole camera preview: a colour-coded square * around the detected face, plus the capture progress traced along that square's contour so it - * follows the face instead of a fixed screen cutout. + * follows the face instead of a fixed screen cutout. Progress can instead be pinned to a fixed + * rect - see the `progressAnchor` of [update]. * * Nothing is dimmed - the preview stays fully visible and the square colour carries the state. * @@ -49,6 +50,9 @@ internal class FaceTrackingOverlay @JvmOverloads constructor( private val boxCornerRadius = 8f.dpToPx(context) private val progressStrokeWidth = 8f.dpToPx(context) + /** Matches the corners of `feedback_chip_white`, so the ring follows the button's outline. */ + private val captureButtonCornerRadius = 20f.dpToPx(context) + /** Clears the box outline so the two strokes sit side by side instead of overlapping. */ private val progressOutset = (boxStrokeWidth + progressStrokeWidth) / 2f + 4f.dpToPx(context) @@ -108,6 +112,9 @@ internal class FaceTrackingOverlay @JvmOverloads constructor( private var progress: Progress = Progress.HIDDEN private var showAimGuide: Boolean = false + /** Where the progress is drawn when it is not following the face. See [update]. */ + private var progressAnchor: RectF? = null + init { setBackgroundColor(Color.TRANSPARENT) } @@ -116,14 +123,25 @@ internal class FaceTrackingOverlay @JvmOverloads constructor( * Renders one frame of feedback. A null [target] means no face is currently tracked. * * [showAimGuide] draws the corner brackets that tell the operator where to put the subject. + * + * [progressAnchor] pins the progress to a fixed rect in this view's coordinates - the capture + * button - instead of tracing the square that follows the face. Null keeps it on the face. */ fun update( target: FaceTargetBox?, progress: Progress, showAimGuide: Boolean, + progressAnchor: RectF? = null, ) { - if (this.target == target && this.progress == progress && this.showAimGuide == showAimGuide) return + if (this.target == target && + this.progress == progress && + this.showAimGuide == showAimGuide && + this.progressAnchor == progressAnchor + ) { + return + } this.showAimGuide = showAimGuide + this.progressAnchor = progressAnchor?.let { RectF(it) } if (target == null) { // Nothing to glide from once the face is gone; the next one appears where it is @@ -169,11 +187,12 @@ internal class FaceTrackingOverlay @JvmOverloads constructor( } if (progress.visible) { + val anchor = progressAnchor progressPainter.progressColor = colorFor(progress.tint) progressPainter.setValue(progress.value) progressPainter.setContour( - progressContourAround(displayedRect.takeIf { box != null }), - boxCornerRadius + progressOutset, + progressContourAround(anchor ?: displayedRect.takeIf { box != null }), + (if (anchor != null) captureButtonCornerRadius else boxCornerRadius) + progressOutset, ) progressPainter.draw(canvas) } @@ -283,8 +302,9 @@ internal class FaceTrackingOverlay @JvmOverloads constructor( ) = from + (to - from) * fraction /** - * The progress traces the square itself, just outside its outline. Without a tracked face it - * falls back to a centred square so progress stays visible between detections. + * The progress traces [box] just outside its outline, whether that is the tracked square or a + * fixed anchor. With neither it falls back to a centred square, so progress stays visible + * between detections rather than disappearing with the face. */ private fun progressContourAround(box: RectF?): RectF { if (box == null) { diff --git a/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt b/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt index 9816bfe4b8..9a3e8d1285 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCase.kt @@ -15,6 +15,7 @@ internal class GetFaceTrackingConfigurationUseCase @Inject constructor() { // A cap under the floor would scale every accepted face back below it, leaving // nothing capturable at all, so the two are kept in order however they are configured maxImageSizePx = experimental.faceTrackingMaxImageSizePx.coerceAtLeast(minFaceSizePx), + progressAroundCaptureButton = experimental.faceTrackingProgressAroundCaptureButton, ) } } diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index c4b2cb91c0..a8879f7439 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -1108,6 +1108,34 @@ internal class LiveFeedbackViewModelTest { verify { cropToFaceSquare.invoke(any(), any(), 512) } } + @Test + fun `tracking - progress follows the tracked face unless the placement flag is set`() = runTest { + enableFaceTracking() + viewModel.initAutoCapture() + + // The screen picks where to draw progress off the state, so the default has to travel too + assertThat(viewModel.state.value.isProgressAroundCaptureButton).isFalse() + } + + @Test + fun `tracking - the progress placement flag is published in the state`() = runTest { + enableFaceTracking(progressAroundCaptureButton = true) + viewModel.initAutoCapture() + + assertThat(viewModel.state.value.isProgressAroundCaptureButton).isTrue() + } + + @Test + fun `tracking - the progress placement is known by the first state the screen acts on`() = runTest { + enableFaceTracking(progressAroundCaptureButton = true) + val states = collectStates() + + viewModel.initAutoCapture() + + // Otherwise progress would be drawn on the face first and jump to the button + assertThat(states.map { it.isProgressAroundCaptureButton }.distinct()).containsExactly(true) + } + /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ private fun previewFrame(size: Int) = mockk(relaxed = true) { every { width } returns size @@ -1118,9 +1146,14 @@ internal class LiveFeedbackViewModelTest { private fun enableFaceTracking( minFaceSizePx: Int = FaceTrackingConfiguration.DISABLED.minFaceSizePx, maxImageSizePx: Int = FaceTrackingConfiguration.DISABLED.maxImageSizePx, + progressAroundCaptureButton: Boolean = false, ) { - every { getFaceTrackingConfiguration.invoke(any()) } returns - FaceTrackingConfiguration(enabled = true, minFaceSizePx = minFaceSizePx, maxImageSizePx = maxImageSizePx) + every { getFaceTrackingConfiguration.invoke(any()) } returns FaceTrackingConfiguration( + enabled = true, + minFaceSizePx = minFaceSizePx, + maxImageSizePx = maxImageSizePx, + progressAroundCaptureButton = progressAroundCaptureButton, + ) } private fun detectorSees(vararg faces: Rect) { diff --git a/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt b/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt index 7a4f96fecb..a1dc32bfd2 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/usecases/GetFaceTrackingConfigurationUseCaseTest.kt @@ -37,6 +37,7 @@ internal class GetFaceTrackingConfigurationUseCaseTest { assertThat(enabled).isFalse() assertThat(minFaceSizePx).isEqualTo(150) assertThat(maxImageSizePx).isEqualTo(300) + assertThat(progressAroundCaptureButton).isFalse() } } @@ -55,6 +56,23 @@ internal class GetFaceTrackingConfigurationUseCaseTest { } } + @Test + fun `progress follows the tracked face unless the placement flag is set`() { + customConfig("faceTrackingCaptureEnabled" to JsonPrimitive(true)) + + assertThat(useCase(projectConfiguration).progressAroundCaptureButton).isFalse() + } + + @Test + fun `the progress placement flag is carried through`() { + customConfig( + "faceTrackingCaptureEnabled" to JsonPrimitive(true), + "faceTrackingProgressAroundCaptureButton" to JsonPrimitive(true), + ) + + assertThat(useCase(projectConfiguration).progressAroundCaptureButton).isTrue() + } + @Test fun `a cap configured below the floor is raised to it rather than left unusable`() { customConfig( diff --git a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt index f020a8da38..5cc99cbb79 100644 --- a/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt +++ b/infra/config-store/src/main/java/com/simprints/infra/config/store/models/ExperimentalProjectConfiguration.kt @@ -66,6 +66,17 @@ data class ExperimentalProjectConfiguration( ?.coerceAtLeast(FACE_TRACKING_FACE_SIZE_PX_MIN) ?: FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT + /** + * Draws the face tracking capture progress around the capture button rather than around the + * square that follows the subject's face. Has no effect unless [faceTrackingCaptureEnabled]. + */ + val faceTrackingProgressAroundCaptureButton: Boolean + get() = customConfig + ?.get(FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON) + ?.jsonPrimitive + ?.booleanOrNull + .let { it == true } + val recordsDbMigrationFromRealmEnabled: Boolean get() = customConfig ?.get(RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED) @@ -281,6 +292,7 @@ data class ExperimentalProjectConfiguration( const val FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT = 150 internal const val FACE_TRACKING_MAX_IMAGE_SIZE_PX = "faceTrackingMaxImageSizePx" const val FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT = 300 + internal const val FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON = "faceTrackingProgressAroundCaptureButton" internal const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_ENABLED = "recordsDbMigrationFromRealmEnabled" const val RECORDS_DB_MIGRATION_FROM_REALM_TO_ROOM_MAX_RETRIES = "recordsDbMigrationFromRealmMaxRetries" diff --git a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt index ec8ea8e238..f53548119b 100644 --- a/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt +++ b/infra/config-store/src/test/java/com/simprints/infra/config/store/models/ExperimentalProjectConfigurationTest.kt @@ -12,6 +12,7 @@ import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration. import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MAX_IMAGE_SIZE_PX_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MIN_FACE_SIZE_PX import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_MIN_FACE_SIZE_PX_DEFAULT +import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.FALLBACK_TO_COMMCARE_THRESHOLD_DAYS_DEFAULT import com.simprints.infra.config.store.models.ExperimentalProjectConfiguration.Companion.MFID_LIGHTING_CONDITIONS_ASSESSMENT_ENABLED @@ -113,6 +114,24 @@ internal class ExperimentalProjectConfigurationTest { } } + @Test + fun `check face tracking progress placement flag correctly`() { + mapOf( + // Value not present - progress keeps following the tracked face + emptyMap() to false, + // Value not boolean + mapOf(FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON to JsonPrimitive(1)) to false, + // Value quoted, which the JSON primitive still reads as a boolean + mapOf(FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON to JsonPrimitive("true")) to true, + // Value present and FALSE + mapOf(FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON to JsonPrimitive(false)) to false, + // Value present and TRUE + mapOf(FACE_TRACKING_PROGRESS_AROUND_CAPTURE_BUTTON to JsonPrimitive(true)) to true, + ).forEach { (config, result) -> + assertThat(ExperimentalProjectConfiguration(config).faceTrackingProgressAroundCaptureButton).isEqualTo(result) + } + } + @Test fun `check face auto capture imaging duration flag correctly`() { mapOf( From 438e383a490d2cd888aae110e4851bb2556c276b Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Tue, 22 Sep 2026 16:02:06 +0300 Subject: [PATCH 7/8] MS-1608 Fix potential bitmap leaks in edge cases --- .../livefeedback/LiveFeedbackViewModel.kt | 14 ++- .../livefeedback/LiveFeedbackViewModelTest.kt | 98 +++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt index 326e685b8d..89b51b0bf3 100644 --- a/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt +++ b/face/capture/src/main/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModel.kt @@ -308,6 +308,8 @@ internal class LiveFeedbackViewModel @Inject constructor( if (isAutoCapture) { if (isQualifying(faceDetection)) { updateUserCapturesWith(cropBitmapToTrackedSquare(faceDetection, trackedSquare)) + } else { + releaseFrames(faceDetection) } } else { userCaptures.add(cropBitmapToTrackedSquare(faceDetection, trackedSquare)) @@ -374,6 +376,7 @@ internal class LiveFeedbackViewModel @Inject constructor( userCaptures[index].face?.quality ?: -1f }?.takeIf { it >= 0 } ?.let { worseQualityCaptureIndex -> + releaseFrames(userCaptures[worseQualityCaptureIndex]) userCaptures[worseQualityCaptureIndex] = faceDetection } } else { @@ -538,7 +541,8 @@ internal class LiveFeedbackViewModel @Inject constructor( val crop = cropToFaceSquare(faceDetection.bitmap, trackedSquare, faceTracking.maxImageSizePx) // The use case hands the frame straight back when the square is unusable if (crop === faceDetection.bitmap) return faceDetection - faceDetection.bitmap.recycle() + // Only a copy made for this screen is ours to release. + if (faceDetection.bitmap !== faceDetection.original) faceDetection.bitmap.recycle() return faceDetection.copy(bitmap = crop) } @@ -596,11 +600,19 @@ internal class LiveFeedbackViewModel @Inject constructor( if (faceDetection.hasValidStatus() && detectionQuality >= fallbackQuality) { Simber.i("Fallback capture updated", tag = FACE_CAPTURE) val kept = cropBitmapToTrackedSquare(faceDetection, trackedSquare).apply { isFallback = true } + fallbackCapture?.let { releaseFrames(it) } fallbackCapture = kept createFirstFallbackCaptureEvent(kept) + } else { + releaseFrames(faceDetection) } } + private fun releaseFrames(faceDetection: FaceDetection) { + faceDetection.original.recycle() + faceDetection.bitmap.recycle() + } + /** * Send a fallback capture event only once */ diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index a8879f7439..cab940ca1f 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -891,6 +891,80 @@ internal class LiveFeedbackViewModelTest { assertThat(selectorGiven.single()).isNotNull() } + @Test + fun `auto - a frame that does not qualify hands its own frames back`() = runTest { + val analysed = trackedPreviewFrame() + // Valid, so it reaches the qualifying check, but a better capture already fills the quota + every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace(quality = 0.5f) + every { isUsingAutoCapture.invoke(any()) } returns true + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + // First frame takes the only slot with a better face + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(quality = 0.9f) + viewModel.process(frame, frame) + viewModel.process(analysed, analysed) + + assertThat(viewModel.userCaptures.single().face?.quality).isEqualTo(0.9f) + verify(atLeast = 1) { analysed.recycle() } + } + + @Test + fun `auto - a qualifying frame keeps its frames`() = runTest { + val analysed = trackedPreviewFrame() + every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace() + every { isUsingAutoCapture.invoke(any()) } returns true + every { cropToFaceSquare.invoke(analysed, any(), any()) } returns analysed + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(analysed, analysed) + + assertThat(viewModel.userCaptures).hasSize(1) + verify(exactly = 0) { analysed.recycle() } + } + + @Test + fun `a frame too poor to become the fallback hands its frames back`() = runTest { + val analysed = trackedPreviewFrame() + // Too far to be a fallback, so nothing keeps it + every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace(Rect(0, 0, 40, 40)) + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(analysed, analysed) + + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) + verify(atLeast = 1) { analysed.recycle() } + } + + @Test + fun `a superseded fallback hands its frames back`() = runTest { + val first = trackedPreviewFrame() + every { faceDetector.analyze(first, any(), any()) } returns trackedFace(quality = 0.5f) + every { cropToFaceSquare.invoke(first, any(), any()) } returns first + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.process(first, first) + verify(exactly = 0) { first.recycle() } + + // A better face takes over as the fallback, so the one it displaces is no longer needed + val better = trackedPreviewFrame() + every { faceDetector.analyze(better, any(), any()) } returns trackedFace(quality = 0.9f) + every { cropToFaceSquare.invoke(better, any(), any()) } returns better + viewModel.process(better, better) + + verify(atLeast = 1) { first.recycle() } + verify(exactly = 0) { better.recycle() } + } + @Test fun `frames that are only looked at are never cropped`() = runTest { // Too far to be kept, so its only job is to produce feedback @@ -927,6 +1001,27 @@ internal class LiveFeedbackViewModelTest { assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(squareCrop) } + @Test + fun `the stored frame survives when it is also the frame that was analysed`() = runTest { + // Nothing cropped the frame on the way in, so the capture holds one bitmap under both names + val squareCrop = mockk(relaxed = true) + every { cropToFaceSquare.invoke(frame, any(), any()) } returns squareCrop + every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + + enableFaceTracking() + viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + viewModel.startCapture() + viewModel.process(frame, frame) + + with(viewModel.userCaptures.single()) { + assertThat(bitmap).isSameInstanceAs(squareCrop) + assertThat(original).isSameInstanceAs(frame) + } + // An enabled spoof check reads original back off the capture, so it has to still be usable + verify(exactly = 0) { frame.recycle() } + } + @Test fun `the analysed frame survives when the square turns out to be unusable`() = runTest { // The use case hands the frame straight back rather than cropping @@ -1137,6 +1232,9 @@ internal class LiveFeedbackViewModelTest { } /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ + /** A frame mock the size of the tracked preview, distinct per call so recycling can be traced. */ + private fun trackedPreviewFrame() = previewFrame(FRAME_SIZE_PX) + private fun previewFrame(size: Int) = mockk(relaxed = true) { every { width } returns size every { height } returns size From 6b9c61f49b21406fe613524d0e59f16398e65de8 Mon Sep 17 00:00:00 2001 From: Sergejs Luhmirins Date: Tue, 22 Sep 2026 16:36:09 +0300 Subject: [PATCH 8/8] MS-1608 Simplify LiveFeedbackViewModelTest for readability --- .../livefeedback/LiveFeedbackViewModelTest.kt | 590 ++++++++---------- 1 file changed, 258 insertions(+), 332 deletions(-) diff --git a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt index cab940ca1f..334c213c0d 100644 --- a/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt +++ b/face/capture/src/test/java/com/simprints/face/capture/screens/livefeedback/LiveFeedbackViewModelTest.kt @@ -147,7 +147,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `initAutoCapture reflects auto-capture flag in state`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true + enableAutoCapture() viewModel.initAutoCapture() advanceUntilIdle() @@ -187,11 +187,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - valid face before start keeps NOT_STARTED, shows VALID feedback and stores fallback`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() with(viewModel.state.value) { assertThat(phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) @@ -203,13 +202,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - starting capture moves to CAPTURING and valid frames become VALID_CAPTURING`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.process(frame, frame) - viewModel.startCapture() - viewModel.process(frame, frame) + capturingAfterFallback(2) + processFrame() with(viewModel.state.value) { assertThat(phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) @@ -221,7 +217,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - invalid faces map to the correct feedback`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + detectsInTurn( getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame getFace(Rect(0, 0, 110, 110)), // too close - the square cannot fit the frame getFace(yaw = 45f), // off yaw @@ -231,9 +227,8 @@ internal class LiveFeedbackViewModelTest { ) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - repeat(6) { viewModel.process(frame, frame) } + screenReady(2) + processFrame(6) val feedbacks = states.map { it.feedback } assertThat(feedbacks).containsExactly( @@ -249,7 +244,7 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - bad quality faces considered valid after a valid fallback capture`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + detectsInTurn( getFace(quality = 0f), getFace(), getFace(yaw = 45f), // to switch out the result @@ -257,9 +252,8 @@ internal class LiveFeedbackViewModelTest { ) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - repeat(5) { viewModel.process(frame, frame) } + screenReady(2) + processFrame(5) val feedbacks = states.map { it.feedback } assertThat(feedbacks).containsExactly( @@ -273,12 +267,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - progress reflects captured sample ratio`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing(2) + processFrame() assertThat(viewModel.state.value.progress.value).isEqualTo(0.5f) } @@ -286,15 +278,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - capturing enough samples finishes and publishes sorted result`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.process(frame, frame) - viewModel.startCapture() - viewModel.process(frame, frame) - viewModel.process(frame, frame) + capturingAfterFallback(2) + processFrame() + processFrame() val phases = states.map { it.phase } assertThat(phases) @@ -313,18 +302,16 @@ internal class LiveFeedbackViewModelTest { @Test fun `manual - frames arriving while finishing is still in progress are dropped, not appended`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) // Simulate the camera delivering another frame while finishCapture() every { faceDetector.analyze(frame, true, any()) } answers { assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) - viewModel.process(frame, frame) + processFrame() validFace } - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) // reaches the requested sample count and triggers finishCapture() + capturing() + processFrame() // reaches the requested sample count and triggers finishCapture() assertThat(viewModel.userCaptures).hasSize(1) verify(exactly = 1) { faceDetector.analyze(frame, false, any()) } @@ -332,12 +319,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - does not start until start capture is pressed`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableAutoCapture() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) // Guidance is suppressed before imaging starts in auto-capture. @@ -346,28 +332,24 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - held off capture does not start`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableAutoCapture() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() + capturing() viewModel.holdOffAutoCapture() - viewModel.process(frame, frame) + processFrame() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) } @Test fun `auto - valid face after start begins CAPTURING and finishes after imaging duration`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableAutoCapture() + detects(getFace()) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) @@ -383,9 +365,9 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - frames arriving while finishing is still in progress are dropped, not appended`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true + enableAutoCapture() val validFace = getFace() - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) var elapsedMs = 0L every { timeHelper.now() } answers { Timestamp(elapsedMs) } @@ -393,14 +375,12 @@ internal class LiveFeedbackViewModelTest { // Simulate a frame arriving while finishCapture() is still running every { faceDetector.analyze(frame, true, any()) } answers { assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.CAPTURING) - viewModel.process(frame, frame) + processFrame() validFace } - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) // begins CAPTURING at t=0 + capturing() + processFrame() // begins CAPTURING at t=0 elapsedMs = AUTO_CAPTURE_IMAGING_DURATION_MS + 1 advanceTimeBy((AUTO_CAPTURE_IMAGING_DURATION_MS + 1).milliseconds) // fires the timeout job -> finishCapture() @@ -412,8 +392,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - invalid faces map to the correct feedback`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + enableAutoCapture() + detectsInTurn( getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame getFace(Rect(0, 0, 110, 110)), // too close - the square cannot fit the frame getFace(yaw = 45f), // off yaw @@ -423,12 +403,8 @@ internal class LiveFeedbackViewModelTest { ) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - repeat(6) { - viewModel.process(frame, frame) - } + capturing() + processFrame(6) advanceUntilIdle() val feedbacks = states.map { it.feedback } @@ -445,8 +421,8 @@ internal class LiveFeedbackViewModelTest { @Test fun `auto - returns correct amount of valid faces after finishing`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + enableAutoCapture() + detectsInTurn( getFace(Rect(0, 0, 10, 10)), // too far - 100px on a 1000px frame getFace(quality = 0.95f), // good getFace(quality = 0f), // bad quality @@ -456,12 +432,8 @@ internal class LiveFeedbackViewModelTest { ) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.startCapture() - repeat(7) { - viewModel.process(frame, frame) - } + capturing(2) + processFrame(7) advanceUntilIdle() assertThat(viewModel.sortedQualifyingCaptures).hasSize(2) @@ -471,16 +443,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof RECORDED finishes regardless of score`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame, any(), any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) + enableSpoofCheck() + detects(getFace()) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) - viewModel.startCapture() - viewModel.process(frame, frame) + capturingAfterFallback() + processFrame() advanceUntilIdle() assertThat(states.map { it.phase }).contains(LiveFeedbackState.Phase.VALIDATING) @@ -490,15 +458,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED passing finishes capture`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame, any(), any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.1f) + enableSpoofCheck(FaceConfiguration.SpoofCheckMode.ENFORCED, score = 0.1f) + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) - viewModel.startCapture() - viewModel.process(frame, frame) + capturingAfterFallback() + processFrame() advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.FINISHED) @@ -506,16 +470,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED failing goes through VALIDATION_FAILED and resets to NOT_STARTED`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame, any(), any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) + enableSpoofCheck(FaceConfiguration.SpoofCheckMode.ENFORCED) + detects(getFace()) val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) - viewModel.startCapture() - viewModel.process(frame, frame) + capturingAfterFallback() + processFrame() advanceUntilIdle() assertThat(states.map { it.phase }).contains(LiveFeedbackState.Phase.VALIDATION_FAILED) @@ -526,51 +486,47 @@ internal class LiveFeedbackViewModelTest { @Test fun `spoof ENFORCED failing max attempts finishes capture`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame, any(), any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) + enableSpoofCheck(FaceConfiguration.SpoofCheckMode.ENFORCED) + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() // Attempt 1 - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) // Attempt 2 reaches maxAttempts - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.FINISHED) } @Test fun `spoof ENFORCED failing retry reports an incrementing attempt number`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame, any(), any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) + enableSpoofCheck(FaceConfiguration.SpoofCheckMode.ENFORCED) + detects(getFace()) val attemptNumbers = mutableListOf() coEvery { eventReporter.addCaptureEvents(any(), capture(attemptNumbers), any(), any(), any()) } just runs - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() // Attempt 0 fails spoof check - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() // Attempt 1 (retry) reaches maxAttempts and finishes - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() assertThat(attemptNumbers).containsAtLeast(0, 1) @@ -578,20 +534,18 @@ internal class LiveFeedbackViewModelTest { @Test fun `frames are skipped while validating and progress uses the validation tint`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableSpoofCheck() + detects(getFace()) coEvery { faceDetector.spoofCheck(any(), any(), any()) } answers { // A frame arriving mid-validation must not trigger another analysis. - viewModel.process(frame, frame) + processFrame() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.VALIDATING) assertThat(viewModel.state.value.progress.tint).isEqualTo(Progress.Tint.VALIDATION) SpoofCheckResult(score = 0.1f) } - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() advanceUntilIdle() verify(exactly = 1) { faceDetector.analyze(frame, false, any()) } @@ -599,13 +553,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - fallback capture event is saved only once across multiple valid pre-start frames`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) - viewModel.process(frame, frame) - viewModel.process(frame, frame) + screenReady() + processFrame() + processFrame() + processFrame() // A single fallback event despite several valid frames, and no capture events yet. coVerify(exactly = 1) { eventReporter.addFallbackCaptureEvent(any(), any()) } @@ -614,13 +567,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - single sample capture saves one capture event and the fallback`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) // fallback frame before start - viewModel.startCapture() - viewModel.process(frame, frame) // captured sample + capturingAfterFallback() + processFrame() // captured sample // 1 captured sample + 1 fallback capture. coVerify(exactly = 2) { eventReporter.addCaptureEvents(any(), any(), any(), any(), any()) } @@ -630,15 +580,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - captured samples are stored as non-fallback with one event per sample plus fallback`() = runTest { val validFace = getFace() - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) every { faceDetector.analyze(any(), true, any()) } returns null - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.process(frame, frame) // fallback frame before start - viewModel.startCapture() - viewModel.process(frame, frame) - viewModel.process(frame, frame) + capturingAfterFallback(2) + processFrame() + processFrame() assertThat(viewModel.userCaptures).hasSize(2) assertThat(viewModel.userCaptures.none { it.isFallback }).isTrue() @@ -656,14 +603,14 @@ internal class LiveFeedbackViewModelTest { fun `event saving - enriches only the final accepted captures with age and gender`() = runTest { val validFace = getFace() val enrichedFace = getFace().copy(age = 34f, gender = Face.Gender(0.2f, 0.8f)) - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) every { faceDetector.analyze(any(), true, any()) } returns enrichedFace viewModel.initAutoCapture() viewModel.initCapture(ModalitySdkType.RANK_ONE, 1) - viewModel.process(frame, frame) // fallback frame before start + processFrame() // fallback frame before start viewModel.startCapture() - viewModel.process(frame, frame) // captured sample -> finishes + processFrame() // captured sample -> finishes with(viewModel.sortedQualifyingCaptures) { assertThat(this).hasSize(1) @@ -676,27 +623,26 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - age and gender estimation runs during CAPTURING for every spoof-check retry`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(FaceConfiguration.SpoofCheckMode.ENFORCED) - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + // The default score sits above the threshold, so every attempt fails + enableSpoofCheck(FaceConfiguration.SpoofCheckMode.ENFORCED) + detects(getFace()) every { faceDetector.analyze(any(), true, any()) } returns getFace() - coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = 0.9f) // always fails - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() // Attempt 1 fails and gets discarded, but enrichment already ran while still CAPTURING, - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.NOT_STARTED) // Once for the captured sample and once for the fallback capture. verify(exactly = 2) { faceDetector.analyze(any(), true, any()) } // Attempt 2 reaches maxAttempts and finishes despite still failing spoof check. - viewModel.process(frame, frame) + processFrame() viewModel.startCapture() - viewModel.process(frame, frame) + processFrame() advanceUntilIdle() assertThat(viewModel.state.value.phase).isEqualTo(LiveFeedbackState.Phase.FINISHED) @@ -708,15 +654,12 @@ internal class LiveFeedbackViewModelTest { fun `event saving - disabled spoof check never shows VALIDATING phase or the orange validation tint`() = runTest { // Default configuration from setUp() is SpoofCheckConfiguration.DISABLED. val validFace = getFace() - every { faceDetector.analyze(frame, any(), any()) } returns validFace + detects(validFace) every { faceDetector.analyze(any(), true, any()) } returns validFace val states = collectStates() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) // fallback frame before start - viewModel.startCapture() - viewModel.process(frame, frame) // captured sample -> finishes + capturingAfterFallback() + processFrame() // captured sample -> finishes advanceUntilIdle() assertThat(states.map { it.phase }).contains(LiveFeedbackState.Phase.FINISHED) @@ -728,16 +671,13 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - falls back to the fallback capture when no captured sample qualifies`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + detectsInTurn( getFace(), // valid fallback frame before start null, // invalid captured sample (no face) ) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) // fallback frame - viewModel.startCapture() - viewModel.process(frame, frame) // invalid capture -> finishes + capturingAfterFallback() + processFrame() // invalid capture -> finishes with(viewModel.sortedQualifyingCaptures) { assertThat(this).hasSize(1) @@ -750,14 +690,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `event saving - auto capture saves an event per stored sample plus the fallback`() = runTest { - every { isUsingAutoCapture.invoke(any()) } returns true - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableAutoCapture() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) // pre-start fallback frame (held off) - viewModel.startCapture() - viewModel.process(frame, frame) // begins imaging + capturingAfterFallback() + processFrame() // begins imaging advanceTimeBy(AUTO_CAPTURE_IMAGING_DURATION_MS + 1) // 1 stored sample + 1 fallback. @@ -766,44 +703,35 @@ internal class LiveFeedbackViewModelTest { } @Test - fun `target box tracks the detection as a square centred on it`() = runTest { - // 300x200px detection centred at (400, 400) in a 1000x1000 frame - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(250, 300, 550, 500)) + fun `target box is a square on the detection, kept inside the frame`() = runTest { + detectsInTurn( + trackedFace(Rect(250, 300, 550, 500)), // 300x200px centred at (400, 400) in a 1000px frame + trackedFace(Rect(-150, 300, 150, 600)), // 300x300px centred on the left edge + ) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() - val box = requireNotNull(viewModel.state.value.targetBox).rect + processFrame() + val centred = requireNotNull(viewModel.state.value.targetBox).rect // Side is the longer detection side (300px), normalised against the 1000px frame - assertThat(box.width()).isWithin(TOLERANCE).of(0.3f) - assertThat(box.height()).isWithin(TOLERANCE).of(0.3f) - assertThat(box.centerX()).isWithin(TOLERANCE).of(0.4f) - assertThat(box.centerY()).isWithin(TOLERANCE).of(0.4f) - } + assertThat(centred.width()).isWithin(TOLERANCE).of(0.3f) + assertThat(centred.height()).isWithin(TOLERANCE).of(0.3f) + assertThat(centred.centerX()).isWithin(TOLERANCE).of(0.4f) + assertThat(centred.centerY()).isWithin(TOLERANCE).of(0.4f) - @Test - fun `target box stays inside the frame when the face is against the edge`() = runTest { - // 300x300px detection whose centre sits on the left edge of the frame - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(-150, 300, 150, 600)) - - enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) - - val box = requireNotNull(viewModel.state.value.targetBox).rect - assertThat(box.left).isAtLeast(0f) - assertThat(box.top).isAtLeast(0f) - assertThat(box.right).isAtMost(1f) - assertThat(box.bottom).isAtMost(1f) - assertThat(box.width()).isWithin(TOLERANCE).of(box.height()) + processFrame() + val clamped = requireNotNull(viewModel.state.value.targetBox).rect + assertThat(clamped.left).isAtLeast(0f) + assertThat(clamped.top).isAtLeast(0f) + assertThat(clamped.right).isAtMost(1f) + assertThat(clamped.bottom).isAtMost(1f) + assertThat(clamped.width()).isWithin(TOLERANCE).of(clamped.height()) } @Test fun `target box is tinted by the detection status`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf( + detectsInTurn( trackedFace(), // valid trackedFace(yaw = 45f), // pose is off trackedFace(quality = 0f), // quality is off @@ -813,9 +741,8 @@ internal class LiveFeedbackViewModelTest { val states = collectStates() enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 5) - repeat(5) { viewModel.process(frame, frame) } + screenReady(5) + processFrame(5) assertThat(states.mapNotNull { it.targetBox?.tint }).containsExactly( FaceTargetBox.Tint.VALID, @@ -828,15 +755,14 @@ internal class LiveFeedbackViewModelTest { @Test fun `target box is cleared when no face is detected`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returnsMany listOf(trackedFace(), null) + detectsInTurn(trackedFace(), null) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 2) - viewModel.process(frame, frame) + screenReady(2) + processFrame() assertThat(viewModel.state.value.targetBox).isNotNull() - viewModel.process(frame, frame) + processFrame() assertThat(viewModel.state.value.targetBox).isNull() } @@ -844,13 +770,11 @@ internal class LiveFeedbackViewModelTest { fun `the square crop, not the full frame, is stored as the capture bitmap`() = runTest { val squareCrop = mockk(relaxed = true) every { cropToFaceSquare.invoke(any(), any(), any()) } returns squareCrop - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + detects(trackedFace()) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() with(viewModel.userCaptures.first()) { assertThat(bitmap).isSameInstanceAs(squareCrop) @@ -860,14 +784,12 @@ internal class LiveFeedbackViewModelTest { @Test fun `cutout capture leaves the spoof check on the SDK's own face choice`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + enableSpoofCheck() + detects(getFace()) val selectorGiven = recordSpoofCheckSelector() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() advanceUntilIdle() // Only one person can be in a cutout, so there is nothing for a policy to choose between @@ -877,15 +799,13 @@ internal class LiveFeedbackViewModelTest { @Test fun `face tracking gives the spoof check a face to pick, since the frame may hold several`() = runTest { - every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig() - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + enableSpoofCheck() + detects(trackedFace()) val selectorGiven = recordSpoofCheckSelector() enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() advanceUntilIdle() assertThat(selectorGiven.single()).isNotNull() @@ -896,18 +816,21 @@ internal class LiveFeedbackViewModelTest { val analysed = trackedPreviewFrame() // Valid, so it reaches the qualifying check, but a better capture already fills the quota every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace(quality = 0.5f) - every { isUsingAutoCapture.invoke(any()) } returns true + enableAutoCapture() enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() + capturing() // First frame takes the only slot with a better face - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(quality = 0.9f) - viewModel.process(frame, frame) + detects(trackedFace(quality = 0.9f)) + processFrame() viewModel.process(analysed, analysed) - assertThat(viewModel.userCaptures.single().face?.quality).isEqualTo(0.9f) + assertThat( + viewModel.userCaptures + .single() + .face + ?.quality, + ).isEqualTo(0.9f) verify(atLeast = 1) { analysed.recycle() } } @@ -915,13 +838,11 @@ internal class LiveFeedbackViewModelTest { fun `auto - a qualifying frame keeps its frames`() = runTest { val analysed = trackedPreviewFrame() every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace() - every { isUsingAutoCapture.invoke(any()) } returns true + enableAutoCapture() every { cropToFaceSquare.invoke(analysed, any(), any()) } returns analysed enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() + capturing() viewModel.process(analysed, analysed) assertThat(viewModel.userCaptures).hasSize(1) @@ -935,8 +856,7 @@ internal class LiveFeedbackViewModelTest { every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace(Rect(0, 0, 40, 40)) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() viewModel.process(analysed, analysed) assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) @@ -950,8 +870,7 @@ internal class LiveFeedbackViewModelTest { every { cropToFaceSquare.invoke(first, any(), any()) } returns first enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() viewModel.process(first, first) verify(exactly = 0) { first.recycle() } @@ -968,12 +887,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `frames that are only looked at are never cropped`() = runTest { // Too far to be kept, so its only job is to produce feedback - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(0, 0, 40, 40)) + detects(trackedFace(Rect(0, 0, 40, 40))) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) // The square is still measured, for the overlay - only the pixel work is skipped @@ -991,9 +909,7 @@ internal class LiveFeedbackViewModelTest { every { faceDetector.analyze(analysed, any(), any()) } returns trackedFace() enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() + capturing() viewModel.process(frame, analysed) verify(exactly = 1) { analysed.recycle() } @@ -1006,13 +922,11 @@ internal class LiveFeedbackViewModelTest { // Nothing cropped the frame on the way in, so the capture holds one bitmap under both names val squareCrop = mockk(relaxed = true) every { cropToFaceSquare.invoke(frame, any(), any()) } returns squareCrop - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + detects(trackedFace()) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() with(viewModel.userCaptures.single()) { assertThat(bitmap).isSameInstanceAs(squareCrop) @@ -1026,13 +940,11 @@ internal class LiveFeedbackViewModelTest { fun `the analysed frame survives when the square turns out to be unusable`() = runTest { // The use case hands the frame straight back rather than cropping every { cropToFaceSquare.invoke(any(), any(), any()) } returns frame - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + detects(trackedFace()) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() verify(exactly = 0) { frame.recycle() } assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(frame) @@ -1044,23 +956,16 @@ internal class LiveFeedbackViewModelTest { * the selection wiring rather than assuming it. */ @Test - fun `the capture mode is published in the state, not left for the UI to ask about`() = runTest { + fun `the capture mode travels with the state, settled before the screen acts on it`() = runTest { // The UI renders one mode or the other off this flag, so it has to travel with the state assertThat(viewModel.state.value.isFaceTrackingEnabled).isFalse() - enableFaceTracking() - viewModel.initAutoCapture() - - assertThat(viewModel.state.value.isFaceTrackingEnabled).isTrue() - } - - @Test - fun `the capture mode is known by the first state the screen acts on`() = runTest { enableFaceTracking() val states = collectStates() viewModel.initAutoCapture() + assertThat(viewModel.state.value.isFaceTrackingEnabled).isTrue() // The screen picks its mode before any frame is processed, so it never renders one mode // and then switches to the other assertThat(states.first().isFaceTrackingEnabled).isTrue() @@ -1069,11 +974,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `cutout - a tall face is judged by the area it fills, not by its longest side`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns tallFace() + detects(tallFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() // 45% by 95% is 43% of the area, which the cutout accepts assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) @@ -1081,12 +985,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `tracking - the same tall face is rejected on its longest side`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns tallFace() + detects(tallFace()) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() // The square is built from the longest side, and 95% of the frame is past the limit assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_CLOSE) @@ -1095,12 +998,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `tracking - a face filling most of the preview is still accepted`() = runTest { // 88% of the frame: close enough that the cutout would have called it too close - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace(Rect(60, 60, 940, 940)) + detects(trackedFace(Rect(60, 60, 940, 940))) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() // Tracking follows the face across the preview, so it only objects once the square // stops fitting rather than as soon as the face grows past the cutout's target @@ -1109,11 +1011,10 @@ internal class LiveFeedbackViewModelTest { @Test fun `cutout - no target box is reported, since the cutout draws itself`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns getFace() + detects(getFace()) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(frame, frame) + screenReady() + processFrame() assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) assertThat(viewModel.state.value.targetBox).isNull() @@ -1124,9 +1025,7 @@ internal class LiveFeedbackViewModelTest { val analysed = mockk(relaxed = true) every { faceDetector.analyze(analysed, any(), any()) } returns getFace() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() + capturing() viewModel.process(frame, analysed) assertThat(viewModel.userCaptures.single().bitmap).isSameInstanceAs(analysed) @@ -1135,33 +1034,24 @@ internal class LiveFeedbackViewModelTest { } @Test - fun `tracking - a face below the pixel floor is too far even when it fills the preview`() = runTest { - // 140px of face on a 300px preview is 47% of it, which the proportional rule would accept + fun `tracking - the pixel floor decides, whatever share of the preview the face fills`() = runTest { val smallFrame = previewFrame(300) - every { faceDetector.analyze(smallFrame, any(), any()) } returns - Face(300, 300, Rect(80, 80, 220, 220), 0f, 0f, 1f, Random.nextBytes(20), "format") + // Both faces fill ~47% of a 300px preview, which the proportional rule accepts either way, + // so only their pixel count can tell them apart + every { faceDetector.analyze(smallFrame, any(), any()) } returnsMany listOf( + Face(300, 300, Rect(80, 80, 220, 220), 0f, 0f, 1f, Random.nextBytes(20), "format"), // 140px + Face(300, 300, Rect(70, 70, 230, 230), 0f, 0f, 1f, Random.nextBytes(20), "format"), // 160px + ) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.process(smallFrame, smallFrame) + screenReady() // Too few pixels of face for any SDK to extract a template from, whatever it looks like - assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) - } - - @Test - fun `tracking - a face above the pixel floor on the same preview is accepted`() = runTest { - val smallFrame = previewFrame(300) - every { faceDetector.analyze(smallFrame, any(), any()) } returns - Face(300, 300, Rect(70, 70, 230, 230), 0f, 0f, 1f, Random.nextBytes(20), "format") - - enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) viewModel.process(smallFrame, smallFrame) + assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) // 160px clears the floor, so only the proportional rules have a say + viewModel.process(smallFrame, smallFrame) assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.VALID) } @@ -1183,8 +1073,7 @@ internal class LiveFeedbackViewModelTest { Face(300, 300, Rect(70, 70, 230, 230), 0f, 0f, 1f, Random.nextBytes(20), "format") enableFaceTracking(minFaceSizePx = 200) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) + screenReady() viewModel.process(smallFrame, smallFrame) assertThat(viewModel.state.value.feedback).isEqualTo(LiveFeedbackState.Feedback.TOO_FAR) @@ -1192,13 +1081,11 @@ internal class LiveFeedbackViewModelTest { @Test fun `tracking - the configured image cap is the one the crop is taken with`() = runTest { - every { faceDetector.analyze(frame, any(), any()) } returns trackedFace() + detects(trackedFace()) enableFaceTracking(maxImageSizePx = 512) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() verify { cropToFaceSquare.invoke(any(), any(), 512) } } @@ -1213,25 +1100,68 @@ internal class LiveFeedbackViewModelTest { } @Test - fun `tracking - the progress placement flag is published in the state`() = runTest { + fun `tracking - the progress placement is settled before the screen acts on it`() = runTest { enableFaceTracking(progressAroundCaptureButton = true) + val states = collectStates() + viewModel.initAutoCapture() assertThat(viewModel.state.value.isProgressAroundCaptureButton).isTrue() + // Otherwise progress would be drawn on the face first and jump to the button + assertThat(states.map { it.isProgressAroundCaptureButton }.distinct()).containsExactly(true) } - @Test - fun `tracking - the progress placement is known by the first state the screen acts on`() = runTest { - enableFaceTracking(progressAroundCaptureButton = true) - val states = collectStates() + /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ + /** Brings the screen up the way the fragment does, leaving it ready for frames. */ + private suspend fun screenReady(samples: Int = 1) { viewModel.initAutoCapture() + viewModel.initCapture(ModalitySdkType.SIM_FACE, samples) + } - // Otherwise progress would be drawn on the face first and jump to the button - assertThat(states.map { it.isProgressAroundCaptureButton }.distinct()).containsExactly(true) + /** [screenReady] followed by the capture button press. */ + private suspend fun capturing(samples: Int = 1) { + screenReady(samples) + viewModel.startCapture() + } + + /** + * [screenReady] plus one frame before the button, which the screen keeps as the fallback + * capture, then the button itself. + */ + private suspend fun capturingAfterFallback(samples: Int = 1) { + screenReady(samples) + processFrame() + viewModel.startCapture() + } + + /** Feeds the shared [frame] through [times], the way the analyzer would. */ + private fun processFrame(times: Int = 1) = repeat(times) { viewModel.process(frame, frame) } + + /** Switches the screen to auto-capture. Call before [screenReady] or [capturing]. */ + private fun enableAutoCapture() { + every { isUsingAutoCapture.invoke(any()) } returns true + } + + /** What the detector reports for [frame], the bitmap almost every test feeds in. */ + private fun detects(face: Face?) { + every { faceDetector.analyze(frame, any(), any()) } returns face + } + + /** One detection per processed frame, in order. */ + private fun detectsInTurn(vararg faces: Face?) { + every { faceDetector.analyze(frame, any(), any()) } returnsMany faces.toList() + } + + /** Turns the spoof check on, scoring every capture [score] unless a test overrides it. */ + private fun enableSpoofCheck( + mode: FaceConfiguration.SpoofCheckMode = FaceConfiguration.SpoofCheckMode.RECORDED, + score: Float = 0.9f, + ) { + every { getSpoofCheckConfiguration.invoke(any(), any()) } returns spoofConfig(mode) + coEvery { faceDetector.spoofCheck(any(), any(), any()) } returns SpoofCheckResult(score = score) } - /** A mocked frame of [size] square pixels, for the rules that count pixels rather than ratios. */ /** A frame mock the size of the tracked preview, distinct per call so recycling can be traced. */ private fun trackedPreviewFrame() = previewFrame(FRAME_SIZE_PX) @@ -1267,10 +1197,8 @@ internal class LiveFeedbackViewModelTest { enableFaceTracking() detectorSees(Rect(100, 350, 400, 650), Rect(600, 350, 900, 650)) - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() // One of them is chosen and the capture carries on rather than blocking on the ambiguity assertThat(viewModel.userCaptures).hasSize(1) @@ -1287,10 +1215,8 @@ internal class LiveFeedbackViewModelTest { detectorSees(bystander, subject) enableFaceTracking() - viewModel.initAutoCapture() - viewModel.initCapture(ModalitySdkType.SIM_FACE, 1) - viewModel.startCapture() - viewModel.process(frame, frame) + capturing() + processFrame() // The stored capture describes the subject, not the bystander the SDK listed first val stored = viewModel.userCaptures.single().face!!