From 19fa8e28d8a4453c706d026392e19f5bb9159fed Mon Sep 17 00:00:00 2001 From: Yeboahmedia Date: Fri, 22 May 2026 10:56:45 +0100 Subject: [PATCH] test: add runtime RunningMode toggle for MediaPipe diagnostics Diagnostic-only PR. Adds three buttons (IMAGE / VIDEO / LIVE_STREAM) to the smart-selfie capture screen that rebuild the FaceLandmarker with the selected runningMode and surface creation errors inline. Notes: - JS bindings only declare RunningMode = 'IMAGE' | 'VIDEO'. LIVE_STREAM exists in the Java API but not the web build, so it is passed through as a probe and is expected to error at create time. - IMAGE uses detect(); VIDEO/LIVE_STREAM use detectForVideo(). - Singleton cache is now keyed by runningMode; old instance is closed before rebuilding. --- .../SmartSelfieCapture.tsx | 55 +++++++++++++++++++ .../hooks/useFaceCapture.ts | 42 ++++++++++++-- .../utils/mediapipeManager.ts | 53 +++++++++++++++++- 3 files changed, 141 insertions(+), 9 deletions(-) diff --git a/packages/web-components/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx b/packages/web-components/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx index eecfa8da..cf6a52e6 100644 --- a/packages/web-components/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx +++ b/packages/web-components/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx @@ -179,6 +179,29 @@ const SmartSelfieCapture: FunctionComponent = ({ + {/* Diagnostic-only running-mode toggle (test PR). */} +
+ {(['IMAGE', 'VIDEO', 'LIVE_STREAM'] as const).map((mode) => ( + + ))} +
+ {faceCapture.runningModeError.value && ( +

+ {faceCapture.runningMode.value}: {faceCapture.runningModeError.value} +

+ )} + {!faceCapture.isCapturing.value && !faceCapture.hasFinishedCapture.value && ( = ({ font-size: 1rem; font-weight: 500; } + + .running-mode-bar { + display: flex; + gap: 0.5rem; + justify-content: center; + margin: 0.75rem 0; + } + + .running-mode-btn { + background: #eee; + color: #222; + border: 1px solid #ccc; + border-radius: 999px; + padding: 0.4rem 0.9rem; + font-size: 0.85rem; + font-weight: 500; + cursor: pointer; + } + + .running-mode-btn.is-active { + background: ${themeColor || '#001096'}; + color: white; + border-color: transparent; + } + + .running-mode-error { + margin: 0 0 0.75rem; + color: #b00020; + text-align: center; + font-size: 0.85rem; + word-break: break-word; + } `} ); diff --git a/packages/web-components/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts b/packages/web-components/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts index 2b63d5e2..0417d322 100644 --- a/packages/web-components/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts +++ b/packages/web-components/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts @@ -17,6 +17,8 @@ import { captureImageFromVideo } from '../utils/imageCapture'; import { ImageType } from '../constants'; import { MESSAGES, type MessageKey } from '../utils/alertMessages'; import { getMediapipeInstance } from '../utils/mediapipeManager'; +import type { MediapipeRunningMode } from '../utils/mediapipeManager'; +import { DEFAULT_MEDIAPIPE_RUNNING_MODE } from '../utils/mediapipeManager'; import { t } from '../../../../../domain/localisation'; import packageJson from '../../../../../../package.json'; @@ -74,6 +76,10 @@ export const useFaceCapture = ({ const totalCaptures = useSignal(1); const capturesTaken = useSignal(0); const hasFinishedCapture = useSignal(false); + const runningMode = useSignal( + DEFAULT_MEDIAPIPE_RUNNING_MODE, + ); + const runningModeError = useSignal(null); const smileCheckpoint = useComputed(() => Math.floor(totalCaptures.value * 0.4), @@ -118,18 +124,22 @@ export const useFaceCapture = ({ try { const isAlreadyLoaded = window.__smileIdentityMediapipe?.loaded && - window.__smileIdentityMediapipe?.instance; + window.__smileIdentityMediapipe?.instance && + window.__smileIdentityMediapipe?.runningMode === runningMode.value; if (!isAlreadyLoaded) { isInitializing.value = true; updateAlertImmediate('initializing'); } - faceLandmarkerRef.current = await getMediapipeInstance(); + faceLandmarkerRef.current = await getMediapipeInstance(runningMode.value); + runningModeError.value = null; isInitializing.value = false; startFallbackTimer(); } catch (error) { console.error('Failed to initialize MediaPipe:', error); + runningModeError.value = + error instanceof Error ? error.message : String(error); isInitializing.value = false; // MediaPipe failed — start the fallback timer so the button eventually // enables and the user isn't permanently stuck. @@ -138,6 +148,19 @@ export const useFaceCapture = ({ startFallbackTimer(); }; + // Diagnostic-only: rebuild the FaceLandmarker with a new runningMode. + // Driven by the temporary three-button bar on SmartSelfieCapture. + const setRunningMode = async (mode: MediapipeRunningMode) => { + if (mode === runningMode.value) return; + stopDetectionLoop(); + resetFaceDetectionState(); + runningMode.value = mode; + faceLandmarkerRef.current = null; + await initializeFaceLandmarker(); + setupCanvas(); + startDetectionLoop(); + }; + const setupCanvas = () => { if (videoRef.current && canvasRef.current) { const { videoWidth, videoHeight } = videoRef.current; @@ -229,10 +252,14 @@ export const useFaceCapture = ({ const croppedCanvas = createCroppedVideoFrame(videoRef.current); const detectionSource = croppedCanvas || videoRef.current; - const results = faceLandmarkerRef.current.detectForVideo( - detectionSource, - performance.now(), - ); + // Dispatch detection by runningMode. JS bindings only expose `detect()` + // and `detectForVideo()`; LIVE_STREAM is upstream-only and is forced + // through `detectForVideo` here so we can observe behaviour on the wasm. + const landmarker = faceLandmarkerRef.current; + const results = + runningMode.value === 'IMAGE' + ? landmarker.detect(detectionSource) + : landmarker.detectForVideo(detectionSource, performance.now()); faceLandmarks.value = results.faceLandmarks || []; @@ -629,5 +656,8 @@ export const useFaceCapture = ({ handleClose, cleanup, resetFaceDetectionState, + runningMode, + runningModeError, + setRunningMode, }; }; diff --git a/packages/web-components/lib/components/selfie/src/smartselfie-capture/utils/mediapipeManager.ts b/packages/web-components/lib/components/selfie/src/smartselfie-capture/utils/mediapipeManager.ts index 973fdaa6..0e5dfaed 100644 --- a/packages/web-components/lib/components/selfie/src/smartselfie-capture/utils/mediapipeManager.ts +++ b/packages/web-components/lib/components/selfie/src/smartselfie-capture/utils/mediapipeManager.ts @@ -61,6 +61,19 @@ const isExcludedGpuFromWebGL = (renderer?: string | null): boolean => { return matchesExcludedGpu(rendererString); }; +/** + * Running modes the FaceLandmarker can be configured with. + * The MediaPipe JS task bindings only declare `IMAGE | VIDEO` in their + * TypeScript types; `LIVE_STREAM` is exposed in the upstream Java API + * (https://ai.google.dev/edge/api/mediapipe/java/com/google/mediapipe/tasks/vision/core/RunningMode) + * and is included here so this diagnostic build can probe whether the web + * wasm runtime accepts it. Expect a runtime error when LIVE_STREAM is + * selected on the current bindings. + */ +export type MediapipeRunningMode = 'IMAGE' | 'VIDEO' | 'LIVE_STREAM'; + +export const DEFAULT_MEDIAPIPE_RUNNING_MODE: MediapipeRunningMode = 'VIDEO'; + declare global { interface Window { __smileIdentityMediapipe?: { @@ -68,6 +81,7 @@ declare global { loading: Promise | null; loaded: boolean; supportsWasmReftypes?: boolean; + runningMode?: MediapipeRunningMode; }; } } @@ -207,7 +221,9 @@ const hasFP16Support = () => { return !!(hasHalfFloatExt && hasColorBufferHalfFloat && hasHalfFloatLinear); }; -export const getMediapipeInstance = async (): Promise => { +export const getMediapipeInstance = async ( + runningMode: MediapipeRunningMode = DEFAULT_MEDIAPIPE_RUNNING_MODE, +): Promise => { if (!window.__smileIdentityMediapipe) { window.__smileIdentityMediapipe = { instance: null, @@ -218,7 +234,30 @@ export const getMediapipeInstance = async (): Promise => { const mediapipeGlobal = window.__smileIdentityMediapipe; - if (mediapipeGlobal.loaded && mediapipeGlobal.instance) { + // If a cached instance exists but was built for a different runningMode, + // tear it down so we rebuild with the requested mode. This is the test-PR + // toggle path; production callers stick with the default VIDEO mode. + if ( + mediapipeGlobal.loaded && + mediapipeGlobal.instance && + mediapipeGlobal.runningMode !== runningMode + ) { + try { + mediapipeGlobal.instance.close(); + } catch (closeError) { + console.warn('[SmileID] FaceLandmarker.close() failed.', closeError); + } + mediapipeGlobal.instance = null; + mediapipeGlobal.loaded = false; + mediapipeGlobal.loading = null; + mediapipeGlobal.runningMode = undefined; + } + + if ( + mediapipeGlobal.loaded && + mediapipeGlobal.instance && + mediapipeGlobal.runningMode === runningMode + ) { return mediapipeGlobal.instance; } @@ -249,18 +288,26 @@ export const getMediapipeInstance = async (): Promise => { const delegate = gpuDelegate === 'CPU' || !hasFP16Support() ? 'CPU' : 'GPU'; + console.info( + `[SmileID] Creating FaceLandmarker with runningMode=${runningMode}, delegate=${delegate}.`, + ); + const faceLandmarker = await FaceLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: `https://web-models.smileidentity.com/face_landmarker/face_landmarker.task`, delegate, }, outputFaceBlendshapes: true, - runningMode: 'VIDEO', + // Cast covers `LIVE_STREAM`, which is not in the JS RunningMode union + // but is exposed by the upstream Java API. Selecting it here is the + // explicit point of this diagnostic PR. + runningMode: runningMode as 'IMAGE' | 'VIDEO', numFaces: 2, }); mediapipeGlobal.instance = faceLandmarker; mediapipeGlobal.loaded = true; + mediapipeGlobal.runningMode = runningMode; mediapipeGlobal.loading = null; return faceLandmarker;