testRunningModes

packages/web-components/lib/components/selfie/src/smartselfie-capture/SmartSelfieCapture.tsx

@@ -179,6 +179,29 @@ const SmartSelfieCapture: FunctionComponent<Props> = ({
 
           <AlertDisplay alertTitle={faceCapture.alertTitle.value} />
 
+          {/* Diagnostic-only running-mode toggle (test PR). */}
+          <div className="running-mode-bar" role="group" aria-label="MediaPipe running mode">
+            {(['IMAGE', 'VIDEO', 'LIVE_STREAM'] as const).map((mode) => (
+              <button
+                key={mode}
+                type="button"
+                className={`running-mode-btn${
+                  faceCapture.runningMode.value === mode ? ' is-active' : ''
+                }`}
+                onClick={() => {
+                  faceCapture.setRunningMode(mode);
+                }}
+              >
+                {mode}
+              </button>
+            ))}
+          </div>
+          {faceCapture.runningModeError.value && (
+            <p className="running-mode-error" role="status">
+              {faceCapture.runningMode.value}: {faceCapture.runningModeError.value}
+            </p>
+          )}
+
           {!faceCapture.isCapturing.value &&
             !faceCapture.hasFinishedCapture.value && (
               <CaptureControls
@@ -260,6 +283,38 @@ const SmartSelfieCapture: FunctionComponent<Props> = ({
           font-size: 1rem;
           font-weight: 500;
         }
+
+        .running-mode-bar {
+          display: flex;
+          gap: 0.5rem;
+          justify-content: center;
+          margin: 0.75rem 0;
+        }
+
+        .running-mode-btn {
+          background: #eee;
+          color: #222;
+          border: 1px solid #ccc;
+          border-radius: 999px;
+          padding: 0.4rem 0.9rem;
+          font-size: 0.85rem;
+          font-weight: 500;
+          cursor: pointer;
+        }
+
+        .running-mode-btn.is-active {
+          background: ${themeColor || '#001096'};
+          color: white;
+          border-color: transparent;
+        }
+
+        .running-mode-error {
+          margin: 0 0 0.75rem;
+          color: #b00020;
+          text-align: center;
+          font-size: 0.85rem;
+          word-break: break-word;
+        }
       `}</style>
     </div>
   );

packages/web-components/lib/components/selfie/src/smartselfie-capture/hooks/useFaceCapture.ts

@@ -16,7 +16,9 @@
 import { captureImageFromVideo } from '../utils/imageCapture';
 import { ImageType } from '../constants';
 import { MESSAGES, type MessageKey } from '../utils/alertMessages';
 import { getMediapipeInstance } from '../utils/mediapipeManager';
+import type { MediapipeRunningMode } from '../utils/mediapipeManager';
+import { DEFAULT_MEDIAPIPE_RUNNING_MODE } from '../utils/mediapipeManager';
 import { t } from '../../../../../domain/localisation';
 import packageJson from '../../../../../../package.json';
 
@@ -74,6 +76,10 @@
   const totalCaptures = useSignal(1);
   const capturesTaken = useSignal(0);
   const hasFinishedCapture = useSignal(false);
+  const runningMode = useSignal<MediapipeRunningMode>(
+    DEFAULT_MEDIAPIPE_RUNNING_MODE,
+  );
+  const runningModeError = useSignal<string | null>(null);
 
   const smileCheckpoint = useComputed(() =>
     Math.floor(totalCaptures.value * 0.4),
@@ -118,18 +124,22 @@
     try {
       const isAlreadyLoaded =
         window.__smileIdentityMediapipe?.loaded &&
-        window.__smileIdentityMediapipe?.instance;
+        window.__smileIdentityMediapipe?.instance &&
+        window.__smileIdentityMediapipe?.runningMode === runningMode.value;
 
       if (!isAlreadyLoaded) {
         isInitializing.value = true;
         updateAlertImmediate('initializing');
       }
 
-      faceLandmarkerRef.current = await getMediapipeInstance();
+      faceLandmarkerRef.current = await getMediapipeInstance(runningMode.value);
+      runningModeError.value = null;
       isInitializing.value = false;
       startFallbackTimer();
     } catch (error) {
       console.error('Failed to initialize MediaPipe:', error);
+      runningModeError.value =
+        error instanceof Error ? error.message : String(error);
       isInitializing.value = false;
       // MediaPipe failed — start the fallback timer so the button eventually
       // enables and the user isn't permanently stuck.
@@ -138,6 +148,19 @@
     startFallbackTimer();
   };
 
+  // Diagnostic-only: rebuild the FaceLandmarker with a new runningMode.
+  // Driven by the temporary three-button bar on SmartSelfieCapture.
+  const setRunningMode = async (mode: MediapipeRunningMode) => {
+    if (mode === runningMode.value) return;
+    stopDetectionLoop();
+    resetFaceDetectionState();
+    runningMode.value = mode;
+    faceLandmarkerRef.current = null;
+    await initializeFaceLandmarker();
+    setupCanvas();
+    startDetectionLoop();
+  };
+
   const setupCanvas = () => {
     if (videoRef.current && canvasRef.current) {
       const { videoWidth, videoHeight } = videoRef.current;
@@ -229,10 +252,14 @@
       const croppedCanvas = createCroppedVideoFrame(videoRef.current);
       const detectionSource = croppedCanvas || videoRef.current;
 
-      const results = faceLandmarkerRef.current.detectForVideo(
-        detectionSource,
-        performance.now(),
-      );
+      // Dispatch detection by runningMode. JS bindings only expose `detect()`
+      // and `detectForVideo()`; LIVE_STREAM is upstream-only and is forced
+      // through `detectForVideo` here so we can observe behaviour on the wasm.
+      const landmarker = faceLandmarkerRef.current;
+      const results =
+        runningMode.value === 'IMAGE'
+          ? landmarker.detect(detectionSource)
+          : landmarker.detectForVideo(detectionSource, performance.now());
 
       faceLandmarks.value = results.faceLandmarks || [];
 
@@ -629,5 +656,8 @@
     handleClose,
     cleanup,
     resetFaceDetectionState,
+    runningMode,
+    runningModeError,
+    setRunningMode,
   };
 };

packages/web-components/lib/components/selfie/src/smartselfie-capture/utils/mediapipeManager.ts

@@ -61,13 +61,27 @@ const isExcludedGpuFromWebGL = (renderer?: string | null): boolean => {
   return matchesExcludedGpu(rendererString);
 };
 
+/**
+ * Running modes the FaceLandmarker can be configured with.
+ * The MediaPipe JS task bindings only declare `IMAGE | VIDEO` in their
+ * TypeScript types; `LIVE_STREAM` is exposed in the upstream Java API
+ * (https://ai.google.dev/edge/api/mediapipe/java/com/google/mediapipe/tasks/vision/core/RunningMode)
+ * and is included here so this diagnostic build can probe whether the web
+ * wasm runtime accepts it. Expect a runtime error when LIVE_STREAM is
+ * selected on the current bindings.
+ */
+export type MediapipeRunningMode = 'IMAGE' | 'VIDEO' | 'LIVE_STREAM';
+
+export const DEFAULT_MEDIAPIPE_RUNNING_MODE: MediapipeRunningMode = 'VIDEO';
+
 declare global {
   interface Window {
     __smileIdentityMediapipe?: {
       instance: FaceLandmarker | null;
       loading: Promise<FaceLandmarker> | null;
       loaded: boolean;
       supportsWasmReftypes?: boolean;
+      runningMode?: MediapipeRunningMode;
     };
   }
 }
@@ -207,7 +221,9 @@ const hasFP16Support = () => {
   return !!(hasHalfFloatExt && hasColorBufferHalfFloat && hasHalfFloatLinear);
 };
 
-export const getMediapipeInstance = async (): Promise<FaceLandmarker> => {
+export const getMediapipeInstance = async (
+  runningMode: MediapipeRunningMode = DEFAULT_MEDIAPIPE_RUNNING_MODE,
+): Promise<FaceLandmarker> => {
   if (!window.__smileIdentityMediapipe) {
     window.__smileIdentityMediapipe = {
       instance: null,
@@ -218,7 +234,30 @@ export const getMediapipeInstance = async (): Promise<FaceLandmarker> => {
 
   const mediapipeGlobal = window.__smileIdentityMediapipe;
 
-  if (mediapipeGlobal.loaded && mediapipeGlobal.instance) {
+  // If a cached instance exists but was built for a different runningMode,
+  // tear it down so we rebuild with the requested mode. This is the test-PR
+  // toggle path; production callers stick with the default VIDEO mode.
+  if (
+    mediapipeGlobal.loaded &&
+    mediapipeGlobal.instance &&
+    mediapipeGlobal.runningMode !== runningMode
+  ) {
+    try {
+      mediapipeGlobal.instance.close();
+    } catch (closeError) {
+      console.warn('[SmileID] FaceLandmarker.close() failed.', closeError);
+    }
+    mediapipeGlobal.instance = null;
+    mediapipeGlobal.loaded = false;
+    mediapipeGlobal.loading = null;
+    mediapipeGlobal.runningMode = undefined;
+  }
+
+  if (
+    mediapipeGlobal.loaded &&
+    mediapipeGlobal.instance &&
+    mediapipeGlobal.runningMode === runningMode
+  ) {
     return mediapipeGlobal.instance;
   }
 
@@ -249,18 +288,26 @@ export const getMediapipeInstance = async (): Promise<FaceLandmarker> => {
       const delegate =
         gpuDelegate === 'CPU' || !hasFP16Support() ? 'CPU' : 'GPU';
 
+      console.info(
+        `[SmileID] Creating FaceLandmarker with runningMode=${runningMode}, delegate=${delegate}.`,
+      );
+
       const faceLandmarker = await FaceLandmarker.createFromOptions(vision, {
         baseOptions: {
           modelAssetPath: `https://web-models.smileidentity.com/face_landmarker/face_landmarker.task`,
           delegate,
         },
         outputFaceBlendshapes: true,
-        runningMode: 'VIDEO',
+        // Cast covers `LIVE_STREAM`, which is not in the JS RunningMode union
+        // but is exposed by the upstream Java API. Selecting it here is the
+        // explicit point of this diagnostic PR.
+        runningMode: runningMode as 'IMAGE' | 'VIDEO',
         numFaces: 2,
       });
 
       mediapipeGlobal.instance = faceLandmarker;
       mediapipeGlobal.loaded = true;
+      mediapipeGlobal.runningMode = runningMode;
       mediapipeGlobal.loading = null;
 
       return faceLandmarker;