From f45a94d548c90f79e5240b27b1cac6090c6f1858 Mon Sep 17 00:00:00 2001 From: Tim Date: Sun, 12 Jul 2026 19:00:06 +0700 Subject: [PATCH 1/7] feat(desktop): experimental uncompressed capture mode (macOS) Adds an opt-in "Uncompressed" video mode that bypasses getUserMedia (which forces MJPEG on MS2131 devices) and captures raw YUY2 via a small native AVFoundation helper (native/nanokvm-capture.swift, built with pnpm build:capture). ffmpeg's avfoundation input was tried first but silently freezes on >=1080p uncompressed frames. - main: capture engine spawns the helper, selects the device by name (AVFoundation indices shift when Continuity cameras appear), negotiates fps from advertised modes, reports actual delivered dimensions, and streams frames over a MessageChannelMain with ack-based backpressure (one frame in flight; flooding starves renderer input handling). - renderer: WebGL2 YUY2->RGB integer-texture shader rendering into a 2x supersampled canvas via linear blit; rAF-paced with backgroundThrottling disabled so video keeps updating unfocused. - App: capture errors show a toast and fall back to the regular camera path instead of unmounting controls; unticking restores getUserMedia; resolution changes restart the capture session. Includes temporary debug logging to be stripped before release. Co-Authored-By: Claude Fable 5 --- desktop/.gitignore | 4 + desktop/native/nanokvm-capture.swift | 186 ++++++++++++ desktop/package.json | 1 + desktop/src/common/ipc-events.ts | 6 + desktop/src/main/capture/engine.ts | 270 ++++++++++++++++++ desktop/src/main/events/capture.ts | 53 ++++ desktop/src/main/events/index.ts | 1 + desktop/src/main/events/serial-port.ts | 3 + desktop/src/main/index.ts | 11 +- desktop/src/preload/index.ts | 10 +- desktop/src/renderer/src/App.tsx | 136 +++++++-- .../renderer/src/components/device/video.tsx | 3 + .../src/components/menu/video/capture.tsx | 39 +++ .../src/components/menu/video/index.tsx | 2 + .../src/components/menu/video/resolution.tsx | 27 +- .../src/components/mouse/absolute.tsx | 7 +- desktop/src/renderer/src/jotai/device.ts | 4 + .../src/libs/capture/ffmpeg-camera.ts | 186 ++++++++++++ .../renderer/src/libs/capture/yuv-renderer.ts | 177 ++++++++++++ .../src/renderer/src/libs/storage/index.ts | 9 + 20 files changed, 1100 insertions(+), 35 deletions(-) create mode 100644 desktop/native/nanokvm-capture.swift create mode 100644 desktop/src/main/capture/engine.ts create mode 100644 desktop/src/main/events/capture.ts create mode 100644 desktop/src/renderer/src/components/menu/video/capture.tsx create mode 100644 desktop/src/renderer/src/libs/capture/ffmpeg-camera.ts create mode 100644 desktop/src/renderer/src/libs/capture/yuv-renderer.ts diff --git a/desktop/.gitignore b/desktop/.gitignore index 15ee65bf..dd554f92 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -7,3 +7,7 @@ out .idea .vscode + +# compiled native capture helper (pnpm build:capture) +resources/nanokvm-capture +*.tsbuildinfo diff --git a/desktop/native/nanokvm-capture.swift b/desktop/native/nanokvm-capture.swift new file mode 100644 index 00000000..613d5334 --- /dev/null +++ b/desktop/native/nanokvm-capture.swift @@ -0,0 +1,186 @@ +// nanokvm-capture — minimal AVFoundation capture helper for the desktop app's +// uncompressed video mode. Replaces ffmpeg on macOS: ffmpeg's avfoundation +// input freezes on large (>=1080p) uncompressed frames, native capture works. +// +// nanokvm-capture list +// stdout: one JSON object: {"devices":[{"name":...,"formats":[ +// {"pixfmt":"yuvs","width":...,"height":...,"fps":[...]}]}]} +// +// nanokvm-capture stream --device --width --height --fps +// stderr: one line "META {"width":W,"height":H}" with the ACTUAL delivered +// buffer size (the UVC stack may serve the signal-native mode +// regardless of the requested format), then raw yuvs (YUY2) frames +// on stdout, W*2 bytes per row, tightly packed. +// +// SIGINT/SIGTERM stop the session cleanly. Blocking stdout writes provide +// natural backpressure: late frames are discarded by AVFoundation. +import AVFoundation +import Foundation + +func fourCC(_ code: FourCharCode) -> String { + var s = "" + for shift in stride(from: 24, through: 0, by: -8) { + s.append(Character(UnicodeScalar(UInt8((code >> UInt32(shift)) & 0xff)))) + } + return s +} + +func discoverDevices() -> [AVCaptureDevice] { + AVCaptureDevice.DiscoverySession( + deviceTypes: [.external, .builtInWideAngleCamera], + mediaType: .video, + position: .unspecified + ).devices +} + +func listCommand() { + var devices: [[String: Any]] = [] + for d in discoverDevices() { + var formats: [[String: Any]] = [] + for f in d.formats { + let dim = CMVideoFormatDescriptionGetDimensions(f.formatDescription) + let sub = fourCC(CMFormatDescriptionGetMediaSubType(f.formatDescription)) + let fps = f.videoSupportedFrameRateRanges.map { $0.maxFrameRate } + formats.append(["pixfmt": sub, "width": Int(dim.width), "height": Int(dim.height), "fps": fps]) + } + devices.append(["name": d.localizedName, "formats": formats]) + } + let json = try! JSONSerialization.data(withJSONObject: ["devices": devices]) + FileHandle.standardOutput.write(json) + FileHandle.standardOutput.write("\n".data(using: .utf8)!) +} + +final class Writer: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { + var metaSent = false + func captureOutput(_: AVCaptureOutput, didOutput sb: CMSampleBuffer, from _: AVCaptureConnection) { + guard let pb = CMSampleBufferGetImageBuffer(sb) else { return } + CVPixelBufferLockBaseAddress(pb, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pb, .readOnly) } + guard let base = CVPixelBufferGetBaseAddress(pb) else { return } + + let w = CVPixelBufferGetWidth(pb) + let h = CVPixelBufferGetHeight(pb) + let stride = CVPixelBufferGetBytesPerRow(pb) + let rowBytes = w * 2 // yuvs = 2 bytes/pixel + + if !metaSent { + metaSent = true + FileHandle.standardError.write("META {\"width\":\(w),\"height\":\(h)}\n".data(using: .utf8)!) + } + + if stride == rowBytes { + if fwrite(base, 1, rowBytes * h, stdout) != rowBytes * h { exit(0) } // EPIPE: consumer gone + } else { + var p = base + for _ in 0.. DispatchSourceSignal in + signal(sig, SIG_IGN) + let src = DispatchSource.makeSignalSource(signal: sig, queue: .main) + src.setEventHandler { + session.stopRunning() + exit(0) + } + src.resume() + return src + } + let sigint = stop(SIGINT) + let sigterm = stop(SIGTERM) + _ = (sigint, sigterm) + + RunLoop.main.run() +} + +// ---- arg parsing ---- +var args = Array(CommandLine.arguments.dropFirst()) +guard let cmd = args.first else { + FileHandle.standardError.write("usage: nanokvm-capture list | stream --device --width --height --fps \n".data(using: .utf8)!) + exit(2) +} +args.removeFirst() + +switch cmd { +case "list": + listCommand() +case "stream": + var device = "USB3 Video" + var width = 1920 + var height = 1080 + var fps = 60.0 + var i = 0 + while i < args.count - 1 { + switch args[i] { + case "--device": device = args[i + 1] + case "--width": width = Int(args[i + 1]) ?? width + case "--height": height = Int(args[i + 1]) ?? height + case "--fps": fps = Double(args[i + 1]) ?? fps + default: break + } + i += 2 + } + streamCommand(device: device, width: width, height: height, fps: fps) +default: + FileHandle.standardError.write("unknown command: \(cmd)\n".data(using: .utf8)!) + exit(2) +} diff --git a/desktop/package.json b/desktop/package.json index e422c838..125b30ac 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -14,6 +14,7 @@ "start": "electron-vite preview", "dev": "electron-vite dev", "build": "npm run typecheck && electron-vite build", + "build:capture": "swiftc -O -o resources/nanokvm-capture native/nanokvm-capture.swift", "build:unpack": "npm run build && electron-builder --dir", "build:win": "npm run build && electron-builder --win", "build:mac": "electron-vite build && electron-builder --mac", diff --git a/desktop/src/common/ipc-events.ts b/desktop/src/common/ipc-events.ts index b2b43e21..525b8006 100644 --- a/desktop/src/common/ipc-events.ts +++ b/desktop/src/common/ipc-events.ts @@ -6,6 +6,12 @@ export enum IpcEvents { REQUEST_MEDIA_PERMISSION = 'request-media-permission', SET_FULL_SCREEN = 'set-full-screen', + GET_CAPTURE_DEVICES = 'get-capture-devices', + START_CAPTURE = 'start-capture', + STOP_CAPTURE = 'stop-capture', + CAPTURE_PORT = 'capture-port', + CAPTURE_ERROR = 'capture-error', + GET_SERIAL_PORTS = 'get-serial-ports', OPEN_SERIAL_PORT = 'open-serial-port', OPEN_SERIAL_PORT_RSP = 'open-serial-port-rsp', diff --git a/desktop/src/main/capture/engine.ts b/desktop/src/main/capture/engine.ts new file mode 100644 index 00000000..e4853ed7 --- /dev/null +++ b/desktop/src/main/capture/engine.ts @@ -0,0 +1,270 @@ +// Uncompressed (YUY2/yuvs) capture — macOS Phase 1. Spawns the bundled +// nanokvm-capture Swift helper (native AVFoundation; ffmpeg's avfoundation +// input freezes on >=1080p uncompressed frames), assembles exact-size raw +// frames (O(n) fill buffer — a naive Buffer.concat is O(n^2) and throttles the +// pipeline), and pushes each frame to the renderer over a MessagePortMain. +import { ChildProcessWithoutNullStreams, execFile, spawn } from 'child_process' +import { existsSync } from 'fs' +import { join } from 'path' +import { performance } from 'perf_hooks' +import type { MessagePortMain } from 'electron' + +export type CaptureDevice = { index: number; name: string } +export type CaptureOptions = { + deviceIndex?: number + width: number + height: number + fps: number + requestId?: number +} +export type Mode = { width: number; height: number; fps: number[] } +// device is the AVFoundation device NAME: indices are not stable — Continuity +// cameras (iPhone/Desk View) insert and remove themselves and shift the list. +export type ResolvedCapture = { device: string; width: number; height: number; fps: number } + +const HELPER_CANDIDATES = [ + process.env.NANOKVM_CAPTURE_PATH, + join(__dirname, '../../resources/nanokvm-capture') +].filter(Boolean) as string[] + +function helperPath(): string { + const p = HELPER_CANDIDATES.find((c) => existsSync(c)) + if (!p) throw new Error('nanokvm-capture helper not found (build it with pnpm build:capture)') + return p +} + +const nowAbs = (): number => performance.timeOrigin + performance.now() + +type HelperFormat = { pixfmt: string; width: number; height: number; fps: number[] } +type HelperDevice = { name: string; formats: HelperFormat[] } + +function listAll(): Promise { + return new Promise((resolve) => { + execFile(helperPath(), ['list'], (_err, stdout) => { + try { + resolve(JSON.parse(stdout).devices as HelperDevice[]) + } catch { + resolve([]) + } + }) + }) +} + +/** Enumerate video capture devices. */ +export async function listDevices(): Promise { + return (await listAll()).map((d, index) => ({ index, name: d.name })) +} + +/** Pick the most likely capture device when the caller doesn't specify one. */ +function autoPick(devices: HelperDevice[]): string { + const pref = devices.find( + (d) => + /usb|video|capture|hdmi|kvm/i.test(d.name) && + !/facetime|desk view|iphone|capture screen/i.test(d.name) + ) + return (pref || devices[0])?.name ?? '' +} + +/** + * Resolve a requested capture to a device + a mode the device advertises. + * If the requested resolution isn't offered, fall back to the largest one. + * Note the UVC stack may still serve the signal-native mode regardless — the + * helper reports the ACTUAL dimensions when the stream starts. + */ +export async function resolveCapture(opts: CaptureOptions): Promise { + const devices = await listAll() + const byIndex = opts.deviceIndex !== undefined ? devices[opts.deviceIndex] : undefined + const device = byIndex ? byIndex.name : autoPick(devices) + if (!device) throw new Error('no capture device found') + + const modes = new Map() + for (const f of devices.find((d) => d.name === device)?.formats ?? []) { + if (f.pixfmt !== 'yuvs') continue + const key = `${f.width}x${f.height}` + const mode = modes.get(key) || { width: f.width, height: f.height, fps: [] } + for (const v of f.fps) if (v && !mode.fps.includes(v)) mode.fps.push(v) + modes.set(key, mode) + } + + let { width, height } = opts + let mode = modes.get(`${width}x${height}`) + if (!mode && modes.size) { + mode = [...modes.values()].reduce((a, b) => (b.width * b.height > a.width * a.height ? b : a)) + width = mode.width + height = mode.height + } + + // Highest advertised fps <= requested (small tolerance), else the lowest. + let fps = opts.fps + if (mode && mode.fps.length) { + const candidates = [...mode.fps].sort((a, b) => a - b) + const atMost = candidates.filter((f) => f <= opts.fps + 0.5) + fps = atMost.length ? atMost[atMost.length - 1] : candidates[0] + } + + return { device, width, height, fps } +} + +export class CaptureSession { + private proc: ChildProcessWithoutNullStreams | null = null + private stopping = false + + /** + * Spawn the helper and resolve with the ACTUAL frame dimensions (from its + * META line) once the first frame is captured. Frames then flow to `port`. + */ + start( + port: MessagePortMain, + opts: ResolvedCapture, + onError: (msg: string) => void + ): Promise<{ width: number; height: number }> { + this.stopping = false + + const args = [ + 'stream', + '--device', + opts.device, + '--width', + String(opts.width), + '--height', + String(opts.height), + '--fps', + String(opts.fps) + ] + console.log('[capture] spawn helper', args.join(' ')) + const proc = spawn(helperPath(), args) + this.proc = proc + + return new Promise((resolve, reject) => { + let stderr = '' + let seq = 0 + let started = false + + const startupTimeout = setTimeout(() => { + if (!started) { + this.stop() + reject(new Error('capture start timed out')) + } + }, 10000) + + const attachPump = (width: number, height: number): void => { + const frameBytes = width * height * 2 // yuvs + const frameBuf = Buffer.allocUnsafe(frameBytes) + let filled = 0 + + // Ack-based backpressure: only one frame in flight; while the renderer + // is busy the freshest frame waits in pendingBuf (older ones drop). + // Flooding the port faster than the renderer draws backs up the message + // queue and starves its event loop (input goes dead). + let rendererReady = true + let hasPending = false + const pendingBuf = Buffer.allocUnsafe(frameBytes) + + const post = (buf: Buffer): void => { + const ab = new ArrayBuffer(frameBytes) + new Uint8Array(ab).set(buf) + rendererReady = false + try { + port.postMessage({ seq: seq++, emitAbs: nowAbs(), width, height, buf: ab }) + } catch { + this.stop() + } + } + + port.on('message', () => { + if (hasPending) { + hasPending = false + post(pendingBuf) + } else { + rendererReady = true + } + }) + + proc.stdout.on('data', (chunk: Buffer) => { + let off = 0 + while (off < chunk.length) { + const take = Math.min(frameBytes - filled, chunk.length - off) + chunk.copy(frameBuf, filled, off, off + take) + filled += take + off += take + if (filled < frameBytes) break + + filled = 0 + if (rendererReady) { + post(frameBuf) + } else { + frameBuf.copy(pendingBuf) + hasPending = true + } + } + }) + } + + proc.stderr.on('data', (d: Buffer) => { + const text = d.toString() + const meta = text.match(/^META\s+(\{.*\})/m) + if (meta && !started) { + started = true + clearTimeout(startupTimeout) + try { + const { width, height } = JSON.parse(meta[1]) + console.log('[capture] actual mode', width, 'x', height) + attachPump(width, height) + resolve({ width, height }) + } catch (e) { + this.stop() + reject(e) + } + return + } + stderr += text + console.error('[capture][helper]', text.trim()) + }) + + proc.on('close', (code) => { + console.log('[capture] helper closed code=', code, 'frames=', seq) + const wasCurrent = this.proc === proc + if (wasCurrent) this.proc = null + clearTimeout(startupTimeout) + // Any unexpected end (device unplugged, signal loss) => notify the + // renderer so it doesn't keep showing a frozen frame. + if (wasCurrent && !this.stopping) { + const msg = stderr.trim() || 'capture ended (device disconnected?)' + if (started) onError(msg) + else reject(new Error(msg)) + } + }) + proc.on('error', (e) => { + clearTimeout(startupTimeout) + if (started) onError(e.message) + else reject(e) + }) + }) + } + + stop(): void { + this.stopping = true + if (this.proc) { + const proc = this.proc + this.proc = null + // SIGINT lets the helper tear down the AVFoundation session cleanly — + // hard kills can wedge the capture device / camera daemon. + try { + proc.kill('SIGINT') + } catch { + /* already gone */ + } + setTimeout(() => { + try { + proc.kill('SIGKILL') + } catch { + /* already gone */ + } + }, 1500) + } + } + + get running(): boolean { + return this.proc !== null + } +} diff --git a/desktop/src/main/events/capture.ts b/desktop/src/main/events/capture.ts new file mode 100644 index 00000000..bc004858 --- /dev/null +++ b/desktop/src/main/events/capture.ts @@ -0,0 +1,53 @@ +import { ipcMain, IpcMainInvokeEvent, MessageChannelMain } from 'electron' + +import { IpcEvents } from '../../common/ipc-events' +import { + CaptureDevice, + CaptureOptions, + CaptureSession, + listDevices, + resolveCapture +} from '../capture/engine' + +const session = new CaptureSession() + +export function registerCapture(): void { + ipcMain.handle(IpcEvents.GET_CAPTURE_DEVICES, getCaptureDevices) + ipcMain.handle(IpcEvents.START_CAPTURE, startCapture) + ipcMain.handle(IpcEvents.STOP_CAPTURE, stopCapture) +} + +async function getCaptureDevices(): Promise { + return listDevices() +} + +async function startCapture(e: IpcMainInvokeEvent, opts: CaptureOptions): Promise { + session.stop() + + // Resolve to a device (by name) + an advertised mode; the helper reports the + // ACTUAL delivered dimensions once the stream starts (the UVC stack may serve + // the signal-native mode regardless of the request). + const resolved = await resolveCapture(opts) + console.log('[capture] requested', opts, '-> resolved', resolved) + + const { port1, port2 } = new MessageChannelMain() + port1.start() + + const actual = await session.start(port1, resolved, (msg) => { + if (!e.sender.isDestroyed()) e.sender.send(IpcEvents.CAPTURE_ERROR, msg) + }) + + // Hand the renderer end of the channel to the page (preload forwards it on). + // requestId lets the renderer ignore ports from superseded start requests. + e.sender.postMessage( + IpcEvents.CAPTURE_PORT, + { width: actual.width, height: actual.height, requestId: opts.requestId }, + [port2] + ) + return true +} + +async function stopCapture(): Promise { + session.stop() + return true +} diff --git a/desktop/src/main/events/index.ts b/desktop/src/main/events/index.ts index 2ad88c16..9a967c2e 100644 --- a/desktop/src/main/events/index.ts +++ b/desktop/src/main/events/index.ts @@ -1,3 +1,4 @@ export * from './app' +export * from './capture' export * from './serial-port' export * from './updater' diff --git a/desktop/src/main/events/serial-port.ts b/desktop/src/main/events/serial-port.ts index 1e1a0b4b..0af65a1f 100644 --- a/desktop/src/main/events/serial-port.ts +++ b/desktop/src/main/events/serial-port.ts @@ -64,6 +64,7 @@ async function closeSerialPort(): Promise { } async function sendKeyboard(_: IpcMainInvokeEvent, report: number[]): Promise { + console.log('[input] SEND_KEYBOARD') try { await device.sendKeyboardData(report) } catch (error) { @@ -71,7 +72,9 @@ async function sendKeyboard(_: IpcMainInvokeEvent, report: number[]): Promise { + if (mouseLog++ % 30 === 0) console.log('[input] SEND_MOUSE x30') try { await device.sendMouseData(report) } catch (error) { diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts index d3469ee7..e336a1f6 100644 --- a/desktop/src/main/index.ts +++ b/desktop/src/main/index.ts @@ -19,7 +19,10 @@ function createWindow(): void { ...(process.platform === 'linux' ? { icon } : {}), webPreferences: { preload: join(__dirname, '../preload/index.js'), - sandbox: false + sandbox: false, + // Keep the capture canvas (rAF-driven) updating when the window is not + // focused — a KVM is often in the background while you use the target. + backgroundThrottling: false } }) @@ -28,6 +31,11 @@ function createWindow(): void { mainWindow.show() }) + // TEMP (Phase 1 debug): surface renderer console in the terminal + mainWindow.webContents.on('console-message', (_e, _lvl, message) => { + if (/capture|webgl|port|gl error/i.test(message)) console.log('[renderer]', message) + }) + mainWindow.webContents.setWindowOpenHandler((details) => { shell.openExternal(details.url) return { action: 'deny' } @@ -54,6 +62,7 @@ app.whenReady().then(() => { events.registerApp() events.registerSerialPort() + events.registerCapture() createWindow() diff --git a/desktop/src/preload/index.ts b/desktop/src/preload/index.ts index 8fbf134e..887a747d 100644 --- a/desktop/src/preload/index.ts +++ b/desktop/src/preload/index.ts @@ -1,5 +1,13 @@ import { electronAPI } from '@electron-toolkit/preload' -import { contextBridge } from 'electron' +import { contextBridge, ipcRenderer } from 'electron' + +import { IpcEvents } from '../common/ipc-events' + +// MessagePorts cannot cross the contextBridge, so forward the capture channel's +// port directly into the page via window.postMessage (with transfer). +ipcRenderer.on(IpcEvents.CAPTURE_PORT, (e, meta) => { + window.postMessage({ type: 'capture-port', meta }, '*', e.ports) +}) // Custom APIs for renderer const api = {} diff --git a/desktop/src/renderer/src/App.tsx b/desktop/src/renderer/src/App.tsx index 85edcaf2..dbd579de 100644 --- a/desktop/src/renderer/src/App.tsx +++ b/desktop/src/renderer/src/App.tsx @@ -1,7 +1,7 @@ -import { ReactElement, useEffect, useState } from 'react' -import { Result, Spin } from 'antd' +import { ReactElement, useEffect, useRef, useState } from 'react' +import { message, Result, Spin } from 'antd' import clsx from 'clsx' -import { useAtomValue, useSetAtom } from 'jotai' +import { useAtom, useAtomValue } from 'jotai' import { useTranslation } from 'react-i18next' import { useMediaQuery } from 'react-responsive' @@ -12,6 +12,7 @@ import { Menu } from '@renderer/components/menu' import { Mouse } from '@renderer/components/mouse' import { VirtualKeyboard } from '@renderer/components/virtual-keyboard' import { + captureModeAtom, resolutionAtom, serialPortStateAtom, videoScaleAtom, @@ -19,9 +20,10 @@ import { } from '@renderer/jotai/device' import { isKeyboardEnableAtom } from '@renderer/jotai/keyboard' import { mouseModeAtom, mouseStyleAtom } from '@renderer/jotai/mouse' +import { ffmpegCamera } from '@renderer/libs/capture/ffmpeg-camera' import { camera } from '@renderer/libs/media/camera' import { requestCameraPermission } from '@renderer/libs/media/permission' -import { getVideoResolution } from '@renderer/libs/storage' +import * as storage from '@renderer/libs/storage' import type { Resolution } from '@renderer/types' type State = 'loading' | 'success' | 'failed' @@ -31,29 +33,109 @@ const App = (): ReactElement => { const isBigScreen = useMediaQuery({ minWidth: 850 }) const videoScale = useAtomValue(videoScaleAtom) - const videoState = useAtomValue(videoStateAtom) + const [videoState, setVideoState] = useAtom(videoStateAtom) + const [captureMode, setCaptureMode] = useAtom(captureModeAtom) const serialPortState = useAtomValue(serialPortStateAtom) const mouseMode = useAtomValue(mouseModeAtom) const mouseStyle = useAtomValue(mouseStyleAtom) const isKeyboardEnable = useAtomValue(isKeyboardEnableAtom) - const setResolution = useSetAtom(resolutionAtom) + const [resolution, setResolution] = useAtom(resolutionAtom) const [state, setState] = useState('loading') + const prevCaptureMode = useRef(false) useEffect(() => { - const resolution = getVideoResolution() + const resolution = storage.getVideoResolution() if (resolution) { setResolution(resolution) } + setCaptureMode(storage.getCaptureMode()) requestMediaPermissions(resolution) return (): void => { camera.close() + ffmpegCamera.close() window.electron.ipcRenderer.invoke(IpcEvents.CLOSE_SERIAL_PORT) } }, []) + useEffect(() => { + console.log( + 'capture-debug STATE captureMode=', + captureMode, + 'videoState=', + videoState, + 'serialPortState=', + serialPortState + ) + }, [captureMode, videoState, serialPortState]) + + // Drive the FFmpeg uncompressed-capture path when capture mode is on. + // Depends on `resolution` so a resolution change cleanly restarts the session. + useEffect(() => { + if (state !== 'success') return + + const wasCapture = prevCaptureMode.current + prevCaptureMode.current = captureMode + + if (captureMode) { + camera.close() + const canvas = document.getElementById('video') as HTMLCanvasElement | null + if (!canvas) return + ffmpegCamera + .open({ + canvas, + width: resolution.width, + height: resolution.height, + fps: fpsFor(resolution.width, resolution.height), + onError: (msg) => failCapture(msg) + }) + .then(() => setVideoState('connected')) + .catch((err) => failCapture(err instanceof Error ? err.message : String(err))) + return (): void => ffmpegCamera.close() + } + + ffmpegCamera.close() + // Ticked -> unticked: restore the regular getUserMedia camera (the swapped-in + //