diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e473fc0ae2..b5c89bac44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,15 @@ jobs: if: runner.os == 'macOS' run: mise test-desktop-macos-capture + - name: Test Windows game-capture helper + if: runner.os == 'Windows' + shell: pwsh + run: | + $probe = "apps/desktop/native/windows-capture-probe" + cmake -S $probe -B "$probe/build" -A x64 + cmake --build "$probe/build" --config RelWithDebInfo + ctest --test-dir "$probe/build" -C RelWithDebInfo --output-on-failure + - name: Build desktop bundle run: mise desktop-build diff --git a/NOTICE b/NOTICE index 9ae9590466..7d97ad4683 100644 --- a/NOTICE +++ b/NOTICE @@ -105,11 +105,21 @@ Desktop Components: Foundation and Electron contributors - LiveKit client SDK for Swift (https://github.com/livekit/client-sdk-swift) - Apache License 2.0; Copyright 2023 LiveKit, Inc. +- Chatto fork of the LiveKit client SDK for C++ + (https://github.com/chattocorp/client-sdk-cpp) - Apache License 2.0; + Copyright 2025-2026 LiveKit, Inc. +- Chatto fork of the LiveKit client SDK for Rust + (https://github.com/chattocorp/rust-sdks) - Apache License 2.0; + Copyright 2026 LiveKit, Inc. +- FFmpeg nv-codec-headers (https://github.com/FFmpeg/nv-codec-headers) - + MIT License; NVIDIA NVENC API header copyright NVIDIA Corporation - LiveKit UniFFI XCFramework (https://github.com/livekit/livekit-uniffi-xcframework) - Apache License 2.0 - LiveKit WebRTC XCFramework (https://github.com/livekit/webrtc-xcframework) - MIT License; Copyright 2021 WebRTC SDKs; bundled WebRTC is BSD 3-Clause and includes third-party components under their respective licences +- Microsoft Visual C++ Redistributable runtime libraries - distributed under + the Microsoft Visual Studio license terms - Swift Protobuf (https://github.com/apple/swift-protobuf) - Apache License 2.0 - Chromium (https://www.chromium.org/) - BSD 3-Clause License and other licences included in Electron's bundled `LICENSES.chromium.html` diff --git a/apps/desktop/README.md b/apps/desktop/README.md index 2c272baf13..17e5e815b8 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -41,11 +41,14 @@ mise desktop-build The build task first produces the frontend, then packages the host-platform bundle beneath `apps/desktop/dist/`. macOS builds include the native -ScreenCaptureKit game-capture helper and its pinned LiveKit frameworks. CI -checks and packages macOS, Windows, and Linux bundles, verifies the nested macOS -helper, and validates the complete app signature. The current release archives -use ad-hoc signing, so macOS may warn about or block them until production -Developer ID signing and notarisation are added. +ScreenCaptureKit game-capture helper and its pinned LiveKit frameworks. Windows +x64 builds include the Windows Graphics Capture/WASAPI helper and pinned +LiveKit C++ runtime plus its required Visual C++ redistributable DLLs. CI +checks and packages macOS, Windows, and Linux bundles +and tests the native helpers. The current release archives are not +production-signed: macOS may warn or block them, and Windows Smart App Control +may block a newly built helper, until trusted signing and macOS notarisation are +added. Electron handles camera, microphone, and notification permission requests only for the fixed app origin. Screen sharing presents a native source picker. diff --git a/apps/desktop/game_capture.mjs b/apps/desktop/game_capture.mjs index d43d05ea46..2ff1dd9d78 100644 --- a/apps/desktop/game_capture.mjs +++ b/apps/desktop/game_capture.mjs @@ -15,6 +15,48 @@ const sourcePreviewMaximumBytes = 512 * 1024; const sourcePreviewFrameMaximumBytes = 16 * 1024 * 1024; const sourcePreviewMaximumCount = 64; const sourceOfferLifetimeMilliseconds = 2 * 60 * 1000; +const encodedPreviewFrameMaximumBytes = 16 * 1024 * 1024; +const encodedPreviewHeaderBytes = 16; + +/** Parse length-delimited Annex-B H.264 frames from the helper's local preview pipe. */ +export class EncodedPreviewFrameParser { + #pending = Buffer.alloc(0); + + push(chunk) { + if (!Buffer.isBuffer(chunk)) { + throw new Error("The capture helper returned invalid preview data."); + } + this.#pending = + this.#pending.length === 0 ? chunk : Buffer.concat([this.#pending, chunk]); + const frames = []; + for (;;) { + if (this.#pending.length < encodedPreviewHeaderBytes) break; + if (this.#pending.toString("ascii", 0, 4) !== "CTPV") { + throw new Error("The capture helper returned invalid preview data."); + } + const encodedSize = this.#pending.readUInt32LE(4); + const keyFrame = (encodedSize & 0x8000_0000) !== 0; + const size = encodedSize & 0x7fff_ffff; + if (size === 0 || size > encodedPreviewFrameMaximumBytes) { + throw new Error("The capture helper returned invalid preview data."); + } + const recordSize = encodedPreviewHeaderBytes + size; + if (this.#pending.length < recordSize) break; + frames.push({ + timestampUs: Number(this.#pending.readBigInt64LE(8)), + keyFrame, + data: Uint8Array.from( + this.#pending.subarray(encodedPreviewHeaderBytes, recordSize), + ), + }); + this.#pending = this.#pending.subarray(recordSize); + } + if (this.#pending.length > encodedPreviewFrameMaximumBytes + encodedPreviewHeaderBytes) { + throw new Error("The capture helper returned too much preview data."); + } + return frames; + } +} /** Whether the host macOS version can launch the bundled capture helper. */ export function supportsMacOSGameCapture(systemVersion) { @@ -28,6 +70,20 @@ export function supportsMacOSGameCapture(systemVersion) { ); } +/** Whether Windows Graphics Capture and process-loopback audio are available. */ +export function supportsWindowsGameCapture(systemVersion) { + if (typeof systemVersion !== "string") return false; + const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(systemVersion); + if (!match) return false; + const [major, minor, build] = match.slice(1).map(Number); + return ( + Number.isSafeInteger(major) && + Number.isSafeInteger(minor) && + Number.isSafeInteger(build) && + (major > 10 || (major === 10 && build >= 19041)) + ); +} + /** * Parse the trusted macOS helper response. The temporary native IDs in this * result must be replaced with renderer-facing offers before crossing IPC. @@ -105,6 +161,64 @@ export function parseMacOSGameCaptureSources(output) { }; } +/** Parse the Windows helper manifest and attach Electron-owned JPEG previews. */ +export function parseWindowsGameCaptureSources(output, previews = new Map()) { + if ( + !Buffer.isBuffer(output) || + output.length === 0 || + output.length > 1024 * 1024 + ) { + throw new Error("The capture helper returned an unsupported source list."); + } + let response; + try { + response = JSON.parse(output.toString("utf8")); + } catch { + throw new Error("The capture helper returned an unsupported source list."); + } + if ( + response?.protocolVersion !== 1 || + !Array.isArray(response.sources) || + response.sources.length > sourcePreviewMaximumCount + ) { + throw new Error("The capture helper returned an unsupported source list."); + } + return { + protocolVersion: 1, + sources: response.sources.map((source) => { + validateMacOSCaptureSource(source); + const preview = previews.get(source.nativeID) ?? new Uint8Array(); + if ( + !(preview instanceof Uint8Array) || + preview.byteLength > sourcePreviewMaximumBytes + ) { + throw new Error("The capture helper returned too much preview data."); + } + if (source.kind === "display") { + return { + id: `display:${source.nativeID}`, + kind: "display", + displayIndex: source.displayIndex, + isMainDisplay: source.isMainDisplay, + width: source.width, + height: source.height, + preview: Uint8Array.from(preview), + }; + } + return { + id: `window:${source.nativeID}`, + kind: "window", + applicationName: source.applicationName, + bundleIdentifier: source.bundleIdentifier, + title: source.title, + width: source.width, + height: source.height, + preview: Uint8Array.from(preview), + }; + }), + }; +} + function validateMacOSCaptureSource(source) { if ( !source || @@ -239,7 +353,127 @@ export function parseGameCapturePublisherRequest(request) { /** Parse one lifecycle status line emitted by the native helper. */ export function parseGameCapturePublisherStatus(line) { const value = JSON.parse(line); - if (value?.protocolVersion !== 1 || value?.kind !== "started") { + if (value?.protocolVersion !== 1) { + throw new Error( + "The capture helper returned an unsupported publisher status.", + ); + } + if (value.kind === "metrics") { + const integerFields = [ + "submittedFrames", + "publishedFrames", + "droppedFrames", + "sourceWidth", + "sourceHeight", + "dimensionChanges", + "outboundStreams", + "activeOutboundStreams", + "framesEncoded", + "framesSent", + "bytesSent", + "cpuLimitedStreams", + "bandwidthLimitedStreams", + "powerEfficientStreams", + ]; + const numberFields = [ + "captureFps", + "publishFps", + "averageReadbackMs", + "averageScaleMs", + "averagePublishMs", + "lastPublishMs", + "minimumActiveOutboundFps", + "maximumActiveOutboundFps", + "targetBitrate", + "averageEncodeMs", + ]; + if ( + !integerFields.every( + (field) => Number.isSafeInteger(value[field]) && value[field] >= 0, + ) || + !numberFields.every( + (field) => Number.isFinite(value[field]) && value[field] >= 0, + ) || + !["wgc-window", "wgc-monitor", "dxgi-display"].includes( + value.captureBackend, + ) || + typeof value.rtcStatsAvailable !== "boolean" || + !validOptionalEncoderMetrics(value) || + !validOptionalHardwareEncoderMetrics(value) || + !validOptionalNetworkMetrics(value) + ) { + throw new Error("The capture helper returned invalid publisher metrics."); + } + return { + kind: "metrics", + submittedFrames: value.submittedFrames, + publishedFrames: value.publishedFrames, + droppedFrames: value.droppedFrames, + captureFps: value.captureFps, + publishFps: value.publishFps, + averageReadbackMs: value.averageReadbackMs, + averageScaleMs: value.averageScaleMs, + averageGpuCopySubmitMs: value.averageGpuCopySubmitMs ?? 0, + averageGpuConversionSubmitMs: + value.averageGpuConversionSubmitMs ?? 0, + averageEncoderSubmitMs: value.averageEncoderSubmitMs ?? 0, + averageBitstreamWaitMs: value.averageBitstreamWaitMs ?? 0, + averagePublishMs: value.averagePublishMs, + averageHardwareEncodeMs: value.averageHardwareEncodeMs ?? 0, + hardwareEncoderImplementation: + value.hardwareEncoderImplementation ?? "", + requestedEncoderBitrate: value.requestedEncoderBitrate ?? 0, + appliedEncoderBitrate: value.appliedEncoderBitrate ?? 0, + actualHardwareBitrate: value.actualHardwareBitrate ?? 0, + encoderRateControlMode: value.encoderRateControlMode ?? 0, + requestedEncoderFps: value.requestedEncoderFps ?? 0, + hardwareEncodedFrames: value.hardwareEncodedFrames ?? 0, + hardwareEncodedBytes: value.hardwareEncodedBytes ?? 0, + hardwareKeyFrames: value.hardwareKeyFrames ?? 0, + hardwareEncodedWidth: value.hardwareEncodedWidth ?? 0, + hardwareEncodedHeight: value.hardwareEncodedHeight ?? 0, + encoderResolutionChanges: value.encoderResolutionChanges ?? 0, + lastPublishMs: value.lastPublishMs, + sourceWidth: value.sourceWidth, + sourceHeight: value.sourceHeight, + dimensionChanges: value.dimensionChanges, + captureBackend: value.captureBackend, + rtcStatsAvailable: value.rtcStatsAvailable, + outboundStreams: value.outboundStreams, + activeOutboundStreams: value.activeOutboundStreams, + minimumActiveOutboundFps: value.minimumActiveOutboundFps, + maximumActiveOutboundFps: value.maximumActiveOutboundFps, + framesEncoded: value.framesEncoded, + framesSent: value.framesSent, + bytesSent: value.bytesSent, + retransmittedPacketsSent: value.retransmittedPacketsSent ?? 0, + retransmittedBytesSent: value.retransmittedBytesSent ?? 0, + nackCount: value.nackCount ?? 0, + pliCount: value.pliCount ?? 0, + targetBitrate: value.targetBitrate, + averageEncodeMs: value.averageEncodeMs, + encodedWidth: value.encodedWidth ?? 0, + encodedHeight: value.encodedHeight ?? 0, + averageQp: value.averageQp ?? 0, + encoderImplementation: value.encoderImplementation ?? "", + cpuLimitedStreams: value.cpuLimitedStreams, + bandwidthLimitedStreams: value.bandwidthLimitedStreams, + powerEfficientStreams: value.powerEfficientStreams, + remoteInboundStatsAvailable: + value.remoteInboundStatsAvailable ?? false, + remotePacketsLost: value.remotePacketsLost ?? 0, + remoteJitterSeconds: value.remoteJitterSeconds ?? 0, + remoteFractionLost: value.remoteFractionLost ?? 0, + remoteRoundTripTimeMs: value.remoteRoundTripTimeMs ?? 0, + candidatePairStatsAvailable: + value.candidatePairStatsAvailable ?? false, + availableOutgoingBitrate: value.availableOutgoingBitrate ?? 0, + currentRoundTripTimeMs: value.currentRoundTripTimeMs ?? 0, + packetsDiscardedOnSend: value.packetsDiscardedOnSend ?? 0, + bytesDiscardedOnSend: value.bytesDiscardedOnSend ?? 0, + }; + } + if (value.kind !== "started") { throw new Error( "The capture helper returned an unsupported publisher status.", ); @@ -262,6 +496,119 @@ export function parseGameCapturePublisherStatus(line) { }; } +function validOptionalEncoderMetrics(value) { + const fields = [ + value.encodedWidth, + value.encodedHeight, + value.averageQp, + value.encoderImplementation, + ]; + if (fields.every((field) => field === undefined)) return true; + return ( + Number.isSafeInteger(value.encodedWidth) && + value.encodedWidth >= 0 && + Number.isSafeInteger(value.encodedHeight) && + value.encodedHeight >= 0 && + Number.isFinite(value.averageQp) && + value.averageQp >= 0 && + typeof value.encoderImplementation === "string" && + value.encoderImplementation.length <= 256 + ); +} + +function validOptionalHardwareEncoderMetrics(value) { + const fields = [ + value.averageHardwareEncodeMs, + value.hardwareEncoderImplementation, + ]; + if (fields.every((field) => field === undefined)) return true; + return ( + Number.isFinite(value.averageHardwareEncodeMs) && + value.averageHardwareEncodeMs >= 0 && + (value.averageGpuCopySubmitMs === undefined || + (Number.isFinite(value.averageGpuCopySubmitMs) && + value.averageGpuCopySubmitMs >= 0)) && + (value.averageGpuConversionSubmitMs === undefined || + (Number.isFinite(value.averageGpuConversionSubmitMs) && + value.averageGpuConversionSubmitMs >= 0)) && + (value.averageEncoderSubmitMs === undefined || + (Number.isFinite(value.averageEncoderSubmitMs) && + value.averageEncoderSubmitMs >= 0)) && + (value.averageBitstreamWaitMs === undefined || + (Number.isFinite(value.averageBitstreamWaitMs) && + value.averageBitstreamWaitMs >= 0)) && + typeof value.hardwareEncoderImplementation === "string" && + value.hardwareEncoderImplementation.length <= 256 && + (value.requestedEncoderBitrate === undefined || + (Number.isSafeInteger(value.requestedEncoderBitrate) && + value.requestedEncoderBitrate >= 0)) && + (value.appliedEncoderBitrate === undefined || + (Number.isSafeInteger(value.appliedEncoderBitrate) && + value.appliedEncoderBitrate >= 0)) && + (value.actualHardwareBitrate === undefined || + (Number.isFinite(value.actualHardwareBitrate) && + value.actualHardwareBitrate >= 0)) && + (value.encoderRateControlMode === undefined || + (Number.isSafeInteger(value.encoderRateControlMode) && + value.encoderRateControlMode >= 0)) && + (value.requestedEncoderFps === undefined || + (Number.isFinite(value.requestedEncoderFps) && + value.requestedEncoderFps >= 0)) && + (value.hardwareEncodedFrames === undefined || + (Number.isSafeInteger(value.hardwareEncodedFrames) && + value.hardwareEncodedFrames >= 0)) && + (value.hardwareEncodedBytes === undefined || + (Number.isSafeInteger(value.hardwareEncodedBytes) && + value.hardwareEncodedBytes >= 0)) && + (value.hardwareKeyFrames === undefined || + (Number.isSafeInteger(value.hardwareKeyFrames) && + value.hardwareKeyFrames >= 0)) && + (value.hardwareEncodedWidth === undefined || + (Number.isSafeInteger(value.hardwareEncodedWidth) && + value.hardwareEncodedWidth >= 0)) && + (value.hardwareEncodedHeight === undefined || + (Number.isSafeInteger(value.hardwareEncodedHeight) && + value.hardwareEncodedHeight >= 0)) && + (value.encoderResolutionChanges === undefined || + (Number.isSafeInteger(value.encoderResolutionChanges) && + value.encoderResolutionChanges >= 0)) + ); +} + +function validOptionalNetworkMetrics(value) { + const fields = [ + value.remoteInboundStatsAvailable, + value.candidatePairStatsAvailable, + ]; + if (fields.every((field) => field === undefined)) return true; + const nonNegativeIntegers = [ + value.retransmittedPacketsSent, + value.retransmittedBytesSent, + value.nackCount, + value.pliCount, + value.packetsDiscardedOnSend, + value.bytesDiscardedOnSend, + ]; + const nonNegativeNumbers = [ + value.remoteJitterSeconds, + value.remoteFractionLost, + value.remoteRoundTripTimeMs, + value.availableOutgoingBitrate, + value.currentRoundTripTimeMs, + ]; + return ( + typeof value.remoteInboundStatsAvailable === "boolean" && + typeof value.candidatePairStatsAvailable === "boolean" && + Number.isSafeInteger(value.remotePacketsLost) && + nonNegativeIntegers.every( + (field) => Number.isSafeInteger(field) && field >= 0, + ) && + nonNegativeNumbers.every( + (field) => Number.isFinite(field) && field >= 0, + ) + ); +} + function validString(value, maximumLength) { if ( typeof value !== "string" || diff --git a/apps/desktop/game_capture.test.mjs b/apps/desktop/game_capture.test.mjs index bc18c9bd13..8c88c1aef0 100644 --- a/apps/desktop/game_capture.test.mjs +++ b/apps/desktop/game_capture.test.mjs @@ -4,14 +4,36 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + EncodedPreviewFrameParser, MacOSGameCaptureSourceOffers, parseGameCapturePublisherRequest, parseGameCapturePublisherStatus, parseMacOSGameCaptureSourceId, parseMacOSGameCaptureSources, + parseWindowsGameCaptureSources, supportsMacOSGameCapture, + supportsWindowsGameCapture, } from "./game_capture.mjs"; +test("parses fragmented local H.264 preview frames", () => { + const parser = new EncodedPreviewFrameParser(); + const payload = Buffer.from([0, 0, 0, 1, 0x65, 1, 2, 3]); + const header = Buffer.alloc(16); + header.write("CTPV"); + header.writeUInt32LE(payload.length + 0x8000_0000, 4); + header.writeBigInt64LE(123456n, 8); + const record = Buffer.concat([header, payload]); + + assert.deepEqual(parser.push(record.subarray(0, 11)), []); + assert.deepEqual(parser.push(record.subarray(11)), [ + { + timestampUs: 123456, + keyFrame: true, + data: Uint8Array.from(payload), + }, + ]); +}); + test("advertises native game capture only on supported macOS versions", () => { assert.equal(supportsMacOSGameCapture("12.7.6"), false); assert.equal(supportsMacOSGameCapture("14.7.6"), false); @@ -24,6 +46,73 @@ test("advertises native game capture only on supported macOS versions", () => { assert.equal(supportsMacOSGameCapture(undefined), false); }); +test("advertises native game capture only on supported Windows versions", () => { + assert.equal(supportsWindowsGameCapture("10.0.18363"), false); + assert.equal(supportsWindowsGameCapture("10.0.19041"), true); + assert.equal(supportsWindowsGameCapture("10.0.26100"), true); + assert.equal(supportsWindowsGameCapture("11.0.1"), true); + assert.equal(supportsWindowsGameCapture("Version 10.0.26100"), false); +}); + +test("parses Windows window and display sources with bounded previews", () => { + const preview = Uint8Array.from([0xff, 0xd8, 0xff]); + assert.deepEqual( + parseWindowsGameCaptureSources( + Buffer.from( + JSON.stringify({ + protocolVersion: 1, + sources: [ + { + kind: "display", + nativeID: 7, + displayIndex: 1, + isMainDisplay: true, + width: 2560, + height: 1440, + previewByteLength: 0, + }, + { + kind: "window", + nativeID: 42, + applicationName: "game.exe", + bundleIdentifier: `windows-sha256:${"a".repeat(64)}`, + title: "Example Game", + width: 1920, + height: 1080, + previewByteLength: 0, + }, + ], + }), + ), + new Map([[42, preview]]), + ), + { + protocolVersion: 1, + sources: [ + { + id: "display:7", + kind: "display", + displayIndex: 1, + isMainDisplay: true, + width: 2560, + height: 1440, + preview: new Uint8Array(), + }, + { + id: "window:42", + kind: "window", + applicationName: "game.exe", + bundleIdentifier: `windows-sha256:${"a".repeat(64)}`, + title: "Example Game", + width: 1920, + height: 1080, + preview, + }, + ], + }, + ); +}); + test("parses macOS window and display capture sources", () => { const windowPreview = Buffer.from([0xff, 0xd8, 0xff]); const displayPreview = Buffer.from([0xff, 0xd8]); @@ -258,6 +347,128 @@ test("parses native publisher lifecycle status", () => { ); }); +test("parses native publisher performance metrics", () => { + assert.deepEqual( + parseGameCapturePublisherStatus( + JSON.stringify({ + protocolVersion: 1, + kind: "metrics", + submittedFrames: 120, + publishedFrames: 58, + droppedFrames: 61, + captureFps: 59.8, + publishFps: 28.9, + averageReadbackMs: 3.25, + averageScaleMs: 14.5, + averagePublishMs: 16.75, + averageHardwareEncodeMs: 2.4, + averageGpuCopySubmitMs: 0.12, + averageGpuConversionSubmitMs: 0.18, + averageEncoderSubmitMs: 0.09, + averageBitstreamWaitMs: 1.75, + hardwareEncoderImplementation: "NVIDIA H.264 Encoder MFT", + requestedEncoderBitrate: 3500000, + appliedEncoderBitrate: 3400000, + actualHardwareBitrate: 3325000.5, + encoderRateControlMode: 0, + requestedEncoderFps: 60, + hardwareEncodedFrames: 118, + hardwareEncodedBytes: 1456789, + hardwareKeyFrames: 2, + hardwareEncodedWidth: 1280, + hardwareEncodedHeight: 720, + encoderResolutionChanges: 1, + lastPublishMs: 17.1, + sourceWidth: 2560, + sourceHeight: 1440, + dimensionChanges: 2, + captureBackend: "dxgi-display", + rtcStatsAvailable: true, + outboundStreams: 2, + activeOutboundStreams: 1, + minimumActiveOutboundFps: 29.9, + maximumActiveOutboundFps: 29.9, + framesEncoded: 116, + framesSent: 116, + bytesSent: 1234567, + targetBitrate: 4000000, + averageEncodeMs: 4.2, + encodedWidth: 1920, + encodedHeight: 1080, + averageQp: 32.5, + encoderImplementation: "OpenH264", + cpuLimitedStreams: 0, + bandwidthLimitedStreams: 0, + powerEfficientStreams: 1, + }), + ), + { + kind: "metrics", + submittedFrames: 120, + publishedFrames: 58, + droppedFrames: 61, + captureFps: 59.8, + publishFps: 28.9, + averageReadbackMs: 3.25, + averageScaleMs: 14.5, + averagePublishMs: 16.75, + averageHardwareEncodeMs: 2.4, + averageGpuCopySubmitMs: 0.12, + averageGpuConversionSubmitMs: 0.18, + averageEncoderSubmitMs: 0.09, + averageBitstreamWaitMs: 1.75, + hardwareEncoderImplementation: "NVIDIA H.264 Encoder MFT", + requestedEncoderBitrate: 3500000, + appliedEncoderBitrate: 3400000, + actualHardwareBitrate: 3325000.5, + encoderRateControlMode: 0, + requestedEncoderFps: 60, + hardwareEncodedFrames: 118, + hardwareEncodedBytes: 1456789, + hardwareKeyFrames: 2, + hardwareEncodedWidth: 1280, + hardwareEncodedHeight: 720, + encoderResolutionChanges: 1, + lastPublishMs: 17.1, + sourceWidth: 2560, + sourceHeight: 1440, + dimensionChanges: 2, + captureBackend: "dxgi-display", + rtcStatsAvailable: true, + outboundStreams: 2, + activeOutboundStreams: 1, + minimumActiveOutboundFps: 29.9, + maximumActiveOutboundFps: 29.9, + framesEncoded: 116, + framesSent: 116, + bytesSent: 1234567, + retransmittedPacketsSent: 0, + retransmittedBytesSent: 0, + nackCount: 0, + pliCount: 0, + targetBitrate: 4000000, + averageEncodeMs: 4.2, + encodedWidth: 1920, + encodedHeight: 1080, + averageQp: 32.5, + encoderImplementation: "OpenH264", + cpuLimitedStreams: 0, + bandwidthLimitedStreams: 0, + powerEfficientStreams: 1, + remoteInboundStatsAvailable: false, + remotePacketsLost: 0, + remoteJitterSeconds: 0, + remoteFractionLost: 0, + remoteRoundTripTimeMs: 0, + candidatePairStatsAvailable: false, + availableOutgoingBitrate: 0, + currentRoundTripTimeMs: 0, + packetsDiscardedOnSend: 0, + bytesDiscardedOnSend: 0, + }, + ); +}); + function sourcePreviewFrame(manifest, previews = []) { const manifestBuffer = Buffer.from(JSON.stringify(manifest)); const prefix = Buffer.alloc(4); diff --git a/apps/desktop/main.mjs b/apps/desktop/main.mjs index 36e59eca50..bd0d0ca0f0 100644 --- a/apps/desktop/main.mjs +++ b/apps/desktop/main.mjs @@ -24,11 +24,14 @@ import { gameCaptureListSourcesChannel, gameCapturePublisherChannel, gameCaptureStartChannel, + EncodedPreviewFrameParser, MacOSGameCaptureSourceOffers, parseGameCapturePublisherRequest, parseGameCapturePublisherStatus, parseMacOSGameCaptureSources, + parseWindowsGameCaptureSources, supportsMacOSGameCapture, + supportsWindowsGameCapture, } from "./game_capture.mjs"; import { hasAppOrigin, isDesktopPermissionAllowed } from "./security.mjs"; @@ -43,6 +46,7 @@ let activeGameCaptureSourceList; const gameCaptureSourceOffers = new MacOSGameCaptureSourceOffers(); const macOSCaptureProbeListFlag = "--chatto-macos-capture-probe-list"; const macOSCapturePocFlag = "--chatto-macos-capture-poc"; +const windowsCaptureProbeListFlag = "--chatto-windows-capture-probe-list"; protocol.registerSchemesAsPrivileged([ { @@ -73,6 +77,24 @@ if (!app.requestSingleInstanceLock()) { async function start() { await app.whenReady(); + if (process.argv.includes(windowsCaptureProbeListFlag)) { + if (process.platform !== "win32" || !app.isPackaged) { + throw new Error("The Windows capture probe requires a packaged Windows app."); + } + const response = await listNativeGameCaptureSources(); + console.log( + JSON.stringify({ + protocolVersion: response.protocolVersion, + sourceCount: response.sources.length, + previewCount: response.sources.filter( + (source) => source.preview.byteLength > 0, + ).length, + }), + ); + app.quit(); + return; + } + if (process.argv.includes(macOSCapturePocFlag)) { try { await runMacOSCapturePoc(); @@ -294,9 +316,9 @@ function runMacOSCaptureHelper(arguments_) { }); } -function runMacOSCaptureHelperBinary(arguments_) { +function runNativeCaptureHelperBinary(arguments_) { cancelActiveGameCaptureSourceList(); - const executable = macOSCaptureHelperExecutable(); + const executable = nativeCaptureHelperExecutable(); return new Promise((resolve, reject) => { const child = spawn(executable, arguments_, { stdio: ["ignore", "pipe", "pipe"], @@ -342,19 +364,19 @@ function runMacOSCaptureHelperBinary(arguments_) { if (timedOut) { reject( new Error( - "The macOS capture helper timed out while listing sources.", + "The native capture helper timed out while listing sources.", ), ); return; } if (stdoutTooLarge) { - reject(new Error("The macOS capture helper returned too much data.")); + reject(new Error("The native capture helper returned too much data.")); return; } if (stderrTooLarge) { reject( new Error( - "The macOS capture helper produced too much diagnostic output.", + "The native capture helper produced too much diagnostic output.", ), ); return; @@ -367,8 +389,8 @@ function runMacOSCaptureHelperBinary(arguments_) { new Error( stderr.trim() || (signal - ? `The macOS capture helper exited after signal ${signal}.` - : `The macOS capture helper exited with status ${code}.`), + ? `The native capture helper exited after signal ${signal}.` + : `The native capture helper exited with status ${code}.`), ), ); }); @@ -410,6 +432,20 @@ function macOSCaptureHelperExecutable() { ); } +function windowsCaptureHelperExecutable() { + return path.join( + process.resourcesPath, + "windows-capture", + "chatto-windows-capture-probe.exe", + ); +} + +function nativeCaptureHelperExecutable() { + return process.platform === "win32" + ? windowsCaptureHelperExecutable() + : macOSCaptureHelperExecutable(); +} + async function runMacOSCaptureProbeList() { if (process.platform !== "darwin" || !app.isPackaged) { throw new Error("The macOS capture probe requires a packaged macOS app."); @@ -481,9 +517,8 @@ function configureGameCaptureIPC() { ); } - const output = await runMacOSCaptureHelperBinary(["list-previews"]); return gameCaptureSourceOffers.replace( - parseMacOSGameCaptureSources(output), + await listNativeGameCaptureSources(), ); }); @@ -548,7 +583,7 @@ function startGameCaptureSession(source, publisherRequest, port) { ); } const child = spawn( - macOSCaptureHelperExecutable(), + nativeCaptureHelperExecutable(), [ "publish", ...sourceArguments, @@ -559,10 +594,12 @@ function startGameCaptureSession(source, publisherRequest, port) { "--max-height", "1080", ], - { stdio: ["pipe", "pipe", "pipe"] }, + { stdio: ["pipe", "pipe", "pipe", "pipe"] }, ); let stdout = ""; let stderr = ""; + let diagnosticStderr = ""; + const captureDiagnostics = []; let stopping = false; let forceStopTimer; const session = { child, port, stop }; @@ -585,7 +622,20 @@ function startGameCaptureSession(source, publisherRequest, port) { if (lineEnd < 0) break; const line = stdout.slice(0, lineEnd).trim(); stdout = stdout.slice(lineEnd + 1); - if (line) port.postMessage(parseGameCapturePublisherStatus(line)); + if (line) { + const status = parseGameCapturePublisherStatus(line); + if (status.kind === "metrics") { + console.info( + "[Chatto Desktop] Native screen-share publisher metrics", + status, + ); + } + port.postMessage( + status.kind === "started" + ? { ...status, localPreviewAvailable: process.platform === "win32" } + : status, + ); + } } } catch { port.postMessage({ @@ -595,11 +645,40 @@ function startGameCaptureSession(source, publisherRequest, port) { stop(); } }); + const previewParser = new EncodedPreviewFrameParser(); + child.stdio[3].on("data", (chunk) => { + try { + for (const frame of previewParser.push(chunk)) { + port.postMessage({ kind: "preview-frame", ...frame }); + } + } catch { + port.postMessage({ + kind: "error", + message: "The native screen-share helper returned invalid preview data.", + }); + stop(); + } + }); child.stderr.setEncoding("utf8"); child.stderr.on("data", (chunk) => { stderr += chunk; if (stderr.length > 64 * 1024) stderr = stderr.slice(-64 * 1024); + diagnosticStderr += chunk; + for (;;) { + const lineEnd = diagnosticStderr.indexOf("\n"); + if (lineEnd < 0) break; + const line = diagnosticStderr.slice(0, lineEnd).trim(); + diagnosticStderr = diagnosticStderr.slice(lineEnd + 1); + if (line.startsWith("[Chatto Desktop capture]")) { + captureDiagnostics.push(line); + if (captureDiagnostics.length > 100) captureDiagnostics.shift(); + console.warn(line); + } + } }); + // The helper can finish naturally while Desktop is writing a cooperative + // stop command. Its exit handler owns the renderer-visible lifecycle. + child.stdin.on("error", () => {}); child.once("error", () => { port.postMessage({ kind: "error", @@ -610,10 +689,17 @@ function startGameCaptureSession(source, publisherRequest, port) { clearTimeout(forceStopTimer); if (activeGameCaptureSession === session) activeGameCaptureSession = undefined; - if (!stopping && (code !== 0 || signal)) { + console.warn("[Chatto Desktop] Native screen-share helper exited", { + code, + signal, + stopping, + captureDiagnostics, + }); + if (!stopping) { port.postMessage({ kind: "error", message: + captureDiagnostics.at(-1) || stderr.trim() || "The native screen-share helper stopped unexpectedly.", }); @@ -622,19 +708,25 @@ function startGameCaptureSession(source, publisherRequest, port) { port.close(); }); port.on("message", (event) => { - if (event.data?.kind === "stop") stop(); + if (event.data?.kind === "stop") { + port.postMessage({ kind: "stopping" }); + stop(); + } }); port.on("close", stop); port.start(); - child.stdin.end( - JSON.stringify({ - protocolVersion: 1, - livekitURL: publisherRequest.livekitUrl, - token: publisherRequest.token, - e2eeKey: publisherRequest.e2eeKey, - }), - ); + const credential = JSON.stringify({ + protocolVersion: 1, + livekitURL: publisherRequest.livekitUrl, + token: publisherRequest.token, + e2eeKey: publisherRequest.e2eeKey, + }); + if (process.platform === "win32") { + child.stdin.write(`${credential}\n`); + } else { + child.stdin.end(credential); + } function stop() { if (stopping) return; @@ -642,7 +734,11 @@ function startGameCaptureSession(source, publisherRequest, port) { if (activeGameCaptureSession === session) activeGameCaptureSession = undefined; if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGTERM"); + if (process.platform === "win32" && child.stdin.writable) { + child.stdin.end("stop\n"); + } else { + child.kill("SIGTERM"); + } forceStopTimer = setTimeout(() => { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); @@ -658,12 +754,47 @@ function stopActiveGameCaptureSession() { } function isGameCaptureAvailable() { + if (!app.isPackaged) return false; + if (process.platform === "darwin") { + return ( + supportsMacOSGameCapture(process.getSystemVersion()) && + existsSync(macOSCaptureHelperExecutable()) + ); + } return ( - process.platform === "darwin" && - app.isPackaged && - supportsMacOSGameCapture(process.getSystemVersion()) && - existsSync(macOSCaptureHelperExecutable()) + process.platform === "win32" && + supportsWindowsGameCapture(process.getSystemVersion()) && + existsSync(windowsCaptureHelperExecutable()) + ); +} + +async function windowsCapturePreviews() { + const previews = new Map(); + const sources = await desktopCapturer.getSources({ + types: ["window"], + thumbnailSize: { width: 480, height: 270 }, + fetchWindowIcons: false, + }); + for (const source of sources) { + const match = /^window:([1-9][0-9]*):/.exec(source.id); + if (!match || source.thumbnail.isEmpty()) continue; + const nativeId = Number(match[1]); + if (!Number.isSafeInteger(nativeId)) continue; + const preview = source.thumbnail.toJPEG(80); + if (preview.length <= 512 * 1024) previews.set(nativeId, preview); + } + return previews; +} + +async function listNativeGameCaptureSources() { + const output = await runNativeCaptureHelperBinary( + process.platform === "win32" + ? ["list-json", "--exclude-process", String(process.pid)] + : ["list-previews"], ); + return process.platform === "win32" + ? parseWindowsGameCaptureSources(output, await windowsCapturePreviews()) + : parseMacOSGameCaptureSources(output); } function secureWebPreferences() { diff --git a/apps/desktop/native/windows-capture-probe/.gitignore b/apps/desktop/native/windows-capture-probe/.gitignore new file mode 100644 index 0000000000..14e3a003f5 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +# SPDX-License-Identifier: Apache-2.0 + +/build*/ +/captures/ diff --git a/apps/desktop/native/windows-capture-probe/CMakeLists.txt b/apps/desktop/native/windows-capture-probe/CMakeLists.txt new file mode 100644 index 0000000000..45d33b08e6 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/CMakeLists.txt @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.25) + +project(chatto-windows-capture-probe LANGUAGES CXX) + +set(CHATTO_LIVEKIT_VERSION "1.7.0-chatto.3") +set(CHATTO_LIVEKIT_SHA256 "a7a26545c667d4ce8da1f1f13924c32c7613db375db320e97c99f828cfdaff01") +set(CHATTO_LIVEKIT_SDK_ROOT "" CACHE PATH "Path to an extracted LiveKit C++ SDK") +if(NOT CHATTO_LIVEKIT_SDK_ROOT) + set(_livekit_archive "${CMAKE_BINARY_DIR}/_deps/livekit-sdk-windows-x64-${CHATTO_LIVEKIT_VERSION}.zip") + set(_livekit_extract "${CMAKE_BINARY_DIR}/_deps/livekit-sdk-${CHATTO_LIVEKIT_VERSION}") + set(CHATTO_LIVEKIT_SDK_ROOT + "${_livekit_extract}/livekit-sdk-windows-x64-${CHATTO_LIVEKIT_VERSION}") + if(NOT EXISTS "${CHATTO_LIVEKIT_SDK_ROOT}/lib/livekit.lib") + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/_deps") + file(DOWNLOAD + "https://github.com/chattocorp/client-sdk-cpp/releases/download/v${CHATTO_LIVEKIT_VERSION}/livekit-sdk-windows-x64-${CHATTO_LIVEKIT_VERSION}.zip" + "${_livekit_archive}" + EXPECTED_HASH "SHA256=${CHATTO_LIVEKIT_SHA256}" + TLS_VERIFY ON + SHOW_PROGRESS + ) + file(MAKE_DIRECTORY "${_livekit_extract}") + file(ARCHIVE_EXTRACT INPUT "${_livekit_archive}" DESTINATION "${_livekit_extract}") + endif() +endif() +# find_package caches LiveKit_DIR independently of our versioned archive path. +# Clear it so changing the pin also updates an existing helper build directory. +unset(LiveKit_DIR CACHE) +find_package(LiveKit CONFIG REQUIRED PATHS "${CHATTO_LIVEKIT_SDK_ROOT}" NO_DEFAULT_PATH) + +set(CHATTO_NV_CODEC_HEADERS_VERSION "13.0.19.1") +set(CHATTO_NV_CODEC_HEADERS_SHA256 "eab9d02d461035a4baded23d4f2e2834e5b17934820b79cea45baaf0c85383eb") +set(_nv_codec_archive "${CMAKE_BINARY_DIR}/_deps/nv-codec-headers-${CHATTO_NV_CODEC_HEADERS_VERSION}.tar.gz") +set(_nv_codec_extract "${CMAKE_BINARY_DIR}/_deps/nv-codec-headers-${CHATTO_NV_CODEC_HEADERS_VERSION}") +set(CHATTO_NV_CODEC_HEADERS_ROOT + "${_nv_codec_extract}/nv-codec-headers-${CHATTO_NV_CODEC_HEADERS_VERSION}") +if(NOT EXISTS "${CHATTO_NV_CODEC_HEADERS_ROOT}/include/ffnvcodec/nvEncodeAPI.h") + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/_deps") + file(DOWNLOAD + "https://github.com/FFmpeg/nv-codec-headers/releases/download/n${CHATTO_NV_CODEC_HEADERS_VERSION}/nv-codec-headers-${CHATTO_NV_CODEC_HEADERS_VERSION}.tar.gz" + "${_nv_codec_archive}" + EXPECTED_HASH "SHA256=${CHATTO_NV_CODEC_HEADERS_SHA256}" + TLS_VERIFY ON + SHOW_PROGRESS + ) + file(MAKE_DIRECTORY "${_nv_codec_extract}") + file(ARCHIVE_EXTRACT INPUT "${_nv_codec_archive}" DESTINATION "${_nv_codec_extract}") +endif() + +add_executable(chatto-windows-capture-probe + src/audio_capture.cpp + src/h264_encoder.cpp + src/nvenc_encoder.cpp + src/livekit_publisher.cpp + src/main.cpp + src/preview_window.cpp + src/video_capture.cpp + src/window_sources.cpp +) + +target_compile_features(chatto-windows-capture-probe PRIVATE cxx_std_20) +target_compile_definitions(chatto-windows-capture-probe PRIVATE + NOMINMAX + UNICODE + WIN32_LEAN_AND_MEAN + _UNICODE +) +target_compile_options(chatto-windows-capture-probe PRIVATE + /EHsc + /permissive- + /utf-8 + /W4 + /WX +) +target_include_directories(chatto-windows-capture-probe PRIVATE + "${CHATTO_NV_CODEC_HEADERS_ROOT}/include" +) +target_link_libraries(chatto-windows-capture-probe PRIVATE + d3d11 + bcrypt + dwmapi + dxgi + mf + mfplat + mfuuid + mmdevapi + ole32 + psapi + user32 + windowsapp + LiveKit::livekit +) + +add_custom_command(TARGET chatto-windows-capture-probe POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${CHATTO_LIVEKIT_SDK_ROOT}/bin/livekit.dll" + "$" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${CHATTO_LIVEKIT_SDK_ROOT}/bin/livekit_ffi.dll" + "$" +) + +include(CTest) + +if(BUILD_TESTING) + add_executable(chatto-windows-latest-frame-queue-tests + tests/latest_frame_queue_test.cpp + ) + target_compile_features(chatto-windows-latest-frame-queue-tests PRIVATE cxx_std_20) + target_compile_options(chatto-windows-latest-frame-queue-tests PRIVATE + /EHsc + /permissive- + /utf-8 + /W4 + /WX + ) + add_test( + NAME latest-frame-queue-tests + COMMAND chatto-windows-latest-frame-queue-tests + ) + + add_executable(chatto-windows-capture-probe-tests + tests/window_sources_test.cpp + src/window_sources.cpp + ) + target_compile_features(chatto-windows-capture-probe-tests PRIVATE cxx_std_20) + target_compile_definitions(chatto-windows-capture-probe-tests PRIVATE + NOMINMAX + UNICODE + WIN32_LEAN_AND_MEAN + _UNICODE + ) + target_compile_options(chatto-windows-capture-probe-tests PRIVATE + /EHsc + /permissive- + /utf-8 + /W4 + /WX + ) + target_link_libraries(chatto-windows-capture-probe-tests PRIVATE bcrypt dwmapi) + add_test(NAME window-source-tests COMMAND chatto-windows-capture-probe-tests) + + add_executable(chatto-windows-h264-encoder-tests + tests/h264_encoder_test.cpp + src/h264_encoder.cpp + src/nvenc_encoder.cpp + ) + target_compile_features(chatto-windows-h264-encoder-tests PRIVATE cxx_std_20) + target_compile_definitions(chatto-windows-h264-encoder-tests PRIVATE + NOMINMAX + UNICODE + WIN32_LEAN_AND_MEAN + _UNICODE + ) + target_compile_options(chatto-windows-h264-encoder-tests PRIVATE + /EHsc + /permissive- + /utf-8 + /W4 + /WX + ) + target_include_directories(chatto-windows-h264-encoder-tests PRIVATE src) + target_include_directories(chatto-windows-h264-encoder-tests PRIVATE + "${CHATTO_NV_CODEC_HEADERS_ROOT}/include" + ) + target_link_libraries(chatto-windows-h264-encoder-tests PRIVATE + d3d11 + dxgi + mf + mfplat + mfuuid + ) + add_test(NAME h264-encoder-tests COMMAND chatto-windows-h264-encoder-tests) +endif() diff --git a/apps/desktop/native/windows-capture-probe/README.md b/apps/desktop/native/windows-capture-probe/README.md new file mode 100644 index 0000000000..cb73aa701e --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/README.md @@ -0,0 +1,177 @@ +# Windows native game-capture helper + +This helper implements the Windows portion of +[issue #2021](https://github.com/chattocorp/chatto/issues/2021). Chatto Desktop +uses it to enumerate ordinary visible windows, capture a selected window with +isolated owning-process-tree audio, and publish both directly to LiveKit with +the call's shared E2EE key. Diagnostic commands remain available for focused +native verification. + +The capture stack is Windows Graphics Capture into D3D11 textures, WASAPI +process-loopback audio, direct NVIDIA NVENC H.264 with Media Foundation +hardware fallback, and a pinned public Chatto fork of the LiveKit C++ SDK. +The direct NVIDIA path keeps production video frames on the GPU: capture copies +each frame into a shareable D3D11 texture, the NVIDIA video processor scales it +and converts BGRA to BT.709 NV12, and NVENC reads the registered NV12 surface. +It uses the driver's low-latency P5 preset, quarter-resolution multipass, and +spatial adaptive quantisation. Media Foundation remains a compatibility +fallback and performs a CPU readback when direct NVENC is unavailable. The fork +passes complete encoded access units into WebRTC without a second software +encode and forwards keyframe and rate-control requests back to the selected +encoder. A single-slot latest-frame handoff keeps +that work off the Windows capture callback and discards superseded frames +instead of accumulating latency. Production Desktop builds compile and embed +the executable plus its two LiveKit DLLs. + +Publication uses one full-cadence H.264 video layer. LiveKit C++ 1.7 exposes only a +simulcast switch rather than custom screen-share layers, and its default lowest +screen-share layer is capped at 3 fps; compact adaptive-stream tiles therefore +turn a game stream into a slideshow. A custom game-oriented simulcast ladder +remains follow-up work for a newer SDK or a pinned fork. +Dynacast is disabled for this single layer so Desktop's local preview may +throttle or unsubscribe without suspending the independent native publisher. + +During publication, the helper treats a missing source window or a two-second +frame stall as a recoverable application transition. It first selects another +ordinary visible window belonging to the same executable, preferring the +original process, and can restart capture on the same still-valid window. If no +replacement appears within three seconds, it disconnects the companion so the +share ends. Desktop requests shutdown with a line-delimited `stop` control +command and force-terminates only if the helper does not exit in time. +WGC texture resizes do not change the published track dimensions: every frame +is scaled from its current texture size into the stable output selected when +the share started. +WGC can stall when a monitor-covering borderless game enters DirectFlip or +Independent Flip and bypasses desktop composition. If the selected window owns +its complete foreground monitor, the helper immediately captures that monitor +through WGC instead of waiting for sparse window-WGC heartbeat frames to stop. +DXGI Desktop Duplication remains the final +fallback if monitor WGC also stalls. Both display paths end when the game leaves +that presentation. Failure to initialise a fallback is non-fatal and retried +with a cooldown. Publisher metrics identify the active backend as `wgc-window`, +`wgc-monitor`, or `dxgi-display`. +Windows invalidates Desktop Duplication during some presentation-producer +transitions with `DXGI_ERROR_ACCESS_LOST`; the helper treats that as a +recoverable signal and recreates the duplication interface for the new +producer. + +Non-content publisher metrics are emitted every two seconds even while capture +or publication is stalled. Alongside capture, GPU copy, GPU conversion, encoder +submission, bitstream wait, encoding, and RTP counters, they report the latest +WGC texture dimensions and how often those dimensions changed. Desktop +separately acknowledges a received stop command before waiting for the helper +to disconnect and exit. + +## Requirements + +- Windows 11, or a Windows 10 version that supports the APIs under test +- Visual Studio 2022 Build Tools with the Desktop development with C++ workload +- Windows 11 SDK +- CMake 3.25 or newer + +CMake downloads the public `chattocorp/client-sdk-cpp` +`v1.7.0-chatto.3` prerelease and verifies the archive's pinned SHA-256 unless +`CHATTO_LIVEKIT_SDK_ROOT` points at an already extracted SDK. That C++ fork +pins the public `chattocorp/rust-sdks` pre-encoded-video FFI release. +CMake also downloads and verifies the permissively licensed FFmpeg +`nv-codec-headers` 13.0.19.1 release. NVENC itself is loaded dynamically from +the installed NVIDIA display driver; no NVIDIA runtime DLL is bundled. + +## Build and test + +Open a Developer PowerShell for Visual Studio, then run: + +```powershell +cmake -S apps/desktop/native/windows-capture-probe ` + -B apps/desktop/native/windows-capture-probe/build +cmake --build apps/desktop/native/windows-capture-probe/build ` + --config RelWithDebInfo +ctest --test-dir apps/desktop/native/windows-capture-probe/build ` + -C RelWithDebInfo --output-on-failure +``` + +On a development machine with a GPU, the opt-in smoke mode also proves that the +selected hardware backend emits keyframed Annex-B H.264 and accepts a dynamic +bitrate change. It is intentionally not a CI test because hosted Windows +runners may not expose a hardware encoder: + +```powershell +apps/desktop/native/windows-capture-probe/build/RelWithDebInfo/chatto-windows-h264-encoder-tests.exe ` + --hardware +``` + +Check Windows Graphics Capture availability and list candidate windows: + +```powershell +apps/desktop/native/windows-capture-probe/build/RelWithDebInfo/chatto-windows-capture-probe.exe support +apps/desktop/native/windows-capture-probe/build/RelWithDebInfo/chatto-windows-capture-probe.exe list +``` + +Copy a temporary `hwnd` value from the list and capture its frames for 15 +seconds: + +```powershell +apps/desktop/native/windows-capture-probe/build/RelWithDebInfo/chatto-windows-capture-probe.exe ` + capture --window 0x123456 --duration 15 --fps 60 +``` + +Add `--preview` to open a native Win32 window backed by a flip-model DXGI swap +chain. Captured textures are copied directly on the GPU; the preview title shows +live observed FPS, frame and content-sample counts, latest isolated audio peak, +captured audio frames, and audio discontinuities. Closing the preview ends video +acquisition and process-audio capture early. + +The summary reports delivered frames, current content dimensions, native +system-relative timestamp span, observed cadence, longest interval, inferred +gaps, frame-pool resizes, and whether the source closed. Each delivered surface +must expose an `ID3D11Texture2D`; the capture fails instead of counting an +unusable frame. A sparse CPU readback samples a bounded pixel grid roughly four +times per second and reports aggregate luminance, black samples, and changing +sample hashes; it never retains or writes an image. By default, it concurrently captures audio +from the selected window's owning process tree and reports the A/V start delta +on the shared system-relative 100-nanosecond timeline. Use `--video-only` to +isolate video diagnostics. The frame rate is a diagnostic expectation used for +gap detection; Windows Graphics Capture controls actual delivery. + +The completed summary also reports probe wall time, total process CPU time, +single-core-equivalent CPU percentage, and peak working-set size. GPU utilisation +and impact on the game's own frame pacing still require an external measurement +method. + +Capture 48 kHz stereo audio emitted by a process and its child-process tree: + +```powershell +apps/desktop/native/windows-capture-probe/build/RelWithDebInfo/chatto-windows-capture-probe.exe ` + audio --process 1234 --duration 15 +``` + +The audio summary reports packets, frames, format, QPC-derived timestamp span, +peak level, silent packets, discontinuities, and timestamp errors. Samples stay +in memory only and are discarded after their levels and timing are measured. + +The source list reports temporary native window coordinates, executable names, +and dimensions. It mechanically excludes hidden, cloaked, owned, and very small +windows; it does not try to recognise games. Native coordinates are diagnostic +input only and must not become a renderer or public API contract. + +Window titles can contain sensitive information and are omitted by default. +The explicit `list --include-titles` option prints them only to help a person +identify a source during local diagnostics. Do not paste that output into logs +or issue reports without reviewing it. + +## Production boundary + +The `list-json` and `publish` commands are private Desktop protocols. Electron +replaces native handles with short-lived single-use offers, obtains static JPEG +previews itself, and sends only a selected offer plus a fresh LiveKit +credential to the helper. Before publishing, the helper re-resolves the window +and compares a SHA-256 binding of its executable path. Raw paths and handles do +not cross renderer IPC. Media remains in the native helper; only source +descriptions, credentials, acknowledged lifecycle status, and aggregate +non-content performance timings cross Electron. + +Windows offers both window and display capture; its internal monitor WGC and +DXGI paths also protect window sharing across direct-presentation transitions. +Audio-device recovery, GPU scaling and colour conversion, direct handoff of +capture textures to the encoder, a game-oriented simulcast ladder, and broader +DX11/DX12/Vulkan game validation remain follow-up work. diff --git a/apps/desktop/native/windows-capture-probe/src/audio_capture.cpp b/apps/desktop/native/windows-capture-probe/src/audio_capture.cpp new file mode 100644 index 0000000000..225e69c350 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/audio_capture.cpp @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "audio_capture.h" + +#include "live_status.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +using Microsoft::WRL::ComPtr; +using Microsoft::WRL::FtmBase; +using Microsoft::WRL::Make; +using Microsoft::WRL::RuntimeClass; +using Microsoft::WRL::RuntimeClassFlags; +using Microsoft::WRL::ClassicCom; + +class ActivationHandler final + : public RuntimeClass< + RuntimeClassFlags, + FtmBase, + IActivateAudioInterfaceCompletionHandler> { + public: + ActivationHandler() : completed_(CreateEventW(nullptr, FALSE, FALSE, nullptr)) {} + + ~ActivationHandler() override { + if (completed_ != nullptr) { + CloseHandle(completed_); + } + } + + HRESULT RuntimeClassInitialize() noexcept { + return completed_ == nullptr ? HRESULT_FROM_WIN32(GetLastError()) : S_OK; + } + + STDMETHODIMP ActivateCompleted( + IActivateAudioInterfaceAsyncOperation* operation) noexcept override { + HRESULT activation_result = E_UNEXPECTED; + ComPtr activated_interface; + result_ = operation->GetActivateResult( + &activation_result, activated_interface.GetAddressOf()); + if (SUCCEEDED(result_)) { + result_ = activation_result; + } + if (SUCCEEDED(result_)) { + result_ = activated_interface.As(&audio_client_); + } + SetEvent(completed_); + return S_OK; + } + + [[nodiscard]] ComPtr wait_for_client() { + const DWORD wait_result = WaitForSingleObject(completed_, 30'000); + if (wait_result != WAIT_OBJECT_0) { + if (wait_result == WAIT_FAILED) { + winrt::throw_last_error(); + } + throw std::runtime_error("Timed out activating process-loopback audio"); + } + winrt::check_hresult(result_); + return audio_client_; + } + + private: + HANDLE completed_ = nullptr; + HRESULT result_ = E_PENDING; + ComPtr audio_client_; +}; + +[[nodiscard]] WAVEFORMATEXTENSIBLE capture_format() { + WAVEFORMATEXTENSIBLE format{}; + format.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + format.Format.nChannels = 2; + format.Format.nSamplesPerSec = 48'000; + format.Format.wBitsPerSample = 32; + format.Format.nBlockAlign = + format.Format.nChannels * format.Format.wBitsPerSample / 8; + format.Format.nAvgBytesPerSec = + format.Format.nSamplesPerSec * format.Format.nBlockAlign; + format.Format.cbSize = sizeof(WAVEFORMATEXTENSIBLE) - sizeof(WAVEFORMATEX); + format.Samples.wValidBitsPerSample = 32; + format.dwChannelMask = SPEAKER_FRONT_LEFT | SPEAKER_FRONT_RIGHT; + format.SubFormat = KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; + return format; +} + +[[nodiscard]] ComPtr activate_process_loopback(DWORD process_id) { + AUDIOCLIENT_ACTIVATION_PARAMS activation{}; + activation.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; + activation.ProcessLoopbackParams.TargetProcessId = process_id; + activation.ProcessLoopbackParams.ProcessLoopbackMode = + PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; + + PROPVARIANT parameters{}; + parameters.vt = VT_BLOB; + parameters.blob.cbSize = sizeof(activation); + parameters.blob.pBlobData = reinterpret_cast(&activation); + + const auto handler = Make(); + if (!handler) { + throw std::bad_alloc(); + } + winrt::check_hresult(handler->RuntimeClassInitialize()); + + ComPtr operation; + winrt::check_hresult(ActivateAudioInterfaceAsync( + VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, + __uuidof(IAudioClient), + ¶meters, + handler.Get(), + operation.GetAddressOf())); + return handler->wait_for_client(); +} + +void consume_audio_packet( + IAudioCaptureClient& capture_client, + const WAVEFORMATEXTENSIBLE& format, + AudioCaptureMetrics& metrics, + std::uint64_t& first_timestamp, + std::uint64_t& last_timestamp, + const std::shared_ptr& live_status, + const AudioFrameHandler& frame_handler) { + BYTE* bytes = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + UINT64 device_position = 0; + UINT64 qpc_position = 0; + winrt::check_hresult(capture_client.GetBuffer( + &bytes, &frames, &flags, &device_position, &qpc_position)); + + metrics.packets += 1; + metrics.frames += frames; + if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) { + metrics.discontinuities += 1; + } + if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) { + metrics.timestamp_errors += 1; + } else { + if (first_timestamp == 0) { + first_timestamp = qpc_position; + } + const auto packet_duration = + static_cast(frames) * 10'000'000ULL / + format.Format.nSamplesPerSec; + last_timestamp = qpc_position + packet_duration; + } + + float packet_peak = 0; + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0 || bytes == nullptr) { + metrics.silent_packets += 1; + } else { + const auto* samples = reinterpret_cast(bytes); + const std::size_t sample_count = + static_cast(frames) * format.Format.nChannels; + for (std::size_t index = 0; index < sample_count; ++index) { + packet_peak = std::max(packet_peak, std::abs(samples[index])); + } + metrics.peak_level = std::max(metrics.peak_level, packet_peak); + } + if (live_status) { + live_status->audio_frames.store(metrics.frames, std::memory_order_relaxed); + live_status->audio_discontinuities.store( + metrics.discontinuities, std::memory_order_relaxed); + live_status->latest_audio_peak.store(packet_peak, std::memory_order_relaxed); + } + if (frame_handler && frames > 0) { + frame_handler(AudioFrameData{ + .sample_rate = format.Format.nSamplesPerSec, + .channels = format.Format.nChannels, + .frames = frames, + .timestamp_100ns = qpc_position, + .samples = reinterpret_cast(bytes), + .silent = (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0 || bytes == nullptr, + }); + } + + winrt::check_hresult(capture_client.ReleaseBuffer(frames)); +} + +} // namespace + +AudioCaptureMetrics capture_process_audio( + const DWORD process_id, + const std::chrono::seconds duration, + const std::stop_token stop_token, + const std::shared_ptr live_status, + AudioFrameHandler frame_handler) { + if (process_id == 0 || duration.count() <= 0) { + throw std::invalid_argument("Audio process and duration must be positive"); + } + + const auto audio_client = activate_process_loopback(process_id); + const auto format = capture_format(); + constexpr DWORD stream_flags = + AUDCLNT_STREAMFLAGS_LOOPBACK | + AUDCLNT_STREAMFLAGS_EVENTCALLBACK | + AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | + AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY; + winrt::check_hresult(audio_client->Initialize( + AUDCLNT_SHAREMODE_SHARED, + stream_flags, + 0, + 0, + &format.Format, + nullptr)); + + ComPtr capture_client; + winrt::check_hresult(audio_client->GetService(IID_PPV_ARGS(&capture_client))); + const HANDLE sample_ready = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (sample_ready == nullptr) { + winrt::throw_last_error(); + } + + try { + winrt::check_hresult(audio_client->SetEventHandle(sample_ready)); + winrt::check_hresult(audio_client->Start()); + + AudioCaptureMetrics metrics; + metrics.sample_rate = format.Format.nSamplesPerSec; + metrics.channels = format.Format.nChannels; + std::uint64_t first_timestamp = 0; + std::uint64_t last_timestamp = 0; + const auto deadline = std::chrono::steady_clock::now() + duration; + + while (std::chrono::steady_clock::now() < deadline && + !stop_token.stop_requested()) { + const auto remaining = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()); + const DWORD timeout = static_cast(std::clamp( + remaining.count(), 1, 1'000)); + const DWORD wait_result = WaitForSingleObject(sample_ready, timeout); + if (wait_result == WAIT_FAILED) { + winrt::throw_last_error(); + } + if (wait_result != WAIT_OBJECT_0) { + continue; + } + + UINT32 available_frames = 0; + winrt::check_hresult(capture_client->GetNextPacketSize(&available_frames)); + while (available_frames > 0) { + consume_audio_packet( + *capture_client.Get(), + format, + metrics, + first_timestamp, + last_timestamp, + live_status, + frame_handler); + winrt::check_hresult(capture_client->GetNextPacketSize(&available_frames)); + } + } + + winrt::check_hresult(audio_client->Stop()); + CloseHandle(sample_ready); + if (first_timestamp != 0 && last_timestamp >= first_timestamp) { + metrics.first_timestamp_100ns = first_timestamp; + metrics.last_timestamp_100ns = last_timestamp; + metrics.timestamp_span_seconds = + static_cast(last_timestamp - first_timestamp) / 10'000'000.0; + } + return metrics; + } catch (...) { + audio_client->Stop(); + CloseHandle(sample_ready); + throw; + } +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/audio_capture.h b/apps/desktop/native/windows-capture-probe/src/audio_capture.h new file mode 100644 index 0000000000..bd065d6099 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/audio_capture.h @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include + +#include + +namespace chatto::capture { + +struct LiveCaptureStatus; + +struct AudioFrameData { + std::uint32_t sample_rate; + std::uint16_t channels; + std::uint32_t frames; + std::uint64_t timestamp_100ns; + const float* samples; + bool silent; +}; + +using AudioFrameHandler = std::function; + +struct AudioCaptureMetrics { + std::uint64_t packets = 0; + std::uint64_t frames = 0; + std::uint64_t silent_packets = 0; + std::uint64_t discontinuities = 0; + std::uint64_t timestamp_errors = 0; + std::uint32_t sample_rate = 0; + std::uint16_t channels = 0; + std::uint64_t first_timestamp_100ns = 0; + std::uint64_t last_timestamp_100ns = 0; + double timestamp_span_seconds = 0; + float peak_level = 0; +}; + +[[nodiscard]] AudioCaptureMetrics capture_process_audio( + DWORD process_id, + std::chrono::seconds duration, + std::stop_token stop_token = {}, + std::shared_ptr live_status = {}, + AudioFrameHandler frame_handler = {}); + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/h264_encoder.cpp b/apps/desktop/native/windows-capture-probe/src/h264_encoder.cpp new file mode 100644 index 0000000000..486e8a8ad9 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/h264_encoder.cpp @@ -0,0 +1,725 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "h264_encoder.h" +#include "video_frame_scaler.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +using Microsoft::WRL::ComPtr; + +[[noreturn]] void throw_hresult(const char *message, const HRESULT result) { + throw std::runtime_error(std::string(message) + " (0x" + [&result] { + std::array text{}; + static_cast(std::snprintf(text.data(), text.size(), "%08x", + static_cast(result))); + return std::string(text.data()); + }() + ")"); +} + +void check_hresult(const HRESULT result, const char *message) { + if (FAILED(result)) { + throw_hresult(message, result); + } +} + +[[nodiscard]] std::uint8_t clamp_byte(const int value) { + return static_cast(std::clamp(value, 0, 255)); +} + +[[nodiscard]] bool +starts_with_start_code(const std::span bytes, + const std::size_t offset, std::size_t &start_code_size) { + if (offset + 3 <= bytes.size() && bytes[offset] == 0 && + bytes[offset + 1] == 0 && bytes[offset + 2] == 1) { + start_code_size = 3; + return true; + } + if (offset + 4 <= bytes.size() && bytes[offset] == 0 && + bytes[offset + 1] == 0 && bytes[offset + 2] == 0 && + bytes[offset + 3] == 1) { + start_code_size = 4; + return true; + } + return false; +} + +void normalize_h264_access_unit(std::vector &access_unit) { + std::size_t start_code_size = 0; + if (starts_with_start_code(access_unit, 0, start_code_size)) { + return; + } + + std::vector annex_b; + std::size_t offset = 0; + while (offset + 4 <= access_unit.size()) { + const auto nal_size = + (static_cast(access_unit[offset]) << 24U) | + (static_cast(access_unit[offset + 1]) << 16U) | + (static_cast(access_unit[offset + 2]) << 8U) | + static_cast(access_unit[offset + 3]); + offset += 4; + if (nal_size == 0 || nal_size > access_unit.size() - offset) { + return; + } + annex_b.insert(annex_b.end(), {0, 0, 0, 1}); + annex_b.insert(annex_b.end(), access_unit.begin() + offset, + access_unit.begin() + offset + nal_size); + offset += nal_size; + } + if (offset == access_unit.size() && !annex_b.empty()) { + access_unit = std::move(annex_b); + } +} + +void set_codec_u32(ICodecAPI *codec_api, const GUID &property, + const std::uint32_t value, const bool required = false) { + if (codec_api == nullptr) { + if (required) { + throw std::runtime_error("The hardware H.264 encoder has no codec API"); + } + return; + } + VARIANT setting; + VariantInit(&setting); + setting.vt = VT_UI4; + setting.ulVal = value; + const HRESULT result = codec_api->SetValue(&property, &setting); + VariantClear(&setting); + if (required && FAILED(result)) { + throw_hresult("The hardware H.264 encoder rejected a required setting", + result); + } +} + +void set_codec_bool(ICodecAPI *codec_api, const GUID &property, + const bool value) { + if (codec_api == nullptr) { + return; + } + VARIANT setting; + VariantInit(&setting); + setting.vt = VT_BOOL; + setting.boolVal = value ? VARIANT_TRUE : VARIANT_FALSE; + static_cast(codec_api->SetValue(&property, &setting)); + VariantClear(&setting); +} + +[[nodiscard]] std::optional get_codec_u32(ICodecAPI *codec_api, + const GUID &property) { + if (codec_api == nullptr) { + return std::nullopt; + } + VARIANT setting; + VariantInit(&setting); + const HRESULT result = codec_api->GetValue(&property, &setting); + const auto value = SUCCEEDED(result) && setting.vt == VT_UI4 + ? std::optional(setting.ulVal) + : std::nullopt; + VariantClear(&setting); + return value; +} + +[[nodiscard]] ComPtr +make_video_type(const GUID &subtype, const std::uint32_t width, + const std::uint32_t height, + const std::uint32_t frames_per_second) { + ComPtr type; + check_hresult(MFCreateMediaType(&type), + "Could not create a video media type"); + check_hresult(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), + "Could not set the video major type"); + check_hresult(type->SetGUID(MF_MT_SUBTYPE, subtype), + "Could not set the video subtype"); + check_hresult(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, width, height), + "Could not set the video frame size"); + check_hresult( + MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, frames_per_second, 1), + "Could not set the video frame rate"); + check_hresult(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1), + "Could not set the video pixel aspect ratio"); + check_hresult( + type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive), + "Could not set progressive video"); + return type; +} + +[[nodiscard]] std::string narrow_name(IMFActivate *activation) { + wchar_t *wide_name = nullptr; + std::uint32_t length = 0; + if (FAILED(activation->GetAllocatedString(MFT_FRIENDLY_NAME_Attribute, + &wide_name, &length)) || + wide_name == nullptr) { + return "Media Foundation hardware H.264 encoder"; + } + const int byte_count = + WideCharToMultiByte(CP_UTF8, 0, wide_name, static_cast(length), + nullptr, 0, nullptr, nullptr); + std::string name(static_cast(std::max(byte_count, 0)), '\0'); + if (byte_count > 0) { + static_cast(WideCharToMultiByte(CP_UTF8, 0, wide_name, + static_cast(length), name.data(), + byte_count, nullptr, nullptr)); + } + CoTaskMemFree(wide_name); + return name; +} + +} // namespace + +std::vector bgra_to_nv12(const std::span bgra, + const std::uint32_t width, + const std::uint32_t height) { + const auto expected_size = + static_cast(width) * static_cast(height) * 4; + if (width == 0 || height == 0 || (width % 2) != 0 || (height % 2) != 0 || + bgra.size() != expected_size) { + throw std::invalid_argument( + "NV12 conversion requires a tightly packed, even-sized BGRA frame"); + } + + const auto luma_size = + static_cast(width) * static_cast(height); + std::vector nv12(luma_size + luma_size / 2); + auto *luma = nv12.data(); + auto *chroma = nv12.data() + luma_size; + + for (std::uint32_t y = 0; y < height; ++y) { + for (std::uint32_t x = 0; x < width; ++x) { + const auto offset = (static_cast(y) * width + x) * 4; + const int blue = bgra[offset]; + const int green = bgra[offset + 1]; + const int red = bgra[offset + 2]; + luma[static_cast(y) * width + x] = + clamp_byte(16 + ((47 * red + 157 * green + 16 * blue + 128) >> 8)); + } + } + + for (std::uint32_t y = 0; y < height; y += 2) { + for (std::uint32_t x = 0; x < width; x += 2) { + int blue = 0; + int green = 0; + int red = 0; + for (std::uint32_t dy = 0; dy < 2; ++dy) { + for (std::uint32_t dx = 0; dx < 2; ++dx) { + const auto offset = + (static_cast(y + dy) * width + x + dx) * 4; + blue += bgra[offset]; + green += bgra[offset + 1]; + red += bgra[offset + 2]; + } + } + blue /= 4; + green /= 4; + red /= 4; + const auto chroma_offset = static_cast(y / 2) * width + x; + chroma[chroma_offset] = + clamp_byte(128 + ((-26 * red - 87 * green + 112 * blue + 128) >> 8)); + chroma[chroma_offset + 1] = + clamp_byte(128 + ((112 * red - 102 * green - 10 * blue + 128) >> 8)); + } + } + return nv12; +} + +bool h264_access_unit_is_key_frame( + const std::span access_unit) { + for (std::size_t offset = 0; offset < access_unit.size(); ++offset) { + std::size_t start_code_size = 0; + if (!starts_with_start_code(access_unit, offset, start_code_size)) { + continue; + } + const auto nal_offset = offset + start_code_size; + if (nal_offset < access_unit.size() && + (access_unit[nal_offset] & 0x1fU) == 5U) { + return true; + } + offset = nal_offset; + } + return false; +} + +bool h264_access_unit_is_annex_b( + const std::span access_unit) { + std::size_t start_code_size = 0; + return starts_with_start_code(access_unit, 0, start_code_size); +} + +std::optional h264_access_unit_profile_level( + const std::span access_unit) { + for (std::size_t offset = 0; offset < access_unit.size(); ++offset) { + std::size_t start_code_size = 0; + if (!starts_with_start_code(access_unit, offset, start_code_size)) { + continue; + } + const auto nal_offset = offset + start_code_size; + if (nal_offset + 3 < access_unit.size() && + (access_unit[nal_offset] & 0x1fU) == 7U) { + return H264ProfileLevel{ + .profile_idc = access_unit[nal_offset + 1], + .profile_iop = access_unit[nal_offset + 2], + .level_idc = access_unit[nal_offset + 3], + }; + } + offset = nal_offset; + } + return std::nullopt; +} + +class MediaFoundationH264Encoder::Implementation final { +public: + Implementation(const std::uint32_t width, const std::uint32_t height, + const std::uint32_t frames_per_second, + const std::uint32_t target_bitrate_bps) + : width_(width), height_(height), frames_per_second_(frames_per_second), + frame_duration_100ns_(10'000'000LL / frames_per_second) { + if (width == 0 || height == 0 || (width % 2) != 0 || (height % 2) != 0 || + frames_per_second == 0 || target_bitrate_bps == 0) { + throw std::invalid_argument( + "The hardware H.264 encoder settings are invalid"); + } + check_hresult(MFStartup(MF_VERSION, MFSTARTUP_FULL), + "Could not start Media Foundation"); + media_foundation_started_ = true; + try { + activate_encoder(target_bitrate_bps); + } catch (...) { + MFShutdown(); + media_foundation_started_ = false; + throw; + } + } + + ~Implementation() { + if (transform_) { + static_cast( + transform_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)); + static_cast( + transform_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0)); + static_cast( + transform_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0)); + } + if (activation_) { + static_cast(activation_->ShutdownObject()); + } + if (media_foundation_started_) { + static_cast(MFShutdown()); + } + } + + [[nodiscard]] std::vector + encode(const std::span bgra, + const std::int64_t timestamp_us, const bool force_key_frame) { + std::vector output; + wait_for_input(output); + + if (force_key_frame) { + set_codec_bool(codec_api_.Get(), CODECAPI_AVEncVideoForceKeyFrame, true); + } + const auto nv12 = bgra_to_nv12(bgra, width_, height_); + ComPtr sample; + ComPtr buffer; + check_hresult(MFCreateSample(&sample), + "Could not create an encoder sample"); + check_hresult( + MFCreateMemoryBuffer(static_cast(nv12.size()), &buffer), + "Could not create an encoder input buffer"); + BYTE *destination = nullptr; + DWORD capacity = 0; + check_hresult(buffer->Lock(&destination, &capacity, nullptr), + "Could not lock the encoder input buffer"); + std::copy(nv12.begin(), nv12.end(), destination); + check_hresult(buffer->Unlock(), + "Could not unlock the encoder input buffer"); + check_hresult(buffer->SetCurrentLength(static_cast(nv12.size())), + "Could not size the encoder input buffer"); + check_hresult(sample->AddBuffer(buffer.Get()), + "Could not attach the encoder input buffer"); + check_hresult(sample->SetSampleTime(timestamp_us * 10), + "Could not timestamp the encoder input sample"); + check_hresult(sample->SetSampleDuration(frame_duration_100ns_), + "Could not set the encoder input duration"); + if (first_input_) { + static_cast( + sample->SetUINT32(MFSampleExtension_Discontinuity, TRUE)); + first_input_ = false; + } + check_hresult(transform_->ProcessInput(0, sample.Get(), 0), + "The hardware H.264 encoder rejected an input frame"); + --input_requests_; + pump_events(false, output); + return output; + } + + [[nodiscard]] std::vector + encode_gpu(ID3D11Texture2D &bgra_texture, const std::uint32_t source_width, + const std::uint32_t source_height, const std::int64_t timestamp_us, + const bool force_key_frame) { + D3D11_TEXTURE2D_DESC description{}; + bgra_texture.GetDesc(&description); + if (description.Width != source_width || + description.Height != source_height || + description.Format != DXGI_FORMAT_B8G8R8A8_UNORM) { + throw std::invalid_argument( + "The Media Foundation fallback received an invalid GPU frame"); + } + ComPtr keyed_mutex; + check_hresult(bgra_texture.QueryInterface(IID_PPV_ARGS(&keyed_mutex)), + "The GPU frame has no keyed mutex"); + check_hresult(keyed_mutex->AcquireSync(1, 5'000), + "Could not acquire the GPU frame for fallback readback"); + ComPtr device; + bgra_texture.GetDevice(&device); + ComPtr context; + device->GetImmediateContext(&context); + D3D11_TEXTURE2D_DESC staging_description = description; + staging_description.BindFlags = 0; + staging_description.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + staging_description.MiscFlags = 0; + staging_description.Usage = D3D11_USAGE_STAGING; + ComPtr staging; + try { + check_hresult( + device->CreateTexture2D(&staging_description, nullptr, &staging), + "Could not create the fallback readback texture"); + context->CopyResource(staging.Get(), &bgra_texture); + } catch (...) { + static_cast(keyed_mutex->ReleaseSync(0)); + throw; + } + check_hresult(keyed_mutex->ReleaseSync(0), + "Could not release the GPU frame after fallback readback"); + D3D11_MAPPED_SUBRESOURCE mapped{}; + check_hresult(context->Map(staging.Get(), 0, D3D11_MAP_READ, 0, &mapped), + "Could not map the fallback readback texture"); + std::vector bgra(static_cast(source_width) * + source_height * 4); + const auto row_bytes = static_cast(source_width) * 4; + for (std::uint32_t row = 0; row < source_height; ++row) { + std::copy_n(static_cast(mapped.pData) + + static_cast(row) * mapped.RowPitch, + row_bytes, + bgra.data() + static_cast(row) * row_bytes); + } + context->Unmap(staging.Get(), 0); + auto scaled = scale_bgra_frame(std::move(bgra), source_width, source_height, + width_, height_); + return encode(scaled, timestamp_us, force_key_frame); + } + + void set_target_bitrate(const std::uint32_t target_bitrate_bps) { + if (target_bitrate_bps == 0 || target_bitrate_bps == target_bitrate_bps_) { + return; + } + set_codec_u32(codec_api_.Get(), CODECAPI_AVEncCommonMeanBitRate, + target_bitrate_bps); + target_bitrate_bps_ = + get_codec_u32(codec_api_.Get(), CODECAPI_AVEncCommonMeanBitRate) + .value_or(target_bitrate_bps); + } + + [[nodiscard]] std::uint32_t target_bitrate_bps() const noexcept { + return target_bitrate_bps_; + } + + [[nodiscard]] std::uint32_t rate_control_mode() const noexcept { + return rate_control_mode_; + } + + [[nodiscard]] std::vector finish() { + if (finished_) { + return {}; + } + finished_ = true; + check_hresult( + transform_->ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0), + "Could not end the hardware encoder stream"); + check_hresult(transform_->ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0), + "Could not drain the hardware encoder"); + std::vector output; + while (!drain_complete_) { + pump_one_event(false, output); + } + return output; + } + + [[nodiscard]] const std::string &implementation_name() const noexcept { + return implementation_name_; + } + +private: + void activate_encoder(const std::uint32_t target_bitrate_bps) { + const MFT_REGISTER_TYPE_INFO input_type{MFMediaType_Video, + MFVideoFormat_NV12}; + const MFT_REGISTER_TYPE_INFO output_type{MFMediaType_Video, + MFVideoFormat_H264}; + IMFActivate **activations = nullptr; + UINT32 activation_count = 0; + check_hresult( + MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, + &input_type, &output_type, &activations, &activation_count), + "Could not enumerate hardware H.264 encoders"); + if (activation_count == 0) { + CoTaskMemFree(activations); + throw std::runtime_error( + "Windows did not report a hardware Media Foundation H.264 encoder"); + } + activation_.Attach(activations[0]); + for (UINT32 index = 1; index < activation_count; ++index) { + activations[index]->Release(); + } + CoTaskMemFree(activations); + implementation_name_ = narrow_name(activation_.Get()); + check_hresult(activation_->ActivateObject(IID_PPV_ARGS(&transform_)), + "Could not activate the hardware H.264 encoder"); + + ComPtr attributes; + if (SUCCEEDED(transform_->GetAttributes(&attributes))) { + UINT32 asynchronous = FALSE; + if (SUCCEEDED(attributes->GetUINT32(MF_TRANSFORM_ASYNC, &asynchronous)) && + asynchronous != FALSE) { + check_hresult(attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE), + "Could not unlock the asynchronous H.264 encoder"); + } + static_cast(attributes->SetUINT32(MF_LOW_LATENCY, TRUE)); + } + + static_cast(transform_.As(&codec_api_)); + set_codec_bool(codec_api_.Get(), CODECAPI_AVLowLatencyMode, true); + // Rate-control mode is a static CodecAPI property. Microsoft requires it + // to be set before SetOutputType so that the subsequent media-type change + // activates the requested mode for this encoding session. + set_codec_u32(codec_api_.Get(), CODECAPI_AVEncCommonRateControlMode, + eAVEncCommonRateControlMode_CBR, true); + + auto output = make_video_type(MFVideoFormat_H264, width_, height_, + frames_per_second_); + check_hresult(output->SetUINT32(MF_MT_AVG_BITRATE, target_bitrate_bps), + "Could not set the H.264 output bitrate"); + check_hresult( + output->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base), + "Could not set the H.264 output profile"); + check_hresult(transform_->SetOutputType(0, output.Get(), 0), + "The hardware encoder rejected the H.264 output type"); + auto input = make_video_type(MFVideoFormat_NV12, width_, height_, + frames_per_second_); + check_hresult(input->SetUINT32(MF_MT_DEFAULT_STRIDE, width_), + "Could not set the NV12 input stride"); + check_hresult(transform_->SetInputType(0, input.Get(), 0), + "The hardware encoder rejected the NV12 input type"); + + set_codec_u32(codec_api_.Get(), CODECAPI_AVEncCommonMeanBitRate, + target_bitrate_bps, true); + set_codec_u32(codec_api_.Get(), CODECAPI_AVEncMPVGOPSize, + frames_per_second_ * 2); + target_bitrate_bps_ = target_bitrate_bps; + rate_control_mode_ = + get_codec_u32(codec_api_.Get(), CODECAPI_AVEncCommonRateControlMode) + .value_or(std::numeric_limits::max()); + + check_hresult(transform_.As(&events_), + "The hardware H.264 encoder is not asynchronous"); + check_hresult( + transform_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0), + "Could not begin hardware H.264 streaming"); + check_hresult( + transform_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0), + "Could not start the hardware H.264 stream"); + } + + void wait_for_input(std::vector &output) { + while (input_requests_ == 0) { + pump_one_event(false, output); + } + } + + void pump_events(const bool blocking, + std::vector &output) { + bool first = true; + while (true) { + const HRESULT result = pump_one_event(!blocking || !first, output); + if (result == MF_E_NO_EVENTS_AVAILABLE) { + break; + } + first = false; + if (blocking) { + break; + } + } + } + + HRESULT pump_one_event(const bool no_wait, + std::vector &output) { + ComPtr event; + const HRESULT result = + events_->GetEvent(no_wait ? MF_EVENT_FLAG_NO_WAIT : 0, &event); + if (result == MF_E_NO_EVENTS_AVAILABLE) { + return result; + } + check_hresult(result, "Could not read a hardware encoder event"); + HRESULT event_status = S_OK; + check_hresult(event->GetStatus(&event_status), + "Could not read the hardware encoder event status"); + check_hresult(event_status, "The hardware H.264 encoder reported an error"); + MediaEventType type = MEUnknown; + check_hresult(event->GetType(&type), + "Could not identify a hardware encoder event"); + if (type == METransformNeedInput) { + ++input_requests_; + } else if (type == METransformHaveOutput) { + take_output(output); + } else if (type == METransformDrainComplete) { + drain_complete_ = true; + } + return S_OK; + } + + void take_output(std::vector &output) { + MFT_OUTPUT_STREAM_INFO information{}; + check_hresult(transform_->GetOutputStreamInfo(0, &information), + "Could not query the hardware encoder output stream"); + ComPtr sample; + if ((information.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES) == 0) { + ComPtr buffer; + check_hresult(MFCreateSample(&sample), + "Could not create a hardware encoder output sample"); + check_hresult( + MFCreateMemoryBuffer(std::max(information.cbSize, 1), &buffer), + "Could not create a hardware encoder output buffer"); + check_hresult(sample->AddBuffer(buffer.Get()), + "Could not attach a hardware encoder output buffer"); + } + + MFT_OUTPUT_DATA_BUFFER data{}; + data.dwStreamID = 0; + data.pSample = sample.Get(); + DWORD status = 0; + const HRESULT result = transform_->ProcessOutput(0, 1, &data, &status); + if (data.pEvents != nullptr) { + data.pEvents->Release(); + } + if (result == MF_E_TRANSFORM_NEED_MORE_INPUT) { + return; + } + check_hresult(result, "Could not read a hardware H.264 access unit"); + if (data.pSample != nullptr && sample.Get() != data.pSample) { + sample = data.pSample; + data.pSample->Release(); + } + if (!sample) { + throw std::runtime_error("The hardware H.264 encoder returned no sample"); + } + + ComPtr contiguous; + check_hresult(sample->ConvertToContiguousBuffer(&contiguous), + "Could not combine the H.264 output buffers"); + BYTE *bytes = nullptr; + DWORD length = 0; + check_hresult(contiguous->Lock(&bytes, nullptr, &length), + "Could not lock the H.264 output buffer"); + EncodedH264AccessUnit access_unit; + access_unit.data.assign(bytes, bytes + length); + check_hresult(contiguous->Unlock(), + "Could not unlock the H.264 output buffer"); + LONGLONG timestamp_100ns = 0; + if (SUCCEEDED(sample->GetSampleTime(×tamp_100ns))) { + access_unit.timestamp_us = timestamp_100ns / 10; + } + normalize_h264_access_unit(access_unit.data); + if (!h264_access_unit_is_annex_b(access_unit.data)) { + throw std::runtime_error( + "The hardware H.264 encoder returned an unsupported bitstream"); + } + UINT32 clean_point = FALSE; + access_unit.key_frame = (SUCCEEDED(sample->GetUINT32( + MFSampleExtension_CleanPoint, &clean_point)) && + clean_point != FALSE) || + h264_access_unit_is_key_frame(access_unit.data); + if (!access_unit.data.empty()) { + output.push_back(std::move(access_unit)); + } + } + + std::uint32_t width_; + std::uint32_t height_; + std::uint32_t frames_per_second_; + LONGLONG frame_duration_100ns_; + std::uint32_t target_bitrate_bps_ = 0; + std::uint32_t rate_control_mode_ = std::numeric_limits::max(); + bool media_foundation_started_ = false; + bool first_input_ = true; + bool finished_ = false; + bool drain_complete_ = false; + std::uint32_t input_requests_ = 0; + std::string implementation_name_; + ComPtr activation_; + ComPtr transform_; + ComPtr events_; + ComPtr codec_api_; +}; + +MediaFoundationH264Encoder::MediaFoundationH264Encoder( + const std::uint32_t width, const std::uint32_t height, + const std::uint32_t frames_per_second, + const std::uint32_t target_bitrate_bps) + : implementation_(std::make_unique( + width, height, frames_per_second, target_bitrate_bps)) {} + +MediaFoundationH264Encoder::~MediaFoundationH264Encoder() = default; + +std::vector +MediaFoundationH264Encoder::encode(const std::span bgra, + const std::int64_t timestamp_us, + const bool force_key_frame) { + return implementation_->encode(bgra, timestamp_us, force_key_frame); +} + +std::vector MediaFoundationH264Encoder::encode_gpu( + ID3D11Texture2D &bgra_texture, const std::uint32_t source_width, + const std::uint32_t source_height, const std::int64_t timestamp_us, + const bool force_key_frame) { + return implementation_->encode_gpu(bgra_texture, source_width, source_height, + timestamp_us, force_key_frame); +} + +void MediaFoundationH264Encoder::set_target_bitrate( + const std::uint32_t target_bitrate_bps) { + implementation_->set_target_bitrate(target_bitrate_bps); +} + +std::uint32_t MediaFoundationH264Encoder::target_bitrate_bps() const noexcept { + return implementation_->target_bitrate_bps(); +} + +std::uint32_t MediaFoundationH264Encoder::rate_control_mode() const noexcept { + return implementation_->rate_control_mode(); +} + +std::vector MediaFoundationH264Encoder::finish() { + return implementation_->finish(); +} + +const std::string & +MediaFoundationH264Encoder::implementation_name() const noexcept { + return implementation_->implementation_name(); +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/h264_encoder.h b/apps/desktop/native/windows-capture-probe/src/h264_encoder.h new file mode 100644 index 0000000000..56eb47bea9 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/h264_encoder.h @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace chatto::capture { + +struct EncodedH264AccessUnit { + std::vector data; + std::int64_t timestamp_us = 0; + bool key_frame = false; +}; + +struct H264ProfileLevel { + std::uint8_t profile_idc = 0; + std::uint8_t profile_iop = 0; + std::uint8_t level_idc = 0; +}; + +/** Common interface for a realtime hardware H.264 encoder. */ +class H264Encoder { +public: + virtual ~H264Encoder() = default; + + [[nodiscard]] virtual std::vector + encode(std::span bgra, std::int64_t timestamp_us, + bool force_key_frame) = 0; + /** Scale/convert a GPU BGRA texture and encode it without CPU pixel copies. + */ + [[nodiscard]] virtual std::vector + encode_gpu(ID3D11Texture2D &bgra_texture, std::uint32_t source_width, + std::uint32_t source_height, std::int64_t timestamp_us, + bool force_key_frame) = 0; + virtual void set_target_bitrate(std::uint32_t target_bitrate_bps) = 0; + [[nodiscard]] virtual std::uint32_t target_bitrate_bps() const noexcept = 0; + [[nodiscard]] virtual std::uint32_t rate_control_mode() const noexcept = 0; + [[nodiscard]] virtual std::vector finish() = 0; + [[nodiscard]] virtual double last_gpu_conversion_submit_ms() const noexcept { + return 0; + } + [[nodiscard]] virtual double last_encoder_submit_ms() const noexcept { + return 0; + } + [[nodiscard]] virtual double last_bitstream_wait_ms() const noexcept { + return 0; + } + [[nodiscard]] virtual const std::string & + implementation_name() const noexcept = 0; +}; + +/** Convert a tightly packed, even-sized BGRA frame to BT.709 limited NV12. */ +[[nodiscard]] std::vector +bgra_to_nv12(std::span bgra, std::uint32_t width, + std::uint32_t height); + +/** Whether an Annex-B H.264 access unit contains an IDR slice. */ +[[nodiscard]] bool +h264_access_unit_is_key_frame(std::span access_unit); + +/** Whether an H.264 access unit begins with an Annex-B start code. */ +[[nodiscard]] bool +h264_access_unit_is_annex_b(std::span access_unit); + +/** Read profile-level-id bytes from the first Annex-B SPS, when present. */ +[[nodiscard]] std::optional +h264_access_unit_profile_level(std::span access_unit); + +/** Low-latency hardware H.264 encoder backed by Media Foundation. */ +class MediaFoundationH264Encoder final : public H264Encoder { +public: + MediaFoundationH264Encoder(std::uint32_t width, std::uint32_t height, + std::uint32_t frames_per_second, + std::uint32_t target_bitrate_bps); + ~MediaFoundationH264Encoder() override; + + MediaFoundationH264Encoder(const MediaFoundationH264Encoder &) = delete; + MediaFoundationH264Encoder & + operator=(const MediaFoundationH264Encoder &) = delete; + + /** Encode one BGRA frame and return any access units now available. */ + [[nodiscard]] std::vector + encode(std::span bgra, std::int64_t timestamp_us, + bool force_key_frame) override; + + [[nodiscard]] std::vector + encode_gpu(ID3D11Texture2D &bgra_texture, std::uint32_t source_width, + std::uint32_t source_height, std::int64_t timestamp_us, + bool force_key_frame) override; + + /** Apply the latest WebRTC target bitrate when the encoder supports it. */ + void set_target_bitrate(std::uint32_t target_bitrate_bps) override; + + /** Latest bitrate value accepted and reported by the encoder. */ + [[nodiscard]] std::uint32_t target_bitrate_bps() const noexcept override; + + /** Active Media Foundation rate-control mode reported by the encoder. */ + [[nodiscard]] std::uint32_t rate_control_mode() const noexcept override; + + /** Drain delayed access units before shutting the publisher down. */ + [[nodiscard]] std::vector finish() override; + + /** Friendly name reported by the selected hardware Media Foundation MFT. */ + [[nodiscard]] const std::string & + implementation_name() const noexcept override; + +private: + class Implementation; + std::unique_ptr implementation_; +}; + +/** Prefer direct NVENC and fall back to the Windows Media Foundation MFT. */ +[[nodiscard]] std::unique_ptr +create_hardware_h264_encoder(std::uint32_t width, std::uint32_t height, + std::uint32_t frames_per_second, + std::uint32_t target_bitrate_bps); + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/latest_frame_queue.h b/apps/desktop/native/windows-capture-probe/src/latest_frame_queue.h new file mode 100644 index 0000000000..e6547adaaf --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/latest_frame_queue.h @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +namespace chatto::capture { + +// A single-slot handoff for realtime media. Producers replace stale work +// instead of allowing latency to accumulate behind a slow consumer. +template class LatestFrameQueue final { +public: + LatestFrameQueue() = default; + LatestFrameQueue(const LatestFrameQueue &) = delete; + LatestFrameQueue &operator=(const LatestFrameQueue &) = delete; + + // Returns true when an older pending value was dropped. + [[nodiscard]] bool push(Value value) { + std::scoped_lock lock(mutex_); + if (closed_) { + return false; + } + const bool replaced = pending_.has_value(); + pending_ = std::move(value); + changed_.notify_one(); + return replaced; + } + + // Drains the final pending value after close, then returns nullopt. + [[nodiscard]] std::optional wait_pop() { + std::unique_lock lock(mutex_); + changed_.wait(lock, [this] { return closed_ || pending_.has_value(); }); + if (!pending_) { + return std::nullopt; + } + auto value = std::move(pending_); + pending_.reset(); + return value; + } + + void close() { + std::scoped_lock lock(mutex_); + closed_ = true; + changed_.notify_all(); + } + +private: + std::mutex mutex_; + std::condition_variable changed_; + std::optional pending_; + bool closed_ = false; +}; + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/live_status.h b/apps/desktop/native/windows-capture-probe/src/live_status.h new file mode 100644 index 0000000000..d244300509 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/live_status.h @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace chatto::capture { + +// Cross-thread latest-value diagnostics for the optional local preview. +struct LiveCaptureStatus { + std::atomic audio_frames = 0; + std::atomic audio_discontinuities = 0; + std::atomic latest_audio_peak = 0; +}; + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/livekit_publisher.cpp b/apps/desktop/native/windows-capture-probe/src/livekit_publisher.cpp new file mode 100644 index 0000000000..e01145f039 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/livekit_publisher.cpp @@ -0,0 +1,1167 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "livekit_publisher.h" + +#include "audio_capture.h" +#include "h264_encoder.h" +#include "latest_frame_queue.h" +#include "video_capture.h" +#include "window_sources.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +class LiveKitRuntime final { +public: + LiveKitRuntime() : initialized_here_(livekit::initialize()) {} + + ~LiveKitRuntime() { + if (initialized_here_) { + livekit::shutdown(); + } + } + + LiveKitRuntime(const LiveKitRuntime &) = delete; + LiveKitRuntime &operator=(const LiveKitRuntime &) = delete; + +private: + bool initialized_here_; +}; + +[[nodiscard]] std::int16_t float_to_pcm16(const float sample) { + const float bounded = std::clamp(sample, -1.0F, 1.0F); + return static_cast(std::lrint( + bounded * static_cast(std::numeric_limits::max()))); +} + +[[nodiscard]] std::pair +bounded_size(const std::uint32_t width, const std::uint32_t height, + const std::uint32_t maximum_width, + const std::uint32_t maximum_height) { + const double scale = std::min({ + 1.0, + static_cast(maximum_width) / width, + static_cast(maximum_height) / height, + }); + const auto even_dimension = [](const double value) { + return std::max(2U, static_cast(value) & ~1U); + }; + return {even_dimension(width * scale), even_dimension(height * scale)}; +} + +struct VideoPumpMetrics { + std::uint64_t submitted = 0; + std::uint64_t published = 0; + std::uint64_t dropped = 0; +}; + +struct RtcVideoMetrics { + std::mutex mutex; + bool available = false; + std::uint32_t outbound_streams = 0; + std::uint32_t active_outbound_streams = 0; + double minimum_active_fps = 0; + double maximum_active_fps = 0; + std::uint64_t frames_encoded = 0; + std::uint64_t frames_sent = 0; + std::uint64_t bytes_sent = 0; + std::uint64_t retransmitted_packets_sent = 0; + std::uint64_t retransmitted_bytes_sent = 0; + std::uint32_t nack_count = 0; + std::uint32_t pli_count = 0; + double target_bitrate = 0; + double average_encode_ms = 0; + std::uint32_t encoded_width = 0; + std::uint32_t encoded_height = 0; + double average_qp = 0; + std::string encoder_implementation; + std::uint32_t cpu_limited_streams = 0; + std::uint32_t bandwidth_limited_streams = 0; + std::uint32_t power_efficient_streams = 0; + bool remote_inbound_available = false; + std::int64_t remote_packets_lost = 0; + double remote_jitter_seconds = 0; + double remote_fraction_lost = 0; + double remote_round_trip_time_ms = 0; + bool candidate_pair_available = false; + double available_outgoing_bitrate = 0; + double current_round_trip_time_ms = 0; + std::uint32_t packets_discarded_on_send = 0; + std::uint64_t bytes_discarded_on_send = 0; +}; + +enum class CaptureBackend { + WgcWindow, + WgcMonitor, + DxgiDisplay, +}; + +// Keep scaling, hardware encoding and the synchronous LiveKit FFI off the +// Windows Graphics Capture callback. Realtime capture replaces the one pending +// frame rather than accumulating latency when any downstream stage falls +// behind. +class LiveKitVideoPump final { +public: + LiveKitVideoPump(std::shared_ptr video_source, + std::shared_ptr rtc_metrics, + const std::uint32_t output_width, + const std::uint32_t output_height, + const std::uint32_t frames_per_second, + const std::uint32_t target_bitrate_bps, + EncodedPreviewCallback preview_callback) + : video_source_(std::move(video_source)), + rtc_metrics_(std::move(rtc_metrics)), output_width_(output_width), + output_height_(output_height), frames_per_second_(frames_per_second), + target_bitrate_bps_(target_bitrate_bps), + preview_callback_(std::move(preview_callback)), + requested_encoder_bitrate_bps_(target_bitrate_bps), + requested_encoder_fps_(static_cast(frames_per_second)), + worker_([this] { run(); }), reporter_([this] { report(); }) {} + + ~LiveKitVideoPump() { + queue_.close(); + if (worker_.joinable()) { + worker_.join(); + } + stop_reporter(); + } + + LiveKitVideoPump(const LiveKitVideoPump &) = delete; + LiveKitVideoPump &operator=(const LiveKitVideoPump &) = delete; + + void submit(VideoFrameData frame) { + rethrow_failure(); + submitted_.fetch_add(1, std::memory_order_relaxed); + const auto dimensions = + (static_cast(frame.width) << 32U) | frame.height; + const auto previous_dimensions = + latest_dimensions_.exchange(dimensions, std::memory_order_relaxed); + if (previous_dimensions != 0 && previous_dimensions != dimensions) { + dimension_changes_.fetch_add(1, std::memory_order_relaxed); + } + gpu_copy_submit_microseconds_.fetch_add( + static_cast(frame.gpu_copy_submit_duration_ms * 1'000.0), + std::memory_order_relaxed); + if (queue_.push(std::move(frame))) { + dropped_.fetch_add(1, std::memory_order_relaxed); + } + } + + void set_capture_backend(const CaptureBackend backend) { + capture_backend_.store(backend, std::memory_order_relaxed); + } + + [[nodiscard]] VideoPumpMetrics finish() { + queue_.close(); + if (worker_.joinable()) { + worker_.join(); + } + stop_reporter(); + rethrow_failure(); + return { + .submitted = submitted_.load(std::memory_order_relaxed), + .published = published_.load(std::memory_order_relaxed), + .dropped = dropped_.load(std::memory_order_relaxed), + }; + } + +private: + void run() noexcept { + try { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + const auto encoder_width = output_width_; + const auto encoder_height = output_height_; + auto encoder = + create_hardware_h264_encoder(output_width_, output_height_, + frames_per_second_, target_bitrate_bps_); + { + std::scoped_lock lock(encoder_metrics_mutex_); + hardware_encoder_implementation_ = encoder->implementation_name(); + } + encoder_width_.store(encoder_width, std::memory_order_relaxed); + encoder_height_.store(encoder_height, std::memory_order_relaxed); + applied_encoder_bitrate_bps_.store(encoder->target_bitrate_bps(), + std::memory_order_relaxed); + encoder_rate_control_mode_.store(encoder->rate_control_mode(), + std::memory_order_relaxed); + bool keyframe_request_in_flight = false; + while (auto frame = queue_.wait_pop()) { + const std::int64_t timestamp_100ns = frame->timestamp_100ns; + const auto feedback = video_source_->takeFeedback(); + if (feedback.rate_control && + feedback.rate_control->target_bitrate_bps > 0) { + const auto target = + static_cast(std::min( + feedback.rate_control->target_bitrate_bps, + std::numeric_limits::max())); + requested_encoder_bitrate_bps_.store(target, + std::memory_order_relaxed); + requested_encoder_fps_.store(feedback.rate_control->framerate_fps, + std::memory_order_relaxed); + encoder->set_target_bitrate(target); + applied_encoder_bitrate_bps_.store(encoder->target_bitrate_bps(), + std::memory_order_relaxed); + } + const auto encode_start = std::chrono::steady_clock::now(); + // WebRTC repeats its keyframe request until the requested IDR reaches + // the RTP sender. An asynchronous MFT can have several frames in + // flight, so forwarding every repetition would create a burst of IDRs + // and waste much of the available bitrate. + const bool force_key_frame = + feedback.keyframe_requested && !keyframe_request_in_flight; + keyframe_request_in_flight |= force_key_frame; + auto access_units = encoder->encode_gpu( + *frame->bgra_texture, frame->width, frame->height, + timestamp_100ns / 10, force_key_frame); + if (std::any_of(access_units.begin(), access_units.end(), + [](const auto &access_unit) { + return access_unit.key_frame; + })) { + keyframe_request_in_flight = false; + } + const auto encode_end = std::chrono::steady_clock::now(); + gpu_conversion_submit_microseconds_.fetch_add( + static_cast( + encoder->last_gpu_conversion_submit_ms() * 1'000.0), + std::memory_order_relaxed); + encoder_submit_microseconds_.fetch_add( + static_cast(encoder->last_encoder_submit_ms() * + 1'000.0), + std::memory_order_relaxed); + bitstream_wait_microseconds_.fetch_add( + static_cast(encoder->last_bitstream_wait_ms() * + 1'000.0), + std::memory_order_relaxed); + encode_microseconds_.fetch_add( + static_cast( + std::chrono::duration(encode_end - + encode_start) + .count()), + std::memory_order_relaxed); + encoded_.fetch_add(1, std::memory_order_relaxed); + publish_access_units(std::move(access_units), encoder_width, + encoder_height); + } + publish_access_units(encoder->finish(), encoder_width, encoder_height); + } catch (...) { + { + std::scoped_lock lock(failure_mutex_); + failure_ = std::current_exception(); + } + queue_.close(); + } + } + + void publish_access_units(std::vector access_units, + const std::uint32_t encoded_width, + const std::uint32_t encoded_height) { + for (auto &access_unit : access_units) { + hardware_encoded_frames_.fetch_add(1, std::memory_order_relaxed); + hardware_encoded_bytes_.fetch_add(access_unit.data.size(), + std::memory_order_relaxed); + if (access_unit.key_frame) { + hardware_key_frames_.fetch_add(1, std::memory_order_relaxed); + } + if (preview_callback_) { + preview_callback_(access_unit.data, access_unit.timestamp_us, + access_unit.key_frame); + } + livekit::EncodedVideoFrame frame; + frame.data = access_unit.data.data(); + frame.size = access_unit.data.size(); + frame.codec = livekit::EncodedVideoCodec::H264; + frame.frame_type = access_unit.key_frame + ? livekit::EncodedVideoFrameType::Key + : livekit::EncodedVideoFrameType::Delta; + frame.timestamp_us = access_unit.timestamp_us; + frame.width = encoded_width; + frame.height = encoded_height; + const auto publish_start = std::chrono::steady_clock::now(); + const bool accepted = video_source_->captureFrame(frame); + const auto publish_end = std::chrono::steady_clock::now(); + const auto publish_microseconds = static_cast( + std::chrono::duration(publish_end - publish_start) + .count()); + publish_microseconds_.fetch_add(publish_microseconds, + std::memory_order_relaxed); + last_publish_microseconds_.store(publish_microseconds, + std::memory_order_relaxed); + if (accepted) { + published_.fetch_add(1, std::memory_order_relaxed); + } else { + dropped_.fetch_add(1, std::memory_order_relaxed); + } + } + } + + void report() noexcept { + std::unique_lock lock(reporter_mutex_); + auto previous_report_at = started_; + std::uint64_t previous_hardware_encoded_bytes = 0; + while (!reporter_changed_.wait_for(lock, std::chrono::seconds(2), + [this] { return reporter_stopping_; })) { + lock.unlock(); + const auto now = std::chrono::steady_clock::now(); + const auto hardware_encoded_bytes = + hardware_encoded_bytes_.load(std::memory_order_relaxed); + const double interval_seconds = + std::chrono::duration(now - previous_report_at).count(); + const double actual_hardware_bitrate = + interval_seconds > 0 + ? static_cast(hardware_encoded_bytes - + previous_hardware_encoded_bytes) * + 8.0 / interval_seconds + : 0; + emit_metrics(last_publish_microseconds_.load(std::memory_order_relaxed), + now, actual_hardware_bitrate); + previous_report_at = now; + previous_hardware_encoded_bytes = hardware_encoded_bytes; + lock.lock(); + } + } + + void stop_reporter() { + { + std::scoped_lock lock(reporter_mutex_); + reporter_stopping_ = true; + } + reporter_changed_.notify_all(); + if (reporter_.joinable()) { + reporter_.join(); + } + } + + void emit_metrics(const std::uint64_t last_publish_microseconds, + const std::chrono::steady_clock::time_point now, + const double actual_hardware_bitrate) const { + const auto submitted = submitted_.load(std::memory_order_relaxed); + const auto published = published_.load(std::memory_order_relaxed); + const double elapsed_seconds = + std::chrono::duration(now - started_).count(); + const double average_gpu_copy_submit_ms = + submitted == 0 ? 0 + : static_cast(gpu_copy_submit_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(submitted) / 1'000.0; + const double average_publish_ms = + published == 0 ? 0 + : static_cast(publish_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(published) / 1'000.0; + const auto encoded = encoded_.load(std::memory_order_relaxed); + const double average_hardware_encode_ms = + encoded == 0 ? 0 + : static_cast(encode_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(encoded) / 1'000.0; + const double average_gpu_conversion_submit_ms = + encoded == 0 + ? 0 + : static_cast(gpu_conversion_submit_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(encoded) / 1'000.0; + const double average_encoder_submit_ms = + encoded == 0 ? 0 + : static_cast(encoder_submit_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(encoded) / 1'000.0; + const double average_bitstream_wait_ms = + encoded == 0 ? 0 + : static_cast(bitstream_wait_microseconds_.load( + std::memory_order_relaxed)) / + static_cast(encoded) / 1'000.0; + const auto dimensions = latest_dimensions_.load(std::memory_order_relaxed); + std::string hardware_encoder; + { + std::scoped_lock encoder_lock(encoder_metrics_mutex_); + hardware_encoder = hardware_encoder_implementation_; + } + std::scoped_lock rtc_lock(rtc_metrics_->mutex); + std::cout + << std::fixed << std::setprecision(3) + << "{\"protocolVersion\":1,\"kind\":\"metrics\"" + << ",\"submittedFrames\":" << submitted + << ",\"publishedFrames\":" << published + << ",\"droppedFrames\":" << dropped_.load(std::memory_order_relaxed) + << ",\"captureFps\":" + << (elapsed_seconds > 0 ? submitted / elapsed_seconds : 0) + << ",\"publishFps\":" + << (elapsed_seconds > 0 ? published / elapsed_seconds : 0) + << ",\"averageReadbackMs\":0" + << ",\"averageScaleMs\":0" + << ",\"averageGpuCopySubmitMs\":" << average_gpu_copy_submit_ms + << ",\"averageGpuConversionSubmitMs\":" + << average_gpu_conversion_submit_ms + << ",\"averageEncoderSubmitMs\":" << average_encoder_submit_ms + << ",\"averageBitstreamWaitMs\":" << average_bitstream_wait_ms + << ",\"averagePublishMs\":" << average_publish_ms + << ",\"averageHardwareEncodeMs\":" << average_hardware_encode_ms + << ",\"hardwareEncoderImplementation\":" + << std::quoted(hardware_encoder) << ",\"requestedEncoderBitrate\":" + << requested_encoder_bitrate_bps_.load(std::memory_order_relaxed) + << ",\"appliedEncoderBitrate\":" + << applied_encoder_bitrate_bps_.load(std::memory_order_relaxed) + << ",\"actualHardwareBitrate\":" << actual_hardware_bitrate + << ",\"encoderRateControlMode\":" + << encoder_rate_control_mode_.load(std::memory_order_relaxed) + << ",\"requestedEncoderFps\":" + << requested_encoder_fps_.load(std::memory_order_relaxed) + << ",\"hardwareEncodedFrames\":" + << hardware_encoded_frames_.load(std::memory_order_relaxed) + << ",\"hardwareEncodedBytes\":" + << hardware_encoded_bytes_.load(std::memory_order_relaxed) + << ",\"hardwareKeyFrames\":" + << hardware_key_frames_.load(std::memory_order_relaxed) + << ",\"hardwareEncodedWidth\":" + << encoder_width_.load(std::memory_order_relaxed) + << ",\"hardwareEncodedHeight\":" + << encoder_height_.load(std::memory_order_relaxed) + << ",\"encoderResolutionChanges\":" + << encoder_resolution_changes_.load(std::memory_order_relaxed) + << ",\"lastPublishMs\":" + << static_cast(last_publish_microseconds) / 1'000.0 + << ",\"sourceWidth\":" << (dimensions >> 32U) << ",\"sourceHeight\":" + << (dimensions & std::numeric_limits::max()) + << ",\"dimensionChanges\":" + << dimension_changes_.load(std::memory_order_relaxed) + << ",\"captureBackend\":\"" + << capture_backend_name( + capture_backend_.load(std::memory_order_relaxed)) + << "\"" + << ",\"rtcStatsAvailable\":" + << (rtc_metrics_->available ? "true" : "false") + << ",\"outboundStreams\":" << rtc_metrics_->outbound_streams + << ",\"activeOutboundStreams\":" + << rtc_metrics_->active_outbound_streams + << ",\"minimumActiveOutboundFps\":" << rtc_metrics_->minimum_active_fps + << ",\"maximumActiveOutboundFps\":" << rtc_metrics_->maximum_active_fps + << ",\"framesEncoded\":" << rtc_metrics_->frames_encoded + << ",\"framesSent\":" << rtc_metrics_->frames_sent + << ",\"bytesSent\":" << rtc_metrics_->bytes_sent + << ",\"retransmittedPacketsSent\":" + << rtc_metrics_->retransmitted_packets_sent + << ",\"retransmittedBytesSent\":" + << rtc_metrics_->retransmitted_bytes_sent + << ",\"nackCount\":" << rtc_metrics_->nack_count + << ",\"pliCount\":" << rtc_metrics_->pli_count + << ",\"targetBitrate\":" << rtc_metrics_->target_bitrate + << ",\"averageEncodeMs\":" << rtc_metrics_->average_encode_ms + << ",\"encodedWidth\":" << rtc_metrics_->encoded_width + << ",\"encodedHeight\":" << rtc_metrics_->encoded_height + << ",\"averageQp\":" << rtc_metrics_->average_qp + << ",\"encoderImplementation\":" + << std::quoted(rtc_metrics_->encoder_implementation) + << ",\"cpuLimitedStreams\":" << rtc_metrics_->cpu_limited_streams + << ",\"bandwidthLimitedStreams\":" + << rtc_metrics_->bandwidth_limited_streams + << ",\"powerEfficientStreams\":" + << rtc_metrics_->power_efficient_streams + << ",\"remoteInboundStatsAvailable\":" + << (rtc_metrics_->remote_inbound_available ? "true" : "false") + << ",\"remotePacketsLost\":" << rtc_metrics_->remote_packets_lost + << ",\"remoteJitterSeconds\":" << rtc_metrics_->remote_jitter_seconds + << ",\"remoteFractionLost\":" << rtc_metrics_->remote_fraction_lost + << ",\"remoteRoundTripTimeMs\":" + << rtc_metrics_->remote_round_trip_time_ms + << ",\"candidatePairStatsAvailable\":" + << (rtc_metrics_->candidate_pair_available ? "true" : "false") + << ",\"availableOutgoingBitrate\":" + << rtc_metrics_->available_outgoing_bitrate + << ",\"currentRoundTripTimeMs\":" + << rtc_metrics_->current_round_trip_time_ms + << ",\"packetsDiscardedOnSend\":" + << rtc_metrics_->packets_discarded_on_send + << ",\"bytesDiscardedOnSend\":" << rtc_metrics_->bytes_discarded_on_send + << "}\n" + << std::flush; + } + + void rethrow_failure() { + std::scoped_lock lock(failure_mutex_); + if (failure_) { + std::rethrow_exception(failure_); + } + } + + [[nodiscard]] static const char * + capture_backend_name(const CaptureBackend backend) { + switch (backend) { + case CaptureBackend::WgcWindow: + return "wgc-window"; + case CaptureBackend::WgcMonitor: + return "wgc-monitor"; + case CaptureBackend::DxgiDisplay: + return "dxgi-display"; + } + return "unknown"; + } + + std::shared_ptr video_source_; + std::shared_ptr rtc_metrics_; + EncodedPreviewCallback preview_callback_; + std::uint32_t output_width_; + std::uint32_t output_height_; + std::uint32_t frames_per_second_; + std::uint32_t target_bitrate_bps_; + LatestFrameQueue queue_; + std::mutex failure_mutex_; + std::exception_ptr failure_; + std::atomic submitted_{0}; + std::atomic published_{0}; + std::atomic dropped_{0}; + std::atomic gpu_copy_submit_microseconds_{0}; + std::atomic gpu_conversion_submit_microseconds_{0}; + std::atomic encoder_submit_microseconds_{0}; + std::atomic bitstream_wait_microseconds_{0}; + std::atomic publish_microseconds_{0}; + std::atomic encode_microseconds_{0}; + std::atomic encoded_{0}; + std::atomic requested_encoder_bitrate_bps_{0}; + std::atomic applied_encoder_bitrate_bps_{0}; + std::atomic encoder_rate_control_mode_{ + std::numeric_limits::max()}; + std::atomic requested_encoder_fps_{0}; + std::atomic hardware_encoded_frames_{0}; + std::atomic hardware_encoded_bytes_{0}; + std::atomic hardware_key_frames_{0}; + std::atomic encoder_width_{0}; + std::atomic encoder_height_{0}; + std::atomic encoder_resolution_changes_{0}; + std::atomic last_publish_microseconds_{0}; + std::atomic latest_dimensions_{0}; + std::atomic dimension_changes_{0}; + std::atomic capture_backend_{CaptureBackend::WgcWindow}; + std::chrono::steady_clock::time_point started_ = + std::chrono::steady_clock::now(); + std::thread worker_; + std::mutex reporter_mutex_; + std::condition_variable reporter_changed_; + bool reporter_stopping_ = false; + std::thread reporter_; + mutable std::mutex encoder_metrics_mutex_; + std::string hardware_encoder_implementation_; +}; + +class LiveKitVideoStatsReporter final { +public: + LiveKitVideoStatsReporter( + std::shared_ptr video_track, + std::shared_ptr metrics) + : video_track_(std::move(video_track)), metrics_(std::move(metrics)), + worker_([this] { run(); }) {} + + ~LiveKitVideoStatsReporter() { + { + std::scoped_lock lock(stop_mutex_); + stopping_ = true; + } + stop_changed_.notify_all(); + if (worker_.joinable()) { + worker_.join(); + } + } + + LiveKitVideoStatsReporter(const LiveKitVideoStatsReporter &) = delete; + LiveKitVideoStatsReporter & + operator=(const LiveKitVideoStatsReporter &) = delete; + +private: + void run() noexcept { + std::unique_lock stop_lock(stop_mutex_); + while (!stop_changed_.wait_for(stop_lock, std::chrono::seconds(2), + [this] { return stopping_; })) { + stop_lock.unlock(); + try { + auto pending = video_track_->getStats(); + if (pending.wait_for(std::chrono::seconds(1)) == + std::future_status::ready) { + update(pending.get()); + } + } catch (...) { + // Diagnostics are best-effort and must never stop publication. + } + stop_lock.lock(); + } + } + + void update(const std::vector &stats) { + RtcVideoMetrics next; + std::uint32_t encoded_streams = 0; + std::uint64_t encoded_frames = 0; + std::uint64_t qp_sum = 0; + for (const auto &stat : stats) { + const auto *outbound = + std::get_if(&stat.stats); + if (!outbound || outbound->stream.kind != "video") { + if (const auto *remote = + std::get_if(&stat.stats); + remote && remote->stream.kind == "video") { + next.remote_inbound_available = true; + next.remote_packets_lost += remote->received.packets_lost; + next.remote_jitter_seconds = + std::max(next.remote_jitter_seconds, remote->received.jitter); + next.remote_fraction_lost = std::max( + next.remote_fraction_lost, remote->remote_inbound.fraction_lost); + next.remote_round_trip_time_ms = + std::max(next.remote_round_trip_time_ms, + remote->remote_inbound.round_trip_time * 1'000.0); + } else if (const auto *pair = + std::get_if(&stat.stats); + pair && pair->candidate_pair.nominated && + pair->candidate_pair.state == + livekit::IceCandidatePairState::Succeeded) { + next.candidate_pair_available = true; + next.available_outgoing_bitrate = + std::max(next.available_outgoing_bitrate, + pair->candidate_pair.available_outgoing_bitrate); + next.current_round_trip_time_ms = + std::max(next.current_round_trip_time_ms, + pair->candidate_pair.current_round_trip_time * 1'000.0); + next.packets_discarded_on_send += + pair->candidate_pair.packets_discarded_on_send; + next.bytes_discarded_on_send += + pair->candidate_pair.bytes_discarded_on_send; + } + continue; + } + next.outbound_streams += 1; + next.frames_encoded += outbound->outbound.frames_encoded; + next.frames_sent += outbound->outbound.frames_sent; + next.bytes_sent += outbound->sent.bytes_sent; + next.retransmitted_packets_sent += + outbound->outbound.retransmitted_packets_sent; + next.retransmitted_bytes_sent += + outbound->outbound.retransmitted_bytes_sent; + next.nack_count += outbound->outbound.nack_count; + next.pli_count += outbound->outbound.pli_count; + next.target_bitrate += outbound->outbound.target_bitrate; + next.encoded_width = + std::max(next.encoded_width, outbound->outbound.frame_width); + next.encoded_height = + std::max(next.encoded_height, outbound->outbound.frame_height); + encoded_frames += outbound->outbound.frames_encoded; + qp_sum += outbound->outbound.qp_sum; + if (next.encoder_implementation.empty()) { + next.encoder_implementation = outbound->outbound.encoder_implementation; + } + if (outbound->outbound.power_efficient_encoder) { + next.power_efficient_streams += 1; + } + if (outbound->outbound.quality_limitation_reason == + livekit::QualityLimitationReason::Cpu) { + next.cpu_limited_streams += 1; + } else if (outbound->outbound.quality_limitation_reason == + livekit::QualityLimitationReason::Bandwidth) { + next.bandwidth_limited_streams += 1; + } + if (outbound->outbound.active) { + next.active_outbound_streams += 1; + const double fps = outbound->outbound.frames_per_second; + if (next.active_outbound_streams == 1) { + next.minimum_active_fps = fps; + } else { + next.minimum_active_fps = std::min(next.minimum_active_fps, fps); + } + next.maximum_active_fps = std::max(next.maximum_active_fps, fps); + } + if (outbound->outbound.frames_encoded > 0) { + encoded_streams += 1; + next.average_encode_ms += + outbound->outbound.total_encode_time / + static_cast(outbound->outbound.frames_encoded) * 1'000.0; + } + } + if (next.outbound_streams > 0) { + next.available = true; + } + if (encoded_streams > 0) { + next.average_encode_ms /= encoded_streams; + } + if (encoded_frames > 0) { + next.average_qp = + static_cast(qp_sum) / static_cast(encoded_frames); + } + std::scoped_lock lock(metrics_->mutex); + metrics_->available = next.available; + metrics_->outbound_streams = next.outbound_streams; + metrics_->active_outbound_streams = next.active_outbound_streams; + metrics_->minimum_active_fps = next.minimum_active_fps; + metrics_->maximum_active_fps = next.maximum_active_fps; + metrics_->frames_encoded = next.frames_encoded; + metrics_->frames_sent = next.frames_sent; + metrics_->bytes_sent = next.bytes_sent; + metrics_->retransmitted_packets_sent = next.retransmitted_packets_sent; + metrics_->retransmitted_bytes_sent = next.retransmitted_bytes_sent; + metrics_->nack_count = next.nack_count; + metrics_->pli_count = next.pli_count; + metrics_->target_bitrate = next.target_bitrate; + metrics_->average_encode_ms = next.average_encode_ms; + metrics_->encoded_width = next.encoded_width; + metrics_->encoded_height = next.encoded_height; + metrics_->average_qp = next.average_qp; + metrics_->encoder_implementation = std::move(next.encoder_implementation); + metrics_->cpu_limited_streams = next.cpu_limited_streams; + metrics_->bandwidth_limited_streams = next.bandwidth_limited_streams; + metrics_->power_efficient_streams = next.power_efficient_streams; + metrics_->remote_inbound_available = next.remote_inbound_available; + metrics_->remote_packets_lost = next.remote_packets_lost; + metrics_->remote_jitter_seconds = next.remote_jitter_seconds; + metrics_->remote_fraction_lost = next.remote_fraction_lost; + metrics_->remote_round_trip_time_ms = next.remote_round_trip_time_ms; + metrics_->candidate_pair_available = next.candidate_pair_available; + metrics_->available_outgoing_bitrate = next.available_outgoing_bitrate; + metrics_->current_round_trip_time_ms = next.current_round_trip_time_ms; + metrics_->packets_discarded_on_send = next.packets_discarded_on_send; + metrics_->bytes_discarded_on_send = next.bytes_discarded_on_send; + } + + std::shared_ptr video_track_; + std::shared_ptr metrics_; + std::mutex stop_mutex_; + std::condition_variable stop_changed_; + bool stopping_ = false; + std::thread worker_; +}; + +constexpr auto kFrameStallTimeout = std::chrono::seconds(2); +constexpr auto kReplacementWindowTimeout = std::chrono::seconds(3); +constexpr std::uint32_t kInitialVideoBitrateBps = 12'000'000; + +[[nodiscard]] std::optional wait_for_replacement_window( + HWND stale_window, const std::wstring &expected_application_identifier, + const DWORD preferred_process_id, const bool allow_stale_window, + const std::stop_token stop_token) { + const auto deadline = + std::chrono::steady_clock::now() + kReplacementWindowTimeout; + do { + const auto replacement = select_replacement_window_source( + enumerate_window_sources(), expected_application_identifier, + preferred_process_id, stale_window); + if (replacement) { + return replacement->handle; + } + if (allow_stale_window && is_window_capture_candidate(stale_window) && + window_matches_application(stale_window, + expected_application_identifier)) { + return stale_window; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } while (!stop_token.stop_requested() && + std::chrono::steady_clock::now() < deadline); + return std::nullopt; +} + +} // namespace + +int publish_window( + HWND window, const std::wstring &expected_application_identifier, + const std::uint32_t frames_per_second, const std::uint32_t maximum_width, + const std::uint32_t maximum_height, const PublisherCredential &credential, + EncodedPreviewCallback preview_callback, const std::stop_token stop_token) { + if (!window_matches_application(window, expected_application_identifier)) { + throw std::invalid_argument( + "The selected window no longer belongs to the offered application"); + } + if (credential.livekit_url.empty() || credential.token.empty() || + credential.e2ee_key.empty()) { + throw std::invalid_argument("The publisher credential is incomplete"); + } + + // LiveKit requires process-global initialization before constructing any + // source, track, or room. Declare the guard first so shutdown runs only + // after every SDK-backed object below has been destroyed. + LiveKitRuntime livekit_runtime; + const auto [source_width, source_height] = window_capture_size(window); + const auto [width, height] = + bounded_size(source_width, source_height, maximum_width, maximum_height); + auto video_source = + std::make_shared(width, height); + auto audio_source = std::make_shared(48'000, 2, 0); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack( + "game-capture-video", video_source); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack( + "game-capture-audio", audio_source); + auto rtc_metrics = std::make_shared(); + LiveKitVideoPump video_pump(video_source, rtc_metrics, width, height, + frames_per_second, kInitialVideoBitrateBps, + std::move(preview_callback)); + + livekit::RoomOptions room_options; + room_options.auto_subscribe = false; + // This publisher has one video layer and must outlive Desktop's local + // preview subscription. With dynacast enabled, covering or minimizing the + // Electron window can make its adaptive receiver stop consuming the track; + // LiveKit then pauses the helper's only layer even though capture continues. + room_options.dynacast = false; + livekit::E2EEOptions encryption; + encryption.key_provider_options.shared_key = std::vector( + credential.e2ee_key.begin(), credential.e2ee_key.end()); + room_options.encryption = std::move(encryption); + + livekit::Room room; + if (!room.connect(credential.livekit_url, credential.token, room_options)) { + throw std::runtime_error( + "The native publisher could not connect to LiveKit"); + } + if (const auto manager = room.e2eeManager().lock()) { + manager->setEnabled(true); + } else { + throw std::runtime_error("The native publisher could not enable E2EE"); + } + const auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("The native publisher has no local participant"); + } + + livekit::TrackPublishOptions video_options; + video_options.video_encoding = livekit::VideoEncodingOptions{ + .max_bitrate = kInitialVideoBitrateBps, + .max_framerate = static_cast(frames_per_second), + }; + video_options.video_codec = livekit::VideoCodec::H264; + video_options.video_encoder = livekit::VideoEncoderBackend::PreEncoded; + // LiveKit C++ 1.7 only exposes a simulcast switch, not custom screen-share + // layers. Its default lowest screen-share layer is capped at 3 fps, which + // adaptive-stream receivers select for Chatto's compact call tile. Publish + // one full-cadence layer until the SDK lets us define a game-oriented ladder. + video_options.simulcast = false; + video_options.source = livekit::TrackSource::SOURCE_SCREENSHARE; + video_options.stream = "game-capture"; + video_options.degradation_preference = + livekit::DegradationPreference::MaintainFramerate; + participant->publishTrack(video_track, video_options); + + livekit::TrackPublishOptions audio_options; + audio_options.audio_encoding = livekit::AudioEncodingOptions{ + .max_bitrate = 128'000, + }; + audio_options.dtx = false; + // Companion metadata identifies this as isolated application audio. Use the + // microphone wire source allowed by the existing companion credential, as + // the macOS publisher does, rather than requiring a newer server grant. + audio_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + audio_options.stream = "game-capture"; + participant->publishTrack(audio_track, audio_options); + + std::cout << "{\"protocolVersion\":1,\"kind\":\"started\",\"width\":" << width + << ",\"height\":" << height + << ",\"frameRate\":" << frames_per_second << "}\n" + << std::flush; + auto stats_reporter = + std::make_unique(video_track, rtc_metrics); + + DWORD process_id = 0; + GetWindowThreadProcessId(window, &process_id); + std::stop_source audio_stop; + auto audio_future = + std::async(std::launch::async, [process_id, audio_source, + stop_token = audio_stop.get_token()] { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + return capture_process_audio( + process_id, std::chrono::hours(24), stop_token, {}, + [audio_source](const AudioFrameData &frame) { + auto livekit_frame = livekit::AudioFrame::create( + static_cast(frame.sample_rate), + static_cast(frame.channels), + static_cast(frame.frames)); + if (!frame.silent) { + auto &output = livekit_frame.data(); + for (std::size_t index = 0; index < output.size(); ++index) { + output[index] = float_to_pcm16(frame.samples[index]); + } + } + audio_source->captureFrame(livekit_frame); + }); + }); + + try { + auto capture_window = window; + std::uint64_t captured_frames = 0; + auto display_fallback_retry_at = std::chrono::steady_clock::time_point{}; + auto capture_backend = CaptureBackend::WgcWindow; + while (!stop_token.stop_requested()) { + const auto now = std::chrono::steady_clock::now(); + const bool monitor_covering = + is_foreground_monitor_covering_window(capture_window); + if (!monitor_covering) { + capture_backend = CaptureBackend::WgcWindow; + } else if (capture_backend == CaptureBackend::WgcWindow) { + capture_backend = CaptureBackend::WgcMonitor; + } else if (capture_backend == CaptureBackend::DxgiDisplay && + now < display_fallback_retry_at) { + capture_backend = CaptureBackend::WgcWindow; + } + video_pump.set_capture_backend(capture_backend); + const auto submit_frame = [&video_pump](VideoFrameData frame) { + video_pump.submit(std::move(frame)); + }; + VideoCaptureMetrics video_metrics; + switch (capture_backend) { + case CaptureBackend::WgcWindow: + video_metrics = capture_window_video( + capture_window, std::chrono::hours(24), frames_per_second, false, + {}, submit_frame, stop_token, kFrameStallTimeout, true); + break; + case CaptureBackend::WgcMonitor: + video_metrics = capture_monitor_covering_window_wgc_video( + capture_window, std::chrono::hours(24), frames_per_second, + submit_frame, stop_token, kFrameStallTimeout); + break; + case CaptureBackend::DxgiDisplay: + video_metrics = capture_monitor_covering_window_dxgi_video( + capture_window, std::chrono::hours(24), frames_per_second, + submit_frame, stop_token); + break; + } + captured_frames += video_metrics.frames; + if (video_metrics.source_closed || video_metrics.frame_stalled || + video_metrics.presentation_changed || video_metrics.stop_requested) { + std::cerr << "[Chatto Desktop capture] Window capture returned: " + << "backend=" << static_cast(capture_backend) + << " frames=" << video_metrics.frames + << " sourceClosed=" << video_metrics.source_closed + << " frameStalled=" << video_metrics.frame_stalled + << " presentationChanged=" + << video_metrics.presentation_changed + << " stopRequested=" << video_metrics.stop_requested + << " windowValid=" << IsWindow(capture_window) << "\n"; + } + if (!video_metrics.error.empty()) { + if (capture_backend != CaptureBackend::WgcWindow) { + std::cerr << "[Chatto Desktop capture] Display fallback failed: " + << winrt::to_string(winrt::hstring(video_metrics.error)) + << " (0x" << std::hex + << static_cast(video_metrics.error_code) + << std::dec << ")\n"; + video_pump.set_capture_backend(CaptureBackend::WgcWindow); + display_fallback_retry_at = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + capture_backend = CaptureBackend::WgcWindow; + continue; + } + throw std::runtime_error( + winrt::to_string(winrt::hstring(video_metrics.error))); + } + if (video_metrics.stop_requested || stop_token.stop_requested()) { + break; + } + if (video_metrics.presentation_changed) { + std::cerr << "Windows video publisher returned to window capture\n"; + capture_backend = CaptureBackend::WgcWindow; + continue; + } + if (!video_metrics.source_closed && !video_metrics.frame_stalled) { + break; + } + + const bool currently_monitor_covering = + is_foreground_monitor_covering_window(capture_window); + if (video_metrics.frame_stalled && currently_monitor_covering) { + if (capture_backend == CaptureBackend::WgcWindow) { + capture_backend = CaptureBackend::WgcMonitor; + continue; + } + if (capture_backend == CaptureBackend::WgcMonitor) { + capture_backend = CaptureBackend::DxgiDisplay; + continue; + } + } + + const auto replacement = wait_for_replacement_window( + capture_window, expected_application_identifier, process_id, + video_metrics.frame_stalled || video_metrics.source_closed, + stop_token); + if (!replacement) { + std::cerr << "[Chatto Desktop capture] Window capture ended because " + "no matching replacement window remained\n"; + break; + } + capture_window = *replacement; + if (capture_backend == CaptureBackend::DxgiDisplay) { + std::cerr << "Windows video publisher restarted DXGI display capture\n"; + } else { + std::cerr << "Windows video publisher reattached after " + << (video_metrics.source_closed ? "source closure" + : "frame stall") + << "\n"; + } + } + const auto pump_metrics = video_pump.finish(); + std::cerr << "[Chatto Desktop capture] Windows video publisher ended: " + << "stopRequested=" << stop_token.stop_requested() + << " captured=" << captured_frames + << " submitted=" << pump_metrics.submitted + << " published=" << pump_metrics.published + << " dropped=" << pump_metrics.dropped << "\n"; + } catch (...) { + stats_reporter.reset(); + audio_stop.request_stop(); + if (audio_future.valid()) { + try { + static_cast(audio_future.get()); + } catch (...) { + } + } + room.disconnect(); + throw; + } + audio_stop.request_stop(); + if (audio_future.valid()) { + static_cast(audio_future.get()); + } + stats_reporter.reset(); + room.disconnect(); + return 0; +} + +int publish_display(HMONITOR monitor, const std::uint32_t frames_per_second, + const std::uint32_t maximum_width, + const std::uint32_t maximum_height, + const PublisherCredential &credential, + EncodedPreviewCallback preview_callback, + const std::stop_token stop_token) { + if (!is_display_capture_candidate(monitor)) { + throw std::invalid_argument("The selected monitor no longer exists"); + } + if (credential.livekit_url.empty() || credential.token.empty() || + credential.e2ee_key.empty()) { + throw std::invalid_argument("The publisher credential is incomplete"); + } + + MONITORINFO monitor_information{}; + monitor_information.cbSize = sizeof(monitor_information); + winrt::check_bool(GetMonitorInfoW(monitor, &monitor_information)); + const auto source_width = static_cast( + monitor_information.rcMonitor.right - monitor_information.rcMonitor.left); + const auto source_height = static_cast( + monitor_information.rcMonitor.bottom - monitor_information.rcMonitor.top); + const auto [width, height] = + bounded_size(source_width, source_height, maximum_width, maximum_height); + + LiveKitRuntime livekit_runtime; + auto video_source = + std::make_shared(width, height); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack( + "display-capture-video", video_source); + auto rtc_metrics = std::make_shared(); + LiveKitVideoPump video_pump(video_source, rtc_metrics, width, height, + frames_per_second, kInitialVideoBitrateBps, + std::move(preview_callback)); + + livekit::RoomOptions room_options; + room_options.auto_subscribe = false; + room_options.dynacast = false; + livekit::E2EEOptions encryption; + encryption.key_provider_options.shared_key = std::vector( + credential.e2ee_key.begin(), credential.e2ee_key.end()); + room_options.encryption = std::move(encryption); + + livekit::Room room; + if (!room.connect(credential.livekit_url, credential.token, room_options)) { + throw std::runtime_error( + "The native publisher could not connect to LiveKit"); + } + if (const auto manager = room.e2eeManager().lock()) { + manager->setEnabled(true); + } else { + throw std::runtime_error("The native publisher could not enable E2EE"); + } + const auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("The native publisher has no local participant"); + } + + livekit::TrackPublishOptions video_options; + video_options.video_encoding = livekit::VideoEncodingOptions{ + .max_bitrate = kInitialVideoBitrateBps, + .max_framerate = static_cast(frames_per_second), + }; + video_options.video_codec = livekit::VideoCodec::H264; + video_options.video_encoder = livekit::VideoEncoderBackend::PreEncoded; + video_options.simulcast = false; + video_options.source = livekit::TrackSource::SOURCE_SCREENSHARE; + video_options.stream = "game-capture"; + video_options.degradation_preference = + livekit::DegradationPreference::MaintainFramerate; + participant->publishTrack(video_track, video_options); + + std::cout << "{\"protocolVersion\":1,\"kind\":\"started\",\"width\":" << width + << ",\"height\":" << height + << ",\"frameRate\":" << frames_per_second << "}\n" + << std::flush; + auto stats_reporter = + std::make_unique(video_track, rtc_metrics); + video_pump.set_capture_backend(CaptureBackend::WgcMonitor); + + try { + std::uint64_t captured_frames = 0; + while (!stop_token.stop_requested() && + is_display_capture_candidate(monitor)) { + const auto video_metrics = capture_monitor_wgc_video( + monitor, std::chrono::hours(24), frames_per_second, + [&video_pump](VideoFrameData frame) { + video_pump.submit(std::move(frame)); + }, + stop_token, kFrameStallTimeout); + captured_frames += video_metrics.frames; + if (!video_metrics.error.empty()) { + throw std::runtime_error( + winrt::to_string(winrt::hstring(video_metrics.error))); + } + if (video_metrics.stop_requested || stop_token.stop_requested()) { + break; + } + if (!video_metrics.frame_stalled) { + break; + } + std::cerr << "Windows display publisher restarted monitor capture\n"; + } + const auto pump_metrics = video_pump.finish(); + std::cerr << "[Chatto Desktop capture] Windows display publisher ended: " + << "stopRequested=" << stop_token.stop_requested() + << " captured=" << captured_frames + << " submitted=" << pump_metrics.submitted + << " published=" << pump_metrics.published + << " dropped=" << pump_metrics.dropped << "\n"; + } catch (...) { + stats_reporter.reset(); + room.disconnect(); + throw; + } + stats_reporter.reset(); + room.disconnect(); + return 0; +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/livekit_publisher.h b/apps/desktop/native/windows-capture-probe/src/livekit_publisher.h new file mode 100644 index 0000000000..83dd28d171 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/livekit_publisher.h @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace chatto::capture { + +struct PublisherCredential { + std::string livekit_url; + std::string token; + std::string e2ee_key; +}; + +using EncodedPreviewCallback = + std::function, std::int64_t, bool)>; + +[[nodiscard]] int +publish_window(HWND window, const std::wstring &expected_application_identifier, + std::uint32_t frames_per_second, std::uint32_t maximum_width, + std::uint32_t maximum_height, + const PublisherCredential &credential, + EncodedPreviewCallback preview_callback = {}, + std::stop_token stop_token = {}); + +/** Publish an explicitly selected monitor as a video-only screen share. */ +[[nodiscard]] int +publish_display(HMONITOR monitor, std::uint32_t frames_per_second, + std::uint32_t maximum_width, std::uint32_t maximum_height, + const PublisherCredential &credential, + EncodedPreviewCallback preview_callback = {}, + std::stop_token stop_token = {}); + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/main.cpp b/apps/desktop/native/windows-capture-probe/src/main.cpp new file mode 100644 index 0000000000..80c3106600 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/main.cpp @@ -0,0 +1,584 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "audio_capture.h" +#include "live_status.h" +#include "livekit_publisher.h" +#include "video_capture.h" +#include "window_sources.h" + +namespace { + +class EncodedPreviewPipe final { +public: + EncodedPreviewPipe() { + const auto native_handle = _get_osfhandle(3); + if (native_handle != -1) { + handle_ = reinterpret_cast(native_handle); + } + } + + void write(std::span data, + const std::int64_t timestamp_us, const bool key_frame) { + if (handle_ == INVALID_HANDLE_VALUE || data.empty() || + data.size() > 16U * 1024U * 1024U) { + return; + } + std::array header{'C', 'T', 'P', 'V'}; + const auto size = static_cast(data.size()); + for (std::size_t index = 0; index < 4; ++index) { + header[4 + index] = + static_cast((size >> (index * 8U)) & 0xffU); + } + const auto timestamp = static_cast(timestamp_us); + for (std::size_t index = 0; index < 8; ++index) { + header[8 + index] = + static_cast((timestamp >> (index * 8U)) & 0xffU); + } + if (key_frame) header[7] |= 0x80U; + if (!write_all(header) || !write_all(data)) { + handle_ = INVALID_HANDLE_VALUE; + } + } + +private: + bool write_all(std::span bytes) const { + while (!bytes.empty()) { + DWORD written = 0; + if (!WriteFile(handle_, bytes.data(), static_cast(bytes.size()), + &written, nullptr) || written == 0) { + return false; + } + bytes = bytes.subspan(written); + } + return true; + } + + HANDLE handle_ = INVALID_HANDLE_VALUE; +}; + +void print_usage() { + std::wcout + << L"Usage:\n" + << L" chatto-windows-capture-probe support\n" + << L" chatto-windows-capture-probe list [--include-titles]\n" + << L" chatto-windows-capture-probe list-json [--exclude-process ]\n" + << L" chatto-windows-capture-probe capture --window " + L"[--duration ] [--fps ] [--video-only] [--preview]\n" + << L" chatto-windows-capture-probe publish (--window " + L"--expected-window-bundle | --display ) " + L"[--fps ]\n" + << L" chatto-windows-capture-probe audio --process " + L"[--duration ]\n\n" + << L"Window titles may contain sensitive information and are omitted by " + L"default.\n"; +} + +[[nodiscard]] std::uint64_t parse_unsigned(const std::wstring_view value, + const int base, const char *name) { + wchar_t *end = nullptr; + errno = 0; + const auto parsed = std::wcstoull(value.data(), &end, base); + if (errno != 0 || end != value.data() + value.size()) { + throw std::invalid_argument(std::string("Invalid ") + name); + } + return parsed; +} + +int print_support() { + const bool supported = + winrt::Windows::Graphics::Capture::GraphicsCaptureSession::IsSupported(); + std::wcout << L"windows_graphics_capture=" + << (supported ? L"supported" : L"unsupported") << L"\n"; + return supported ? 0 : 2; +} + +int list_windows(const bool include_titles) { + const auto sources = chatto::capture::enumerate_window_sources(); + std::wcout << L"windows=" << sources.size() << L"\n"; + for (const auto &source : sources) { + std::wcout << L"hwnd=0x" << std::hex + << reinterpret_cast(source.handle) << std::dec + << L" pid=" << source.process_id << L" application=" + << std::quoted(source.application_name) << L" size=" + << source.width << L"x" << source.height; + if (include_titles) { + std::wcout << L" title=" << std::quoted(source.title); + } + std::wcout << L"\n"; + } + return 0; +} + +int list_windows_json(const int argument_count, wchar_t *arguments[]) { + DWORD excluded_process_id = 0; + for (int index = 2; index < argument_count; ++index) { + if (std::wstring_view(arguments[index]) == L"--exclude-process" && + index + 1 < argument_count) { + const auto value = + parse_unsigned(arguments[++index], 10, "process identifier"); + if (value > std::numeric_limits::max()) { + throw std::invalid_argument("Process identifier is out of range"); + } + excluded_process_id = static_cast(value); + continue; + } + throw std::invalid_argument("Unknown or incomplete list-json argument"); + } + + using namespace winrt::Windows::Data::Json; + JsonArray sources; + for (const auto &source : chatto::capture::enumerate_display_sources()) { + JsonObject value; + value.SetNamedValue(L"kind", JsonValue::CreateStringValue(L"display")); + value.SetNamedValue(L"nativeID", + JsonValue::CreateNumberValue(static_cast( + reinterpret_cast(source.handle)))); + value.SetNamedValue(L"displayIndex", + JsonValue::CreateNumberValue(source.display_index)); + value.SetNamedValue(L"isMainDisplay", + JsonValue::CreateBooleanValue(source.is_main_display)); + value.SetNamedValue(L"width", JsonValue::CreateNumberValue(source.width)); + value.SetNamedValue(L"height", JsonValue::CreateNumberValue(source.height)); + value.SetNamedValue(L"previewByteLength", JsonValue::CreateNumberValue(0)); + sources.Append(value); + } + for (const auto &source : chatto::capture::enumerate_window_sources()) { + if (source.process_id == excluded_process_id) { + continue; + } + JsonObject value; + value.SetNamedValue(L"kind", JsonValue::CreateStringValue(L"window")); + value.SetNamedValue(L"nativeID", + JsonValue::CreateNumberValue(static_cast( + reinterpret_cast(source.handle)))); + value.SetNamedValue(L"applicationName", + JsonValue::CreateStringValue(source.application_name)); + value.SetNamedValue( + L"bundleIdentifier", + JsonValue::CreateStringValue(source.application_identifier)); + value.SetNamedValue(L"title", JsonValue::CreateStringValue(source.title)); + value.SetNamedValue(L"width", JsonValue::CreateNumberValue(source.width)); + value.SetNamedValue(L"height", JsonValue::CreateNumberValue(source.height)); + value.SetNamedValue(L"previewByteLength", JsonValue::CreateNumberValue(0)); + sources.Append(value); + } + JsonObject response; + response.SetNamedValue(L"protocolVersion", JsonValue::CreateNumberValue(1)); + response.SetNamedValue(L"sources", sources); + std::cout << winrt::to_string(response.Stringify()); + return 0; +} + +void print_audio_metrics(const chatto::capture::AudioCaptureMetrics &metrics) { + std::wcout << std::fixed << std::setprecision(4) << L"audio_packets=" + << metrics.packets << L"\n" + << L"audio_frames=" << metrics.frames << L"\n" + << L"audio_format=" << metrics.sample_rate << L"Hz/" + << metrics.channels << L"ch/float32\n" + << L"audio_timestamp_span_seconds=" + << metrics.timestamp_span_seconds << L"\n" + << L"audio_peak=" << metrics.peak_level << L"\n" + << L"audio_silent_packets=" << metrics.silent_packets << L"\n" + << L"audio_discontinuities=" << metrics.discontinuities << L"\n" + << L"audio_timestamp_errors=" << metrics.timestamp_errors << L"\n"; +} + +int capture_window(const int argument_count, wchar_t *arguments[]) { + HWND window = nullptr; + std::uint32_t duration_seconds = 15; + std::uint32_t frames_per_second = 60; + bool video_only = false; + bool show_preview = false; + + for (int index = 2; index < argument_count; ++index) { + const std::wstring_view argument(arguments[index]); + if (argument == L"--window" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 0, "window handle"); + window = reinterpret_cast(static_cast(value)); + continue; + } + if (argument == L"--duration" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 10, "duration"); + if (value == 0 || value > 3'600) { + throw std::invalid_argument( + "Duration must be between 1 and 3600 seconds"); + } + duration_seconds = static_cast(value); + continue; + } + if (argument == L"--fps" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 10, "frame rate"); + if (value == 0 || value > 240) { + throw std::invalid_argument("Frame rate must be between 1 and 240"); + } + frames_per_second = static_cast(value); + continue; + } + if (argument == L"--video-only") { + video_only = true; + continue; + } + if (argument == L"--preview") { + show_preview = true; + continue; + } + throw std::invalid_argument("Unknown or incomplete capture argument"); + } + + if (window == nullptr) { + throw std::invalid_argument("Capture requires --window "); + } + + DWORD process_id = 0; + GetWindowThreadProcessId(window, &process_id); + std::stop_source audio_stop_source; + auto live_status = std::make_shared(); + std::future audio_future; + if (!video_only && process_id != 0) { + audio_future = std::async( + std::launch::async, [process_id, duration_seconds, live_status, + stop_token = audio_stop_source.get_token()] { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + return chatto::capture::capture_process_audio( + process_id, std::chrono::seconds(duration_seconds), stop_token, + live_status); + }); + } + + chatto::capture::VideoCaptureMetrics metrics; + try { + metrics = chatto::capture::capture_window_video( + window, std::chrono::seconds(duration_seconds), frames_per_second, + show_preview, live_status); + } catch (...) { + audio_stop_source.request_stop(); + if (audio_future.valid()) { + try { + static_cast(audio_future.get()); + } catch (...) { + // Preserve the video failure that initiated cancellation. + } + } + throw; + } + audio_stop_source.request_stop(); + std::wcout << std::fixed << std::setprecision(2) << L"video_frames=" + << metrics.frames << L"\n" + << L"video_size=" << metrics.width << L"x" << metrics.height + << L"\n" + << L"video_timestamp_span_seconds=" + << metrics.timestamp_span_seconds << L"\n" + << L"video_observed_fps=" << metrics.observed_frames_per_second + << L"\n" + << L"video_longest_interval_ms=" + << metrics.longest_frame_interval_ms << L"\n" + << L"video_inferred_gaps=" << metrics.inferred_gaps << L"\n" + << L"video_resizes=" << metrics.resizes << L"\n" + << L"video_sampled_frames=" << metrics.sampled_frames << L"\n" + << L"video_changed_samples=" << metrics.changed_samples << L"\n" + << L"video_black_samples=" << metrics.black_samples << L"\n" + << L"video_sampled_luminance_min_mean_max=" + << static_cast(metrics.sampled_luminance_min) + << L"/" << metrics.sampled_luminance_mean << L"/" + << static_cast(metrics.sampled_luminance_max) + << L"\n" + << L"probe_wall_duration_seconds=" << metrics.wall_duration_seconds + << L"\n" + << L"probe_cpu_seconds=" << metrics.process_cpu_seconds << L"\n" + << L"probe_cpu_single_core_percent=" + << metrics.process_cpu_single_core_percent << L"\n" + << L"probe_peak_working_set_bytes=" + << metrics.peak_working_set_bytes << L"\n" + << L"source_closed=" + << (metrics.source_closed ? L"true" : L"false") << L"\n"; + if (!metrics.error.empty()) { + std::wcout << L"video_error=" << std::quoted(metrics.error) << L"\n"; + return 2; + } + + if (audio_future.valid()) { + const auto audio_metrics = audio_future.get(); + print_audio_metrics(audio_metrics); + if (metrics.first_timestamp_100ns != 0 && + audio_metrics.first_timestamp_100ns != 0) { + const auto start_delta = + static_cast(audio_metrics.first_timestamp_100ns) - + metrics.first_timestamp_100ns; + std::wcout << std::fixed << std::setprecision(2) << L"av_start_delta_ms=" + << static_cast(start_delta) / 10'000.0 << L"\n"; + } + } + return metrics.frames > 0 ? 0 : 2; +} + +int capture_audio(const int argument_count, wchar_t *arguments[]) { + DWORD process_id = 0; + std::uint32_t duration_seconds = 15; + for (int index = 2; index < argument_count; ++index) { + const std::wstring_view argument(arguments[index]); + if (argument == L"--process" && index + 1 < argument_count) { + const auto value = + parse_unsigned(arguments[++index], 10, "process identifier"); + if (value == 0 || value > std::numeric_limits::max()) { + throw std::invalid_argument("Process identifier is out of range"); + } + process_id = static_cast(value); + continue; + } + if (argument == L"--duration" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 10, "duration"); + if (value == 0 || value > 3'600) { + throw std::invalid_argument( + "Duration must be between 1 and 3600 seconds"); + } + duration_seconds = static_cast(value); + continue; + } + throw std::invalid_argument("Unknown or incomplete audio argument"); + } + if (process_id == 0) { + throw std::invalid_argument("Audio capture requires --process "); + } + + const auto metrics = chatto::capture::capture_process_audio( + process_id, std::chrono::seconds(duration_seconds)); + print_audio_metrics(metrics); + return metrics.packets > 0 ? 0 : 2; +} + +chatto::capture::PublisherCredential read_publisher_credential() { + std::string input; + std::getline(std::cin, input); + if (input.empty() || input.size() > 96 * 1024) { + throw std::invalid_argument("The publisher credential is invalid"); + } + const auto value = + winrt::Windows::Data::Json::JsonObject::Parse(winrt::to_hstring(input)); + if (value.GetNamedNumber(L"protocolVersion", 0) != 1) { + throw std::invalid_argument( + "The publisher credential protocol is unsupported"); + } + return { + .livekit_url = winrt::to_string(value.GetNamedString(L"livekitURL")), + .token = winrt::to_string(value.GetNamedString(L"token")), + .e2ee_key = winrt::to_string(value.GetNamedString(L"e2eeKey")), + }; +} + +void monitor_publisher_control(std::stop_source stop_source, + const std::atomic &finished) { + const HANDLE input = GetStdHandle(STD_INPUT_HANDLE); + if (input == nullptr || input == INVALID_HANDLE_VALUE) { + return; + } + + std::string pending; + while (!finished.load(std::memory_order_relaxed) && + !stop_source.stop_requested()) { + DWORD available = 0; + if (!PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr)) { + if (GetLastError() == ERROR_BROKEN_PIPE) { + stop_source.request_stop(); + } + return; + } + if (available == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + continue; + } + + std::array buffer{}; + DWORD read = 0; + constexpr DWORD buffer_size = static_cast(buffer.size()); + if (!ReadFile(input, buffer.data(), std::min(available, buffer_size), &read, + nullptr)) { + if (GetLastError() == ERROR_BROKEN_PIPE) { + stop_source.request_stop(); + } + return; + } + pending.append(buffer.data(), read); + const auto line_end = pending.find('\n'); + if (line_end == std::string::npos) { + if (pending.size() > 256) { + stop_source.request_stop(); + return; + } + continue; + } + if (pending.substr(0, line_end) == "stop" || + pending.substr(0, line_end) == "stop\r") { + stop_source.request_stop(); + } + return; + } +} + +int publish(const int argument_count, wchar_t *arguments[]) { + HWND window = nullptr; + HMONITOR monitor = nullptr; + std::wstring expected_application_identifier; + std::uint32_t frames_per_second = 60; + std::uint32_t maximum_width = 1920; + std::uint32_t maximum_height = 1080; + for (int index = 2; index < argument_count; ++index) { + const std::wstring_view argument(arguments[index]); + if (argument == L"--window" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 0, "window handle"); + window = reinterpret_cast(static_cast(value)); + continue; + } + if (argument == L"--display" && index + 1 < argument_count) { + const auto value = + parse_unsigned(arguments[++index], 0, "monitor handle"); + monitor = reinterpret_cast(static_cast(value)); + continue; + } + if (argument == L"--expected-window-bundle" && index + 1 < argument_count) { + expected_application_identifier = arguments[++index]; + continue; + } + if (argument == L"--fps" && index + 1 < argument_count) { + const auto value = parse_unsigned(arguments[++index], 10, "frame rate"); + if (value == 0 || value > 60) { + throw std::invalid_argument("Frame rate must be between 1 and 60"); + } + frames_per_second = static_cast(value); + continue; + } + if (argument == L"--max-width" && index + 1 < argument_count) { + const auto value = + parse_unsigned(arguments[++index], 10, "maximum width"); + if (value == 0 || value > 16'384) { + throw std::invalid_argument("Maximum width is out of range"); + } + maximum_width = static_cast(value); + continue; + } + if (argument == L"--max-height" && index + 1 < argument_count) { + const auto value = + parse_unsigned(arguments[++index], 10, "maximum height"); + if (value == 0 || value > 16'384) { + throw std::invalid_argument("Maximum height is out of range"); + } + maximum_height = static_cast(value); + continue; + } + throw std::invalid_argument("Unknown or incomplete publish argument"); + } + if ((window == nullptr) == (monitor == nullptr) || + (window != nullptr && expected_application_identifier.empty()) || + (monitor != nullptr && !expected_application_identifier.empty())) { + throw std::invalid_argument( + "Publishing requires either a display or a window and its expected " + "application identity"); + } + const auto credential = read_publisher_credential(); + auto preview_pipe = std::make_shared(); + chatto::capture::EncodedPreviewCallback preview_callback = + [preview_pipe](const std::span data, + const std::int64_t timestamp_us, const bool key_frame) { + preview_pipe->write(data, timestamp_us, key_frame); + }; + std::stop_source stop_source; + std::atomic control_finished = false; + std::thread control_thread(monitor_publisher_control, stop_source, + std::cref(control_finished)); + try { + const int result = window != nullptr + ? chatto::capture::publish_window( + window, expected_application_identifier, + frames_per_second, maximum_width, + maximum_height, credential, preview_callback, + stop_source.get_token()) + : chatto::capture::publish_display( + monitor, frames_per_second, maximum_width, + maximum_height, credential, preview_callback, + stop_source.get_token()); + control_finished.store(true, std::memory_order_relaxed); + control_thread.join(); + return result; + } catch (...) { + control_finished.store(true, std::memory_order_relaxed); + control_thread.join(); + throw; + } +} + +} // namespace + +int wmain(const int argument_count, wchar_t *arguments[]) { + try { + winrt::init_apartment(winrt::apartment_type::multi_threaded); + + if (argument_count == 2 && std::wstring_view(arguments[1]) == L"support") { + return print_support(); + } + + if (argument_count >= 2 && std::wstring_view(arguments[1]) == L"list") { + bool include_titles = false; + for (int index = 2; index < argument_count; ++index) { + if (std::wstring_view(arguments[index]) == L"--include-titles") { + include_titles = true; + continue; + } + std::wcerr << L"Unknown argument: " << arguments[index] << L"\n"; + print_usage(); + return 1; + } + return list_windows(include_titles); + } + + if (argument_count >= 2 && + std::wstring_view(arguments[1]) == L"list-json") { + return list_windows_json(argument_count, arguments); + } + + if (argument_count >= 2 && std::wstring_view(arguments[1]) == L"capture") { + return capture_window(argument_count, arguments); + } + + if (argument_count >= 2 && std::wstring_view(arguments[1]) == L"audio") { + return capture_audio(argument_count, arguments); + } + + if (argument_count >= 2 && std::wstring_view(arguments[1]) == L"publish") { + return publish(argument_count, arguments); + } + + print_usage(); + return 1; + } catch (const winrt::hresult_error &error) { + std::wcerr << L"Windows capture probe failed: " << error.message().c_str() + << L" (0x" << std::hex + << static_cast(error.code()) << L")\n"; + return 1; + } catch (const std::exception &error) { + std::cerr << "Windows capture probe failed: " << error.what() << "\n"; + return 1; + } +} diff --git a/apps/desktop/native/windows-capture-probe/src/nvenc_encoder.cpp b/apps/desktop/native/windows-capture-probe/src/nvenc_encoder.cpp new file mode 100644 index 0000000000..d036faf813 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/nvenc_encoder.cpp @@ -0,0 +1,690 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "h264_encoder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +using Microsoft::WRL::ComPtr; + +constexpr std::uint32_t kNvidiaVendorId = 0x10de; +constexpr std::size_t kSurfaceCount = 4; + +class NvencError final : public std::runtime_error { +public: + explicit NvencError(const std::string &message) + : std::runtime_error(message) {} +}; + +class UniqueModule final { +public: + explicit UniqueModule(const wchar_t *name) : module_(LoadLibraryW(name)) { + if (module_ == nullptr) { + throw NvencError("The NVIDIA NVENC driver library is unavailable"); + } + } + + ~UniqueModule() { + if (module_ != nullptr) { + FreeLibrary(module_); + } + } + + UniqueModule(const UniqueModule &) = delete; + UniqueModule &operator=(const UniqueModule &) = delete; + + [[nodiscard]] FARPROC function(const char *name) const { + const auto address = GetProcAddress(module_, name); + if (address == nullptr) { + throw NvencError(std::string("The NVIDIA driver does not export ") + + name); + } + return address; + } + +private: + HMODULE module_ = nullptr; +}; + +[[nodiscard]] std::string +nvenc_status_message(const NVENCSTATUS status, + const NV_ENCODE_API_FUNCTION_LIST &functions, + void *encoder) { + std::string message = "NVENC error " + std::to_string(status); + if (encoder != nullptr && functions.nvEncGetLastErrorString != nullptr) { + if (const char *detail = functions.nvEncGetLastErrorString(encoder); + detail != nullptr && detail[0] != '\0') { + message += ": "; + message += detail; + } + } + return message; +} + +void check_nvenc(const NVENCSTATUS status, const char *operation, + const NV_ENCODE_API_FUNCTION_LIST &functions, + void *encoder = nullptr) { + if (status != NV_ENC_SUCCESS) { + throw NvencError(std::string(operation) + " failed (" + + nvenc_status_message(status, functions, encoder) + ")"); + } +} + +[[nodiscard]] ComPtr find_nvidia_adapter() { + ComPtr factory; + if (FAILED(CreateDXGIFactory1(IID_PPV_ARGS(&factory)))) { + throw NvencError("Could not create a DXGI factory for NVENC"); + } + for (UINT index = 0;; ++index) { + ComPtr adapter; + if (factory->EnumAdapters1(index, &adapter) == DXGI_ERROR_NOT_FOUND) { + break; + } + DXGI_ADAPTER_DESC1 description{}; + if (SUCCEEDED(adapter->GetDesc1(&description)) && + description.VendorId == kNvidiaVendorId && + (description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) == 0) { + return adapter; + } + } + throw NvencError("No NVIDIA graphics adapter is available"); +} + +class DirectNvencH264Encoder final : public H264Encoder { +public: + DirectNvencH264Encoder(const std::uint32_t width, const std::uint32_t height, + const std::uint32_t frames_per_second, + const std::uint32_t target_bitrate_bps) + : width_(width), height_(height), frames_per_second_(frames_per_second), + target_bitrate_bps_(target_bitrate_bps), module_(L"nvEncodeAPI64.dll") { + if (width == 0 || height == 0 || (width % 2) != 0 || (height % 2) != 0 || + frames_per_second == 0 || target_bitrate_bps == 0) { + throw std::invalid_argument("The NVENC H.264 settings are invalid"); + } + try { + initialize_api(); + initialize_device(); + open_session(); + initialize_encoder(); + create_surfaces(); + } catch (...) { + release(); + throw; + } + } + + ~DirectNvencH264Encoder() override { release(); } + + DirectNvencH264Encoder(const DirectNvencH264Encoder &) = delete; + DirectNvencH264Encoder &operator=(const DirectNvencH264Encoder &) = delete; + + [[nodiscard]] std::vector + encode(const std::span bgra, + const std::int64_t timestamp_us, const bool force_key_frame) override { + if (bgra.size() != static_cast(width_) * height_ * 4) { + throw std::invalid_argument("The NVENC BGRA test frame is invalid"); + } + D3D11_TEXTURE2D_DESC description{}; + description.Width = width_; + description.Height = height_; + description.MipLevels = 1; + description.ArraySize = 1; + description.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + description.SampleDesc.Count = 1; + description.Usage = D3D11_USAGE_DEFAULT; + description.BindFlags = 0; + description.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX; + D3D11_SUBRESOURCE_DATA initial{}; + initial.pSysMem = bgra.data(); + initial.SysMemPitch = width_ * 4; + ComPtr texture; + if (FAILED(device_->CreateTexture2D(&description, &initial, &texture))) { + throw NvencError("Could not upload the NVENC test frame"); + } + ComPtr mutex; + if (FAILED(texture.As(&mutex)) || FAILED(mutex->AcquireSync(0, INFINITE)) || + FAILED(mutex->ReleaseSync(1))) { + throw NvencError("Could not publish the NVENC test frame"); + } + return encode_gpu(*texture.Get(), width_, height_, timestamp_us, + force_key_frame); + } + + [[nodiscard]] std::vector + encode_gpu(ID3D11Texture2D &bgra_texture, const std::uint32_t source_width, + const std::uint32_t source_height, const std::int64_t timestamp_us, + const bool force_key_frame) override { + if (finished_) { + throw NvencError("NVENC cannot encode after the stream has finished"); + } + auto &surface = surfaces_[surface_index_]; + surface_index_ = (surface_index_ + 1) % surfaces_.size(); + + const auto conversion_start = std::chrono::steady_clock::now(); + convert_bgra_to_nv12(bgra_texture, source_width, source_height, surface); + const auto conversion_end = std::chrono::steady_clock::now(); + last_gpu_conversion_submit_ms_ = std::chrono::duration( + conversion_end - conversion_start) + .count(); + + const auto submit_start = std::chrono::steady_clock::now(); + NV_ENC_MAP_INPUT_RESOURCE map{}; + map.version = NV_ENC_MAP_INPUT_RESOURCE_VER; + map.registeredResource = surface.registered; + check_nvenc(functions_.nvEncMapInputResource(encoder_, &map), + "Mapping an NVENC input texture", functions_, encoder_); + + NV_ENC_PIC_PARAMS picture{}; + picture.version = NV_ENC_PIC_PARAMS_VER; + picture.inputWidth = width_; + picture.inputHeight = height_; + picture.inputPitch = width_; + picture.inputBuffer = map.mappedResource; + picture.bufferFmt = map.mappedBufferFmt; + picture.outputBitstream = surface.bitstream; + picture.inputTimeStamp = static_cast(timestamp_us); + picture.inputDuration = 1'000'000U / frames_per_second_; + picture.pictureStruct = NV_ENC_PIC_STRUCT_FRAME; + if (force_key_frame) { + picture.encodePicFlags = + NV_ENC_PIC_FLAG_FORCEIDR | NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; + } + + bool bitstream_locked = false; + try { + const NVENCSTATUS result = + functions_.nvEncEncodePicture(encoder_, &picture); + if (result == NV_ENC_ERR_NEED_MORE_INPUT) { + static_cast( + functions_.nvEncUnmapInputResource(encoder_, map.mappedResource)); + return {}; + } + check_nvenc(result, "Encoding an NVENC frame", functions_, encoder_); + const auto submit_end = std::chrono::steady_clock::now(); + last_encoder_submit_ms_ = + std::chrono::duration(submit_end - submit_start) + .count(); + + const auto wait_start = std::chrono::steady_clock::now(); + NV_ENC_LOCK_BITSTREAM lock{}; + lock.version = NV_ENC_LOCK_BITSTREAM_VER; + lock.outputBitstream = surface.bitstream; + lock.doNotWait = 0; + check_nvenc(functions_.nvEncLockBitstream(encoder_, &lock), + "Locking an NVENC access unit", functions_, encoder_); + bitstream_locked = true; + last_bitstream_wait_ms_ = + std::chrono::duration( + std::chrono::steady_clock::now() - wait_start) + .count(); + + EncodedH264AccessUnit access_unit; + const auto *bytes = + static_cast(lock.bitstreamBufferPtr); + access_unit.data.assign(bytes, bytes + lock.bitstreamSizeInBytes); + access_unit.timestamp_us = + static_cast(lock.outputTimeStamp); + access_unit.key_frame = lock.pictureType == NV_ENC_PIC_TYPE_IDR || + lock.pictureType == NV_ENC_PIC_TYPE_I || + h264_access_unit_is_key_frame(access_unit.data); + + const NVENCSTATUS unlock_result = + functions_.nvEncUnlockBitstream(encoder_, surface.bitstream); + bitstream_locked = false; + const NVENCSTATUS unmap_result = + functions_.nvEncUnmapInputResource(encoder_, map.mappedResource); + map.mappedResource = nullptr; + check_nvenc(unlock_result, "Unlocking an NVENC access unit", functions_, + encoder_); + check_nvenc(unmap_result, "Unmapping an NVENC input texture", functions_, + encoder_); + if (!h264_access_unit_is_annex_b(access_unit.data)) { + throw NvencError("NVENC returned a non-Annex-B H.264 access unit"); + } + return {std::move(access_unit)}; + } catch (...) { + if (bitstream_locked) { + static_cast( + functions_.nvEncUnlockBitstream(encoder_, surface.bitstream)); + } + if (map.mappedResource != nullptr) { + static_cast( + functions_.nvEncUnmapInputResource(encoder_, map.mappedResource)); + } + throw; + } + } + + void set_target_bitrate(const std::uint32_t target_bitrate_bps) override { + if (target_bitrate_bps == 0 || target_bitrate_bps == target_bitrate_bps_) { + return; + } + config_.rcParams.averageBitRate = target_bitrate_bps; + config_.rcParams.maxBitRate = target_bitrate_bps; + config_.rcParams.vbvBufferSize = + std::max(1U, target_bitrate_bps / frames_per_second_); + config_.rcParams.vbvInitialDelay = config_.rcParams.vbvBufferSize; + initialization_.encodeConfig = &config_; + + NV_ENC_RECONFIGURE_PARAMS reconfigure{}; + reconfigure.version = NV_ENC_RECONFIGURE_PARAMS_VER; + reconfigure.reInitEncodeParams = initialization_; + check_nvenc(functions_.nvEncReconfigureEncoder(encoder_, &reconfigure), + "Reconfiguring the NVENC bitrate", functions_, encoder_); + target_bitrate_bps_ = target_bitrate_bps; + } + + [[nodiscard]] std::uint32_t target_bitrate_bps() const noexcept override { + return target_bitrate_bps_; + } + + [[nodiscard]] std::uint32_t rate_control_mode() const noexcept override { + return static_cast(config_.rcParams.rateControlMode); + } + + [[nodiscard]] double last_gpu_conversion_submit_ms() const noexcept override { + return last_gpu_conversion_submit_ms_; + } + + [[nodiscard]] double last_encoder_submit_ms() const noexcept override { + return last_encoder_submit_ms_; + } + + [[nodiscard]] double last_bitstream_wait_ms() const noexcept override { + return last_bitstream_wait_ms_; + } + + [[nodiscard]] std::vector finish() override { + if (finished_) { + return {}; + } + finished_ = true; + NV_ENC_PIC_PARAMS picture{}; + picture.version = NV_ENC_PIC_PARAMS_VER; + picture.encodePicFlags = NV_ENC_PIC_FLAG_EOS; + check_nvenc(functions_.nvEncEncodePicture(encoder_, &picture), + "Finishing the NVENC stream", functions_, encoder_); + return {}; + } + + [[nodiscard]] const std::string & + implementation_name() const noexcept override { + return implementation_name_; + } + +private: + struct Surface { + ComPtr texture; + ComPtr output_view; + NV_ENC_REGISTERED_PTR registered = nullptr; + NV_ENC_OUTPUT_PTR bitstream = nullptr; + }; + + void initialize_api() { + using CreateInstance = + NVENCSTATUS(NVENCAPI *)(NV_ENCODE_API_FUNCTION_LIST *); + using GetMaxVersion = NVENCSTATUS(NVENCAPI *)(std::uint32_t *); + const auto create_instance = reinterpret_cast( + module_.function("NvEncodeAPICreateInstance")); + const auto get_max_version = reinterpret_cast( + module_.function("NvEncodeAPIGetMaxSupportedVersion")); + + std::uint32_t maximum_version = 0; + NV_ENCODE_API_FUNCTION_LIST empty_functions{}; + check_nvenc(get_max_version(&maximum_version), + "Querying the NVENC driver version", empty_functions); + if (maximum_version < NVENCAPI_VERSION) { + throw NvencError("The NVIDIA driver NVENC API is older than the pinned " + "Chatto headers (driver=" + + std::to_string(maximum_version >> 4U) + "." + + std::to_string(maximum_version & 0x0fU) + + ", headers=" + std::to_string(NVENCAPI_MAJOR_VERSION) + + "." + std::to_string(NVENCAPI_MINOR_VERSION) + ")"); + } + functions_.version = NV_ENCODE_API_FUNCTION_LIST_VER; + check_nvenc(create_instance(&functions_), "Loading the NVENC API", + functions_); + } + + void initialize_device() { + adapter_ = find_nvidia_adapter(); + constexpr std::array feature_levels{ + D3D_FEATURE_LEVEL_12_1, + D3D_FEATURE_LEVEL_12_0, + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL selected_level{}; + const HRESULT result = D3D11CreateDevice( + adapter_.Get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + feature_levels.data(), static_cast(feature_levels.size()), + D3D11_SDK_VERSION, &device_, &selected_level, &context_); + if (FAILED(result)) { + throw NvencError("Could not create the NVIDIA D3D11 encoding device"); + } + ComPtr multithread; + if (SUCCEEDED(context_.As(&multithread))) { + multithread->SetMultithreadProtected(TRUE); + } + if (FAILED(device_.As(&video_device_)) || + FAILED(context_.As(&video_context_))) { + throw NvencError("The NVIDIA device has no D3D11 video processor"); + } + static_cast(video_context_.As(&video_context1_)); + } + + void open_session() { + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS open{}; + open.version = NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER; + open.device = device_.Get(); + open.deviceType = NV_ENC_DEVICE_TYPE_DIRECTX; + open.apiVersion = NVENCAPI_VERSION; + check_nvenc(functions_.nvEncOpenEncodeSessionEx(&open, &encoder_), + "Opening the NVENC session", functions_); + } + + void initialize_encoder() { + NV_ENC_PRESET_CONFIG preset{}; + preset.version = NV_ENC_PRESET_CONFIG_VER; + preset.presetCfg.version = NV_ENC_CONFIG_VER; + check_nvenc(functions_.nvEncGetEncodePresetConfigEx( + encoder_, NV_ENC_CODEC_H264_GUID, NV_ENC_PRESET_P5_GUID, + NV_ENC_TUNING_INFO_LOW_LATENCY, &preset), + "Reading the NVENC low-latency preset", functions_, encoder_); + config_ = preset.presetCfg; + config_.version = NV_ENC_CONFIG_VER; + // Our current LiveKit fork negotiates constrained-baseline H.264. Keep the + // bitstream honest until the negotiated profile is returned to this helper. + config_.profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID; + config_.gopLength = frames_per_second_ * 2; + config_.frameIntervalP = 1; + config_.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; + config_.rcParams.averageBitRate = target_bitrate_bps_; + config_.rcParams.maxBitRate = target_bitrate_bps_; + config_.rcParams.vbvBufferSize = + std::max(1U, target_bitrate_bps_ / frames_per_second_); + config_.rcParams.vbvInitialDelay = config_.rcParams.vbvBufferSize; + config_.rcParams.multiPass = NV_ENC_TWO_PASS_QUARTER_RESOLUTION; + config_.rcParams.enableAQ = 1; + config_.rcParams.aqStrength = 8; + config_.encodeCodecConfig.h264Config.idrPeriod = config_.gopLength; + config_.encodeCodecConfig.h264Config.repeatSPSPPS = 1; + config_.encodeCodecConfig.h264Config.entropyCodingMode = + NV_ENC_H264_ENTROPY_CODING_MODE_CAVLC; + + initialization_.version = NV_ENC_INITIALIZE_PARAMS_VER; + initialization_.encodeGUID = NV_ENC_CODEC_H264_GUID; + initialization_.presetGUID = NV_ENC_PRESET_P5_GUID; + initialization_.encodeWidth = width_; + initialization_.encodeHeight = height_; + initialization_.darWidth = width_; + initialization_.darHeight = height_; + initialization_.frameRateNum = frames_per_second_; + initialization_.frameRateDen = 1; + initialization_.enableEncodeAsync = 0; + initialization_.enablePTD = 1; + initialization_.encodeConfig = &config_; + initialization_.maxEncodeWidth = width_; + initialization_.maxEncodeHeight = height_; + initialization_.tuningInfo = NV_ENC_TUNING_INFO_LOW_LATENCY; + check_nvenc(functions_.nvEncInitializeEncoder(encoder_, &initialization_), + "Initializing the NVENC H.264 encoder", functions_, encoder_); + } + + void create_surfaces() { + D3D11_TEXTURE2D_DESC texture_description{}; + texture_description.Width = width_; + texture_description.Height = height_; + texture_description.MipLevels = 1; + texture_description.ArraySize = 1; + texture_description.Format = DXGI_FORMAT_NV12; + texture_description.SampleDesc.Count = 1; + texture_description.Usage = D3D11_USAGE_DEFAULT; + texture_description.BindFlags = D3D11_BIND_RENDER_TARGET; + + for (auto &surface : surfaces_) { + if (FAILED(device_->CreateTexture2D(&texture_description, nullptr, + &surface.texture))) { + throw NvencError("Could not create an NVENC NV12 input texture"); + } + NV_ENC_REGISTER_RESOURCE registration{}; + registration.version = NV_ENC_REGISTER_RESOURCE_VER; + registration.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_DIRECTX; + registration.width = width_; + registration.height = height_; + registration.resourceToRegister = surface.texture.Get(); + registration.bufferFormat = NV_ENC_BUFFER_FORMAT_NV12; + registration.bufferUsage = NV_ENC_INPUT_IMAGE; + check_nvenc(functions_.nvEncRegisterResource(encoder_, ®istration), + "Registering an NVENC input texture", functions_, encoder_); + surface.registered = registration.registeredResource; + + NV_ENC_CREATE_BITSTREAM_BUFFER bitstream{}; + bitstream.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER; + check_nvenc(functions_.nvEncCreateBitstreamBuffer(encoder_, &bitstream), + "Creating an NVENC output buffer", functions_, encoder_); + surface.bitstream = bitstream.bitstreamBuffer; + } + } + + void ensure_video_processor(const std::uint32_t source_width, + const std::uint32_t source_height) { + if (video_processor_ && source_width == processor_source_width_ && + source_height == processor_source_height_) { + return; + } + for (auto &surface : surfaces_) { + surface.output_view.Reset(); + } + video_processor_.Reset(); + video_enumerator_.Reset(); + D3D11_VIDEO_PROCESSOR_CONTENT_DESC content{}; + content.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; + content.InputFrameRate = {frames_per_second_, 1}; + content.InputWidth = source_width; + content.InputHeight = source_height; + content.OutputFrameRate = {frames_per_second_, 1}; + content.OutputWidth = width_; + content.OutputHeight = height_; + content.Usage = D3D11_VIDEO_USAGE_OPTIMAL_QUALITY; + if (FAILED(video_device_->CreateVideoProcessorEnumerator( + &content, &video_enumerator_))) { + throw NvencError("Could not enumerate NVIDIA video processing"); + } + UINT input_flags = 0; + UINT output_flags = 0; + if (FAILED(video_enumerator_->CheckVideoProcessorFormat( + DXGI_FORMAT_B8G8R8A8_UNORM, &input_flags)) || + (input_flags & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT) == 0 || + FAILED(video_enumerator_->CheckVideoProcessorFormat(DXGI_FORMAT_NV12, + &output_flags)) || + (output_flags & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT) == 0) { + throw NvencError( + "The NVIDIA video processor cannot convert BGRA to NV12"); + } + if (FAILED(video_device_->CreateVideoProcessor(video_enumerator_.Get(), 0, + &video_processor_))) { + throw NvencError("Could not create the NVIDIA video processor"); + } + for (auto &surface : surfaces_) { + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC view{}; + view.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + view.Texture2D.MipSlice = 0; + if (FAILED(video_device_->CreateVideoProcessorOutputView( + surface.texture.Get(), video_enumerator_.Get(), &view, + &surface.output_view))) { + throw NvencError("Could not create an NV12 video processor output"); + } + } + processor_source_width_ = source_width; + processor_source_height_ = source_height; + } + + void convert_bgra_to_nv12(ID3D11Texture2D &source, + const std::uint32_t source_width, + const std::uint32_t source_height, + Surface &surface) { + D3D11_TEXTURE2D_DESC source_description{}; + source.GetDesc(&source_description); + if (source_description.Width != source_width || + source_description.Height != source_height || + source_description.Format != DXGI_FORMAT_B8G8R8A8_UNORM) { + throw NvencError("NVENC received an unsupported GPU capture texture"); + } + ComPtr shared_resource; + HANDLE shared_handle = nullptr; + if (FAILED(source.QueryInterface(IID_PPV_ARGS(&shared_resource))) || + FAILED(shared_resource->GetSharedHandle(&shared_handle)) || + shared_handle == nullptr) { + throw NvencError("The GPU capture texture is not shareable"); + } + ComPtr input; + if (FAILED( + device_->OpenSharedResource(shared_handle, IID_PPV_ARGS(&input)))) { + throw NvencError("NVENC could not open the GPU capture texture"); + } + ComPtr keyed_mutex; + if (FAILED(input.As(&keyed_mutex)) || + FAILED(keyed_mutex->AcquireSync(1, 5'000))) { + throw NvencError("NVENC could not acquire the GPU capture texture"); + } + try { + ensure_video_processor(source_width, source_height); + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC view{}; + view.FourCC = 0; + view.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + view.Texture2D.MipSlice = 0; + view.Texture2D.ArraySlice = 0; + ComPtr input_view; + if (FAILED(video_device_->CreateVideoProcessorInputView( + input.Get(), video_enumerator_.Get(), &view, &input_view))) { + throw NvencError("Could not create a BGRA video processor input"); + } + const RECT source_rectangle{0, 0, static_cast(source_width), + static_cast(source_height)}; + const RECT destination_rectangle{0, 0, static_cast(width_), + static_cast(height_)}; + video_context_->VideoProcessorSetStreamSourceRect( + video_processor_.Get(), 0, TRUE, &source_rectangle); + video_context_->VideoProcessorSetStreamDestRect( + video_processor_.Get(), 0, TRUE, &destination_rectangle); + video_context_->VideoProcessorSetOutputTargetRect( + video_processor_.Get(), TRUE, &destination_rectangle); + if (video_context1_) { + video_context1_->VideoProcessorSetStreamColorSpace1( + video_processor_.Get(), 0, DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709); + video_context1_->VideoProcessorSetOutputColorSpace1( + video_processor_.Get(), + DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709); + } + D3D11_VIDEO_PROCESSOR_STREAM stream{}; + stream.Enable = TRUE; + stream.pInputSurface = input_view.Get(); + if (FAILED(video_context_->VideoProcessorBlt( + video_processor_.Get(), surface.output_view.Get(), + static_cast(encoded_frame_index_++), 1, &stream))) { + throw NvencError("The GPU BGRA-to-NV12 conversion failed"); + } + } catch (...) { + static_cast(keyed_mutex->ReleaseSync(0)); + throw; + } + if (FAILED(keyed_mutex->ReleaseSync(0))) { + throw NvencError("NVENC could not release the GPU capture texture"); + } + } + + void release() noexcept { + if (encoder_ != nullptr) { + for (auto &surface : surfaces_) { + if (surface.bitstream != nullptr && + functions_.nvEncDestroyBitstreamBuffer != nullptr) { + static_cast(functions_.nvEncDestroyBitstreamBuffer( + encoder_, surface.bitstream)); + surface.bitstream = nullptr; + } + if (surface.registered != nullptr && + functions_.nvEncUnregisterResource != nullptr) { + static_cast( + functions_.nvEncUnregisterResource(encoder_, surface.registered)); + surface.registered = nullptr; + } + surface.output_view.Reset(); + surface.texture.Reset(); + } + if (functions_.nvEncDestroyEncoder != nullptr) { + static_cast(functions_.nvEncDestroyEncoder(encoder_)); + } + encoder_ = nullptr; + } + } + + std::uint32_t width_; + std::uint32_t height_; + std::uint32_t frames_per_second_; + std::uint32_t target_bitrate_bps_; + UniqueModule module_; + NV_ENCODE_API_FUNCTION_LIST functions_{}; + ComPtr adapter_; + ComPtr device_; + ComPtr context_; + ComPtr video_device_; + ComPtr video_context_; + ComPtr video_context1_; + ComPtr video_enumerator_; + ComPtr video_processor_; + void *encoder_ = nullptr; + NV_ENC_CONFIG config_{}; + NV_ENC_INITIALIZE_PARAMS initialization_{}; + std::array surfaces_{}; + std::size_t surface_index_ = 0; + std::uint32_t processor_source_width_ = 0; + std::uint32_t processor_source_height_ = 0; + std::uint64_t encoded_frame_index_ = 0; + double last_gpu_conversion_submit_ms_ = 0; + double last_encoder_submit_ms_ = 0; + double last_bitstream_wait_ms_ = 0; + bool finished_ = false; + std::string implementation_name_ = "NVIDIA NVENC H.264 (direct D3D11)"; +}; + +} // namespace + +std::unique_ptr +create_hardware_h264_encoder(const std::uint32_t width, + const std::uint32_t height, + const std::uint32_t frames_per_second, + const std::uint32_t target_bitrate_bps) { + try { + return std::make_unique( + width, height, frames_per_second, target_bitrate_bps); + } catch (const std::exception &error) { + std::cerr << "[Chatto Desktop capture] Direct NVENC unavailable; using " + "Media Foundation: " + << error.what() << '\n'; + return std::make_unique( + width, height, frames_per_second, target_bitrate_bps); + } +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/preview_window.cpp b/apps/desktop/native/windows-capture-probe/src/preview_window.cpp new file mode 100644 index 0000000000..67c793505d --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/preview_window.cpp @@ -0,0 +1,241 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "preview_window.h" + +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +constexpr wchar_t kPreviewWindowClass[] = + L"ChattoWindowsCaptureProbePreviewWindow"; +constexpr wchar_t kPreviewWindowTitle[] = + L"Chatto Windows Capture Probe Preview"; + +[[nodiscard]] ATOM register_preview_window_class() { + static const ATOM window_class = [] { + WNDCLASSEXW description{}; + description.cbSize = sizeof(description); + description.hInstance = GetModuleHandleW(nullptr); + description.lpfnWndProc = PreviewWindow::window_procedure; + description.lpszClassName = kPreviewWindowClass; + description.hCursor = LoadCursorW(nullptr, IDC_ARROW); + description.hbrBackground = + reinterpret_cast(GetStockObject(BLACK_BRUSH)); + const ATOM registered = RegisterClassExW(&description); + if (registered == 0 && GetLastError() != ERROR_CLASS_ALREADY_EXISTS) { + winrt::throw_last_error(); + } + return registered; + }(); + return window_class; +} + +[[nodiscard]] SIZE bounded_preview_size( + const std::uint32_t width, + const std::uint32_t height) { + RECT work_area{}; + if (!SystemParametersInfoW(SPI_GETWORKAREA, 0, &work_area, 0)) { + return SIZE{static_cast(width), static_cast(height)}; + } + const double available_width = + static_cast(work_area.right - work_area.left) * 0.8; + const double available_height = + static_cast(work_area.bottom - work_area.top) * 0.8; + const double scale = std::min( + {1.0, + available_width / static_cast(width), + available_height / static_cast(height)}); + return SIZE{ + static_cast(static_cast(width) * scale), + static_cast(static_cast(height) * scale), + }; +} + +} // namespace + +PreviewWindow::PreviewWindow( + ID3D11Device& device, + ID3D11DeviceContext& context, + const std::uint32_t width, + const std::uint32_t height) + : width_(width), height_(height) { + static_cast(register_preview_window_class()); + winrt::check_hresult(context.QueryInterface( + __uuidof(ID3D11DeviceContext), context_.put_void())); + + const SIZE client_size = bounded_preview_size(width, height); + RECT bounds{0, 0, client_size.cx, client_size.cy}; + winrt::check_bool(AdjustWindowRectEx( + &bounds, WS_OVERLAPPEDWINDOW, FALSE, 0)); + window_ = CreateWindowExW( + 0, + kPreviewWindowClass, + kPreviewWindowTitle, + WS_OVERLAPPEDWINDOW, + CW_USEDEFAULT, + CW_USEDEFAULT, + bounds.right - bounds.left, + bounds.bottom - bounds.top, + nullptr, + nullptr, + GetModuleHandleW(nullptr), + this); + if (window_ == nullptr) { + winrt::throw_last_error(); + } + + winrt::com_ptr dxgi_device; + winrt::check_hresult( + device.QueryInterface(__uuidof(IDXGIDevice), dxgi_device.put_void())); + winrt::com_ptr adapter; + winrt::check_hresult(dxgi_device->GetAdapter(adapter.put())); + winrt::com_ptr factory; + winrt::check_hresult(adapter->GetParent(__uuidof(IDXGIFactory2), factory.put_void())); + + DXGI_SWAP_CHAIN_DESC1 description{}; + description.Width = width; + description.Height = height; + description.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + description.SampleDesc.Count = 1; + description.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT; + description.BufferCount = 2; + description.Scaling = DXGI_SCALING_STRETCH; + description.SwapEffect = DXGI_SWAP_EFFECT_FLIP_DISCARD; + description.AlphaMode = DXGI_ALPHA_MODE_IGNORE; + winrt::check_hresult(factory->CreateSwapChainForHwnd( + &device, + window_, + &description, + nullptr, + nullptr, + swap_chain_.put())); + winrt::check_hresult(factory->MakeWindowAssociation(window_, DXGI_MWA_NO_ALT_ENTER)); + winrt::check_hresult( + swap_chain_->GetBuffer(0, __uuidof(ID3D11Texture2D), back_buffer_.put_void())); + + ShowWindow(window_, SW_SHOWNORMAL); + UpdateWindow(window_); +} + +PreviewWindow::~PreviewWindow() { + std::scoped_lock lock(mutex_); + back_buffer_ = nullptr; + swap_chain_ = nullptr; + if (window_ != nullptr && !closed_) { + DestroyWindow(window_); + window_ = nullptr; + } +} + +void PreviewWindow::present(ID3D11Texture2D& texture) { + std::scoped_lock lock(mutex_); + if (closed_ || window_ == nullptr) { + return; + } + + D3D11_TEXTURE2D_DESC description{}; + texture.GetDesc(&description); + if (description.Width != width_ || description.Height != height_) { + resize_swap_chain(description.Width, description.Height); + resize_window(description.Width, description.Height); + } + + context_->CopyResource(back_buffer_.get(), &texture); + const HRESULT result = swap_chain_->Present(0, DXGI_PRESENT_DO_NOT_WAIT); + if (result != DXGI_ERROR_WAS_STILL_DRAWING) { + winrt::check_hresult(result); + } +} + +void PreviewWindow::pump_messages() { + MSG message{}; + while (PeekMessageW(&message, window_, 0, 0, PM_REMOVE)) { + TranslateMessage(&message); + DispatchMessageW(&message); + } +} + +void PreviewWindow::update_status( + const std::uint64_t frames, + const std::uint64_t sampled_frames, + const std::uint64_t changed_samples, + const double observed_frames_per_second, + const std::uint64_t audio_frames, + const float latest_audio_peak, + const std::uint64_t audio_discontinuities) { + if (closed_ || window_ == nullptr) { + return; + } + std::wostringstream title; + title << kPreviewWindowTitle << L" — " << std::fixed << std::setprecision(1) + << observed_frames_per_second << L" fps | frames " << frames + << L" | samples " << changed_samples << L"/" << sampled_frames + << L" changed | audio " << std::setprecision(3) << latest_audio_peak + << L" peak (" << audio_frames << L" frames, " + << audio_discontinuities << L" gaps)"; + SetWindowTextW(window_, title.str().c_str()); +} + +bool PreviewWindow::closed() const noexcept { + return closed_; +} + +LRESULT CALLBACK PreviewWindow::window_procedure( + HWND window, + const UINT message, + const WPARAM word_parameter, + const LPARAM long_parameter) { + auto* preview = reinterpret_cast( + GetWindowLongPtrW(window, GWLP_USERDATA)); + if (message == WM_NCCREATE) { + const auto* creation = reinterpret_cast(long_parameter); + preview = static_cast(creation->lpCreateParams); + SetWindowLongPtrW( + window, GWLP_USERDATA, reinterpret_cast(preview)); + } + if (message == WM_CLOSE) { + DestroyWindow(window); + return 0; + } + if (message == WM_DESTROY && preview != nullptr) { + preview->closed_ = true; + return 0; + } + return DefWindowProcW(window, message, word_parameter, long_parameter); +} + +void PreviewWindow::resize_swap_chain( + const std::uint32_t width, + const std::uint32_t height) { + back_buffer_ = nullptr; + winrt::check_hresult(swap_chain_->ResizeBuffers( + 0, width, height, DXGI_FORMAT_UNKNOWN, 0)); + winrt::check_hresult( + swap_chain_->GetBuffer(0, __uuidof(ID3D11Texture2D), back_buffer_.put_void())); + width_ = width; + height_ = height; +} + +void PreviewWindow::resize_window( + const std::uint32_t width, + const std::uint32_t height) { + const SIZE client_size = bounded_preview_size(width, height); + RECT bounds{0, 0, client_size.cx, client_size.cy}; + winrt::check_bool(AdjustWindowRectEx( + &bounds, WS_OVERLAPPEDWINDOW, FALSE, 0)); + SetWindowPos( + window_, + nullptr, + 0, + 0, + bounds.right - bounds.left, + bounds.bottom - bounds.top, + SWP_NOMOVE | SWP_NOACTIVATE | SWP_NOZORDER); +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/preview_window.h b/apps/desktop/native/windows-capture-probe/src/preview_window.h new file mode 100644 index 0000000000..7c90315fad --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/preview_window.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace chatto::capture { + +// Displays captured GPU textures directly through a flip-model DXGI swap chain. +// The preview owns no capture source identifiers and never reads pixels to disk. +class PreviewWindow { + public: + PreviewWindow( + ID3D11Device& device, + ID3D11DeviceContext& context, + std::uint32_t width, + std::uint32_t height); + ~PreviewWindow(); + + PreviewWindow(const PreviewWindow&) = delete; + PreviewWindow& operator=(const PreviewWindow&) = delete; + + void present(ID3D11Texture2D& texture); + void pump_messages(); + void update_status( + std::uint64_t frames, + std::uint64_t sampled_frames, + std::uint64_t changed_samples, + double observed_frames_per_second, + std::uint64_t audio_frames, + float latest_audio_peak, + std::uint64_t audio_discontinuities); + + [[nodiscard]] bool closed() const noexcept; + + static LRESULT CALLBACK window_procedure( + HWND window, + UINT message, + WPARAM word_parameter, + LPARAM long_parameter); + + private: + void resize_swap_chain(std::uint32_t width, std::uint32_t height); + void resize_window(std::uint32_t width, std::uint32_t height); + + std::mutex mutex_; + std::atomic_bool closed_ = false; + HWND window_ = nullptr; + winrt::com_ptr context_; + winrt::com_ptr swap_chain_; + winrt::com_ptr back_buffer_; + std::uint32_t width_ = 0; + std::uint32_t height_ = 0; +}; + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/video_capture.cpp b/apps/desktop/native/windows-capture-probe/src/video_capture.cpp new file mode 100644 index 0000000000..863b5d1e24 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/video_capture.cpp @@ -0,0 +1,1054 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "video_capture.h" + +#include "live_status.h" +#include "preview_window.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace chatto::capture { +namespace { + +using winrt::Windows::Graphics::Capture::Direct3D11CaptureFramePool; +using winrt::Windows::Graphics::Capture::GraphicsCaptureItem; +using winrt::Windows::Graphics::Capture::GraphicsCaptureSession; +using winrt::Windows::Graphics::DirectX::DirectXPixelFormat; +using winrt::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice; + +struct CaptureState { + std::mutex mutex; + std::condition_variable changed; + std::condition_variable callbacks_changed; + VideoCaptureMetrics metrics; + std::optional first_timestamp; + std::optional previous_timestamp; + std::int64_t last_timestamp = 0; + std::int64_t inferred_gap_threshold = 0; + std::uint32_t pool_width = 0; + std::uint32_t pool_height = 0; + double sampled_luminance_total = 0; + std::optional previous_sample_hash; + std::chrono::steady_clock::time_point last_frame_at = + std::chrono::steady_clock::now(); + std::size_t callbacks_in_flight = 0; + bool accepting_callbacks = true; + bool finished = false; +}; + +class CaptureCallbackLease { +public: + explicit CaptureCallbackLease(std::shared_ptr state) + : state_(std::move(state)) { + std::scoped_lock lock(state_->mutex); + if (state_->accepting_callbacks) { + state_->callbacks_in_flight += 1; + acquired_ = true; + } + } + + CaptureCallbackLease(const CaptureCallbackLease &) = delete; + CaptureCallbackLease &operator=(const CaptureCallbackLease &) = delete; + + ~CaptureCallbackLease() { + if (!acquired_) { + return; + } + { + std::scoped_lock lock(state_->mutex); + state_->callbacks_in_flight -= 1; + } + state_->callbacks_changed.notify_all(); + } + + [[nodiscard]] explicit operator bool() const { return acquired_; } + +private: + std::shared_ptr state_; + bool acquired_ = false; +}; + +void stop_and_wait_for_capture_callbacks( + const std::shared_ptr &state) { + std::unique_lock lock(state->mutex); + state->accepting_callbacks = false; + state->callbacks_changed.wait( + lock, [&state] { return state->callbacks_in_flight == 0; }); +} + +struct FrameSample { + std::uint64_t hash = 0; + double mean = 0; + std::uint8_t minimum = 0; + std::uint8_t maximum = 0; +}; + +[[nodiscard]] std::uint64_t file_time_ticks(const FILETIME &time) { + ULARGE_INTEGER value{}; + value.LowPart = time.dwLowDateTime; + value.HighPart = time.dwHighDateTime; + return value.QuadPart; +} + +[[nodiscard]] std::uint64_t process_cpu_ticks() { + FILETIME creation{}; + FILETIME exit{}; + FILETIME kernel{}; + FILETIME user{}; + winrt::check_bool( + GetProcessTimes(GetCurrentProcess(), &creation, &exit, &kernel, &user)); + return file_time_ticks(kernel) + file_time_ticks(user); +} + +void record_process_metrics( + VideoCaptureMetrics &metrics, + const std::chrono::steady_clock::duration wall_duration, + const std::uint64_t starting_cpu_ticks) { + metrics.wall_duration_seconds = + std::chrono::duration(wall_duration).count(); + metrics.process_cpu_seconds = + static_cast(process_cpu_ticks() - starting_cpu_ticks) / + 10'000'000.0; + if (metrics.wall_duration_seconds > 0) { + metrics.process_cpu_single_core_percent = + metrics.process_cpu_seconds / metrics.wall_duration_seconds * 100.0; + } + + PROCESS_MEMORY_COUNTERS_EX memory{}; + memory.cb = sizeof(memory); + winrt::check_bool(GetProcessMemoryInfo( + GetCurrentProcess(), reinterpret_cast(&memory), + sizeof(memory))); + metrics.peak_working_set_bytes = memory.PeakWorkingSetSize; +} + +[[nodiscard]] GraphicsCaptureItem create_capture_item(HWND window) { + auto interop = winrt::get_activation_factory(); + GraphicsCaptureItem item{nullptr}; + winrt::check_hresult(interop->CreateForWindow( + window, winrt::guid_of(), winrt::put_abi(item))); + return item; +} + +[[nodiscard]] GraphicsCaptureItem create_capture_item(HMONITOR monitor) { + auto interop = winrt::get_activation_factory(); + GraphicsCaptureItem item{nullptr}; + winrt::check_hresult(interop->CreateForMonitor( + monitor, winrt::guid_of(), winrt::put_abi(item))); + return item; +} + +struct DeviceResources { + winrt::com_ptr d3d_device; + winrt::com_ptr d3d_context; + IDirect3DDevice winrt_device{nullptr}; +}; + +struct DuplicationResources { + winrt::com_ptr d3d_device; + winrt::com_ptr d3d_context; + winrt::com_ptr duplication; + DXGI_OUTPUT_DESC output_description{}; +}; + +[[nodiscard]] winrt::com_ptr find_nvidia_adapter() { + winrt::com_ptr factory; + winrt::check_hresult( + CreateDXGIFactory1(__uuidof(IDXGIFactory1), factory.put_void())); + for (UINT index = 0;; ++index) { + winrt::com_ptr adapter; + const HRESULT result = factory->EnumAdapters1(index, adapter.put()); + if (result == DXGI_ERROR_NOT_FOUND) { + return nullptr; + } + winrt::check_hresult(result); + DXGI_ADAPTER_DESC1 description{}; + winrt::check_hresult(adapter->GetDesc1(&description)); + if (description.VendorId == 0x10de && + (description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) == 0) { + return adapter; + } + } +} + +[[nodiscard]] DeviceResources create_device() { + DeviceResources resources; + constexpr D3D_FEATURE_LEVEL feature_levels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL selected_feature_level{}; + + const auto nvidia_adapter = find_nvidia_adapter(); + winrt::check_hresult(D3D11CreateDevice( + nvidia_adapter.get(), + nvidia_adapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE, + nullptr, + D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + feature_levels, static_cast(std::size(feature_levels)), + D3D11_SDK_VERSION, resources.d3d_device.put(), &selected_feature_level, + resources.d3d_context.put())); + + const auto multithread = resources.d3d_context.as(); + multithread->SetMultithreadProtected(TRUE); + + const auto dxgi_device = resources.d3d_device.as(); + winrt::com_ptr inspectable; + winrt::check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgi_device.get(), + inspectable.put())); + resources.winrt_device = inspectable.as(); + return resources; +} + +[[nodiscard]] DuplicationResources +create_duplication_resources(const HMONITOR monitor) { + winrt::com_ptr factory; + winrt::check_hresult( + CreateDXGIFactory1(__uuidof(IDXGIFactory1), factory.put_void())); + + for (UINT adapter_index = 0;; ++adapter_index) { + winrt::com_ptr adapter; + const HRESULT adapter_result = + factory->EnumAdapters1(adapter_index, adapter.put()); + if (adapter_result == DXGI_ERROR_NOT_FOUND) { + break; + } + winrt::check_hresult(adapter_result); + + for (UINT output_index = 0;; ++output_index) { + winrt::com_ptr output; + const HRESULT output_result = + adapter->EnumOutputs(output_index, output.put()); + if (output_result == DXGI_ERROR_NOT_FOUND) { + break; + } + winrt::check_hresult(output_result); + + DXGI_OUTPUT_DESC output_description{}; + winrt::check_hresult(output->GetDesc(&output_description)); + if (output_description.Monitor != monitor) { + continue; + } + + DuplicationResources resources; + constexpr D3D_FEATURE_LEVEL feature_levels[] = { + D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL selected_feature_level{}; + winrt::check_hresult(D3D11CreateDevice( + adapter.get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, feature_levels, + static_cast(std::size(feature_levels)), D3D11_SDK_VERSION, + resources.d3d_device.put(), &selected_feature_level, + resources.d3d_context.put())); + const auto multithread = resources.d3d_context.as(); + multithread->SetMultithreadProtected(TRUE); + const auto output1 = output.as(); + winrt::check_hresult(output1->DuplicateOutput( + resources.d3d_device.get(), resources.duplication.put())); + resources.output_description = output_description; + return resources; + } + } + throw std::runtime_error( + "The fullscreen window's monitor cannot be duplicated"); +} + +[[nodiscard]] FrameSample +sample_texture(ID3D11Device &device, ID3D11DeviceContext &context, + ID3D11Texture2D &texture, + const D3D11_TEXTURE2D_DESC &description, + winrt::com_ptr &staging_texture) { + D3D11_TEXTURE2D_DESC staging_description{}; + if (staging_texture) { + staging_texture->GetDesc(&staging_description); + } + if (!staging_texture || staging_description.Width != description.Width || + staging_description.Height != description.Height || + staging_description.Format != description.Format) { + staging_description = description; + staging_description.BindFlags = 0; + staging_description.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + staging_description.MiscFlags = 0; + staging_description.Usage = D3D11_USAGE_STAGING; + staging_texture = nullptr; + winrt::check_hresult(device.CreateTexture2D(&staging_description, nullptr, + staging_texture.put())); + } + + context.CopyResource(staging_texture.get(), &texture); + D3D11_MAPPED_SUBRESOURCE mapped{}; + winrt::check_hresult( + context.Map(staging_texture.get(), 0, D3D11_MAP_READ, 0, &mapped)); + + FrameSample sample; + sample.hash = 1'469'598'103'934'665'603ULL; + sample.minimum = std::numeric_limits::max(); + const std::uint32_t horizontal_step = std::max(1U, description.Width / 64U); + const std::uint32_t vertical_step = std::max(1U, description.Height / 36U); + std::uint64_t luminance_total = 0; + std::uint64_t sample_count = 0; + + for (std::uint32_t y = vertical_step / 2; y < description.Height; + y += vertical_step) { + const auto *row = static_cast(mapped.pData) + + static_cast(y) * mapped.RowPitch; + for (std::uint32_t x = horizontal_step / 2; x < description.Width; + x += horizontal_step) { + const auto *pixel = row + static_cast(x) * 4; + const auto luminance = static_cast( + (static_cast(pixel[0]) + pixel[1] + pixel[2]) / 3U); + sample.minimum = std::min(sample.minimum, luminance); + sample.maximum = std::max(sample.maximum, luminance); + luminance_total += luminance; + sample_count += 1; + sample.hash ^= luminance; + sample.hash *= 1'099'511'628'211ULL; + } + } + context.Unmap(staging_texture.get(), 0); + if (sample_count > 0) { + sample.mean = static_cast(luminance_total) / + static_cast(sample_count); + } + return sample; +} + +[[nodiscard]] VideoFrameData +copy_texture_gpu(ID3D11Device &device, ID3D11DeviceContext &context, + ID3D11Texture2D &texture, + const D3D11_TEXTURE2D_DESC &description, + const std::int64_t timestamp_100ns, + const std::optional region = std::nullopt) { + const auto copy_start = std::chrono::steady_clock::now(); + const std::uint32_t width = + region ? region->right - region->left : description.Width; + const std::uint32_t height = + region ? region->bottom - region->top : description.Height; + D3D11_TEXTURE2D_DESC copy_description{}; + copy_description.Width = width; + copy_description.Height = height; + copy_description.MipLevels = 1; + copy_description.ArraySize = 1; + copy_description.Format = description.Format; + copy_description.SampleDesc.Count = 1; + copy_description.Usage = D3D11_USAGE_DEFAULT; + copy_description.BindFlags = 0; + copy_description.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX; + winrt::com_ptr copy; + winrt::check_hresult( + device.CreateTexture2D(©_description, nullptr, copy.put())); + const auto keyed_mutex = copy.as(); + winrt::check_hresult(keyed_mutex->AcquireSync(0, INFINITE)); + if (region) { + context.CopySubresourceRegion(copy.get(), 0, 0, 0, 0, &texture, 0, + &*region); + } else { + context.CopyResource(copy.get(), &texture); + } + winrt::check_hresult(keyed_mutex->ReleaseSync(1)); + return { + .width = width, + .height = height, + .timestamp_100ns = timestamp_100ns, + .gpu_copy_submit_duration_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - copy_start) + .count(), + .bgra_texture = std::move(copy), + }; +} + +[[nodiscard]] std::optional +window_region_on_output(HWND window, const DXGI_OUTPUT_DESC &output) { + RECT window_rectangle{}; + if (!GetWindowRect(window, &window_rectangle)) { + return std::nullopt; + } + const auto &desktop = output.DesktopCoordinates; + const LONG left = std::max(window_rectangle.left, desktop.left); + const LONG top = std::max(window_rectangle.top, desktop.top); + const LONG right = std::min(window_rectangle.right, desktop.right); + const LONG bottom = std::min(window_rectangle.bottom, desktop.bottom); + if (right <= left || bottom <= top) { + return std::nullopt; + } + return D3D11_BOX{ + .left = static_cast(left - desktop.left), + .top = static_cast(top - desktop.top), + .front = 0, + .right = static_cast(right - desktop.left), + .bottom = static_cast(bottom - desktop.top), + .back = 1, + }; +} + +[[nodiscard]] bool record_frame( + CaptureState &state, + const winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame &frame, + const D3D11_TEXTURE2D_DESC &texture_description, + const std::optional &sample) { + const std::int64_t timestamp = frame.SystemRelativeTime().count(); + const auto content_size = frame.ContentSize(); + std::scoped_lock lock(state.mutex); + + state.metrics.frames += 1; + state.last_frame_at = std::chrono::steady_clock::now(); + state.metrics.width = texture_description.Width; + state.metrics.height = texture_description.Height; + if (!state.first_timestamp) { + state.first_timestamp = timestamp; + state.metrics.first_timestamp_100ns = timestamp; + } + if (state.previous_timestamp) { + const std::int64_t interval = timestamp - *state.previous_timestamp; + if (interval > 0) { + state.metrics.longest_frame_interval_ms = + std::max(state.metrics.longest_frame_interval_ms, + static_cast(interval) / 10'000.0); + if (interval > state.inferred_gap_threshold) { + state.metrics.inferred_gaps += 1; + } + } + } + state.previous_timestamp = timestamp; + state.last_timestamp = timestamp; + state.metrics.last_timestamp_100ns = timestamp; + + if (sample) { + state.metrics.sampled_frames += 1; + if (state.previous_sample_hash && + *state.previous_sample_hash != sample->hash) { + state.metrics.changed_samples += 1; + } + state.previous_sample_hash = sample->hash; + if (sample->maximum <= 2) { + state.metrics.black_samples += 1; + } + if (state.metrics.sampled_frames == 1) { + state.metrics.sampled_luminance_min = sample->minimum; + } else { + state.metrics.sampled_luminance_min = + std::min(state.metrics.sampled_luminance_min, sample->minimum); + } + state.metrics.sampled_luminance_max = + std::max(state.metrics.sampled_luminance_max, sample->maximum); + state.sampled_luminance_total += sample->mean; + } + + const auto content_width = + static_cast(std::max(0, content_size.Width)); + const auto content_height = + static_cast(std::max(0, content_size.Height)); + if (content_width > 0 && content_height > 0 && + (content_width != state.pool_width || + content_height != state.pool_height)) { + state.pool_width = content_width; + state.pool_height = content_height; + state.metrics.resizes += 1; + return true; + } + return false; +} + +void record_error(CaptureState &state, const winrt::hresult_error &error) { + { + std::scoped_lock lock(state.mutex); + state.metrics.error = error.message().c_str(); + state.finished = true; + } + state.changed.notify_all(); +} + +void record_error(CaptureState &state, const std::exception &error) { + { + std::scoped_lock lock(state.mutex); + state.metrics.error = winrt::to_hstring(error.what()).c_str(); + state.finished = true; + } + state.changed.notify_all(); +} + +void finish_metrics(CaptureState &state) { + if (state.metrics.sampled_frames > 0) { + state.metrics.sampled_luminance_mean = + state.sampled_luminance_total / + static_cast(state.metrics.sampled_frames); + } + if (!state.first_timestamp || state.metrics.frames < 2) { + return; + } + const std::int64_t span = state.last_timestamp - *state.first_timestamp; + if (span <= 0) { + return; + } + state.metrics.timestamp_span_seconds = + static_cast(span) / 10'000'000.0; + state.metrics.observed_frames_per_second = + static_cast(state.metrics.frames - 1) / + state.metrics.timestamp_span_seconds; +} + +} // namespace + +std::pair window_capture_size(HWND window) { + if (!IsWindow(window)) { + throw std::invalid_argument("The selected window no longer exists"); + } + const auto size = create_capture_item(window).Size(); + if (size.Width <= 0 || size.Height <= 0) { + throw std::runtime_error( + "The selected window has no capturable content size"); + } + return { + static_cast(size.Width), + static_cast(size.Height), + }; +} + +bool is_foreground_monitor_covering_window(HWND window) { + if (!IsWindow(window) || IsIconic(window)) { + return false; + } + const HWND foreground = GetForegroundWindow(); + if (foreground != window && GetAncestor(foreground, GA_ROOT) != window) { + return false; + } + + const HMONITOR monitor = MonitorFromWindow(window, MONITOR_DEFAULTTONULL); + if (!monitor) { + return false; + } + MONITORINFO monitor_information{.cbSize = sizeof(MONITORINFO)}; + RECT window_rectangle{}; + if (!GetMonitorInfoW(monitor, &monitor_information) || + !GetWindowRect(window, &window_rectangle)) { + return false; + } + constexpr LONG tolerance = 2; + const RECT &monitor_rectangle = monitor_information.rcMonitor; + return window_rectangle.left <= monitor_rectangle.left + tolerance && + window_rectangle.top <= monitor_rectangle.top + tolerance && + window_rectangle.right >= monitor_rectangle.right - tolerance && + window_rectangle.bottom >= monitor_rectangle.bottom - tolerance; +} + +VideoCaptureMetrics capture_window_video( + HWND window, const std::chrono::seconds duration, + const std::uint32_t requested_frames_per_second, const bool show_preview, + std::shared_ptr live_status, + VideoFrameHandler frame_handler, const std::stop_token stop_token, + const std::chrono::milliseconds frame_stall_timeout, + const bool switch_on_monitor_covering_presentation) { + if (!IsWindow(window)) { + throw std::invalid_argument("The selected window no longer exists"); + } + + const auto wall_start = std::chrono::steady_clock::now(); + const std::uint64_t starting_cpu_ticks = process_cpu_ticks(); + if (duration.count() <= 0 || requested_frames_per_second == 0) { + throw std::invalid_argument( + "Capture duration and frame rate must be positive"); + } + + const auto item = create_capture_item(window); + const auto device = create_device(); + const auto initial_size = item.Size(); + if (initial_size.Width <= 0 || initial_size.Height <= 0) { + throw std::runtime_error( + "The selected window has no capturable content size"); + } + + auto state = std::make_shared(); + state->pool_width = static_cast(initial_size.Width); + state->pool_height = static_cast(initial_size.Height); + const auto expected_interval = + 10'000'000LL / static_cast(requested_frames_per_second); + state->inferred_gap_threshold = expected_interval + (expected_interval / 2); + std::shared_ptr preview; + if (show_preview) { + if (!live_status) { + live_status = std::make_shared(); + } + preview = std::make_shared( + *device.d3d_device, *device.d3d_context, + static_cast(initial_size.Width), + static_cast(initial_size.Height)); + } + + auto frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( + device.winrt_device, DirectXPixelFormat::B8G8R8A8UIntNormalized, 3, + initial_size); + auto session = frame_pool.CreateCaptureSession(item); + session.IsCursorCaptureEnabled(false); + + const auto frame_token = frame_pool.FrameArrived( + [state, winrt_device = device.winrt_device, + d3d_device = device.d3d_device, d3d_context = device.d3d_context, + preview, frame_handler = std::move(frame_handler), + staging_texture = winrt::com_ptr{}, + frames_seen = std::uint64_t{0}](const Direct3D11CaptureFramePool &sender, + const auto &) mutable noexcept { + CaptureCallbackLease callback(state); + if (!callback) { + return; + } + try { + const auto frame = sender.TryGetNextFrame(); + if (frame) { + const auto surface_access = + frame.Surface() + .as(); + winrt::com_ptr texture; + winrt::check_hresult(surface_access->GetInterface( + __uuidof(ID3D11Texture2D), texture.put_void())); + D3D11_TEXTURE2D_DESC texture_description{}; + texture->GetDesc(&texture_description); + frames_seen += 1; + std::optional sample; + if (preview && (frames_seen == 1 || frames_seen % 12 == 0)) { + sample = sample_texture(*d3d_device, *d3d_context, *texture, + texture_description, staging_texture); + } + if (preview) { + preview->present(*texture); + } + if (frame_handler) { + frame_handler(copy_texture_gpu( + *d3d_device, *d3d_context, *texture, texture_description, + frame.SystemRelativeTime().count())); + } + const bool resized = + record_frame(*state, frame, texture_description, sample); + frame.Close(); + if (resized) { + winrt::Windows::Graphics::SizeInt32 new_size{}; + { + std::scoped_lock lock(state->mutex); + new_size.Width = static_cast(state->pool_width); + new_size.Height = static_cast(state->pool_height); + } + sender.Recreate(winrt_device, + DirectXPixelFormat::B8G8R8A8UIntNormalized, 3, + new_size); + } + } + } catch (const winrt::hresult_error &error) { + record_error(*state, error); + } catch (const std::exception &error) { + record_error(*state, error); + } + }); + const auto closed_token = + item.Closed([state](const GraphicsCaptureItem &, const auto &) noexcept { + { + std::scoped_lock lock(state->mutex); + state->metrics.source_closed = true; + state->finished = true; + } + state->changed.notify_all(); + }); + + session.StartCapture(); + if (preview) { + const auto deadline = std::chrono::steady_clock::now() + duration; + while (std::chrono::steady_clock::now() < deadline && !preview->closed()) { + preview->pump_messages(); + { + std::scoped_lock lock(state->mutex); + if (state->finished) { + break; + } + double live_frames_per_second = 0; + if (state->first_timestamp && state->metrics.frames >= 2 && + state->last_timestamp > *state->first_timestamp) { + live_frames_per_second = + static_cast(state->metrics.frames - 1) / + (static_cast(state->last_timestamp - + *state->first_timestamp) / + 10'000'000.0); + } + preview->update_status( + state->metrics.frames, state->metrics.sampled_frames, + state->metrics.changed_samples, live_frames_per_second, + live_status->audio_frames.load(std::memory_order_relaxed), + live_status->latest_audio_peak.load(std::memory_order_relaxed), + live_status->audio_discontinuities.load(std::memory_order_relaxed)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(16)); + } + std::scoped_lock lock(state->mutex); + state->finished = true; + } else { + std::unique_lock lock(state->mutex); + const auto deadline = wall_start + duration; + while (!state->finished && std::chrono::steady_clock::now() < deadline) { + if (stop_token.stop_requested()) { + state->metrics.stop_requested = true; + break; + } + if (!IsWindow(window)) { + state->metrics.source_closed = true; + break; + } + if (switch_on_monitor_covering_presentation && + is_foreground_monitor_covering_window(window)) { + // Window WGC can continue emitting sparse heartbeat frames while a + // monitor-covering flip-model game bypasses composition. Switch source + // types based on presentation state rather than waiting for zero + // frames. + state->metrics.frame_stalled = true; + break; + } + if (frame_stall_timeout.count() > 0 && + std::chrono::steady_clock::now() - state->last_frame_at >= + frame_stall_timeout) { + state->metrics.frame_stalled = true; + break; + } + state->changed.wait_for(lock, std::chrono::milliseconds(100), + [state] { return state->finished; }); + } + state->finished = true; + } + + item.Closed(closed_token); + frame_pool.FrameArrived(frame_token); + stop_and_wait_for_capture_callbacks(state); + session.Close(); + frame_pool.Close(); + + std::scoped_lock lock(state->mutex); + finish_metrics(*state); + record_process_metrics(state->metrics, + std::chrono::steady_clock::now() - wall_start, + starting_cpu_ticks); + return state->metrics; +} + +VideoCaptureMetrics capture_monitor_wgc_video_impl( + HMONITOR monitor, HWND presentation_window, + const std::chrono::seconds duration, + const std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler, const std::stop_token stop_token, + const std::chrono::milliseconds frame_stall_timeout) { + MONITORINFO monitor_information{}; + monitor_information.cbSize = sizeof(monitor_information); + if (!monitor || !GetMonitorInfoW(monitor, &monitor_information)) { + throw std::invalid_argument("The selected monitor no longer exists"); + } + if (duration.count() <= 0 || requested_frames_per_second == 0) { + throw std::invalid_argument( + "Capture duration and frame rate must be positive"); + } + const auto wall_start = std::chrono::steady_clock::now(); + const std::uint64_t starting_cpu_ticks = process_cpu_ticks(); + const auto item = create_capture_item(monitor); + const auto device = create_device(); + const auto initial_size = item.Size(); + if (initial_size.Width <= 0 || initial_size.Height <= 0) { + throw std::runtime_error("The selected monitor has no capturable size"); + } + + auto state = std::make_shared(); + state->pool_width = static_cast(initial_size.Width); + state->pool_height = static_cast(initial_size.Height); + const auto expected_interval = + 10'000'000LL / static_cast(requested_frames_per_second); + state->inferred_gap_threshold = expected_interval + (expected_interval / 2); + + auto frame_pool = Direct3D11CaptureFramePool::CreateFreeThreaded( + device.winrt_device, DirectXPixelFormat::B8G8R8A8UIntNormalized, 3, + initial_size); + auto session = frame_pool.CreateCaptureSession(item); + session.IsCursorCaptureEnabled(presentation_window == nullptr); + + const auto frame_token = frame_pool.FrameArrived( + [state, winrt_device = device.winrt_device, + d3d_device = device.d3d_device, d3d_context = device.d3d_context, + frame_handler = + std::move(frame_handler)](const Direct3D11CaptureFramePool &sender, + const auto &) mutable noexcept { + CaptureCallbackLease callback(state); + if (!callback) { + return; + } + try { + const auto frame = sender.TryGetNextFrame(); + if (!frame) { + return; + } + const auto surface_access = + frame.Surface() + .as(); + winrt::com_ptr texture; + winrt::check_hresult(surface_access->GetInterface( + __uuidof(ID3D11Texture2D), texture.put_void())); + D3D11_TEXTURE2D_DESC texture_description{}; + texture->GetDesc(&texture_description); + if (frame_handler) { + frame_handler(copy_texture_gpu(*d3d_device, *d3d_context, *texture, + texture_description, + frame.SystemRelativeTime().count())); + } + const bool resized = + record_frame(*state, frame, texture_description, std::nullopt); + frame.Close(); + if (resized) { + winrt::Windows::Graphics::SizeInt32 new_size{}; + { + std::scoped_lock lock(state->mutex); + new_size.Width = static_cast(state->pool_width); + new_size.Height = static_cast(state->pool_height); + } + sender.Recreate(winrt_device, + DirectXPixelFormat::B8G8R8A8UIntNormalized, 3, + new_size); + } + } catch (const winrt::hresult_error &error) { + record_error(*state, error); + } catch (const std::exception &error) { + record_error(*state, error); + } + }); + const auto closed_token = + item.Closed([state](const GraphicsCaptureItem &, const auto &) noexcept { + { + std::scoped_lock lock(state->mutex); + state->metrics.frame_stalled = true; + state->finished = true; + } + state->changed.notify_all(); + }); + + session.StartCapture(); + std::unique_lock lock(state->mutex); + const auto deadline = wall_start + duration; + while (!state->finished && std::chrono::steady_clock::now() < deadline) { + if (stop_token.stop_requested()) { + state->metrics.stop_requested = true; + break; + } + if (presentation_window && !IsWindow(presentation_window)) { + state->metrics.source_closed = true; + break; + } + if (presentation_window && + (!is_foreground_monitor_covering_window(presentation_window) || + MonitorFromWindow(presentation_window, MONITOR_DEFAULTTONULL) != + monitor)) { + state->metrics.presentation_changed = true; + break; + } + if (frame_stall_timeout.count() > 0 && + std::chrono::steady_clock::now() - state->last_frame_at >= + frame_stall_timeout) { + state->metrics.frame_stalled = true; + break; + } + state->changed.wait_for(lock, std::chrono::milliseconds(100), + [state] { return state->finished; }); + } + state->finished = true; + lock.unlock(); + + item.Closed(closed_token); + frame_pool.FrameArrived(frame_token); + stop_and_wait_for_capture_callbacks(state); + session.Close(); + frame_pool.Close(); + + std::scoped_lock final_lock(state->mutex); + finish_metrics(*state); + record_process_metrics(state->metrics, + std::chrono::steady_clock::now() - wall_start, + starting_cpu_ticks); + return state->metrics; +} + +VideoCaptureMetrics +capture_monitor_wgc_video(HMONITOR monitor, const std::chrono::seconds duration, + const std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler, + const std::stop_token stop_token, + const std::chrono::milliseconds frame_stall_timeout) { + return capture_monitor_wgc_video_impl( + monitor, nullptr, duration, requested_frames_per_second, + std::move(frame_handler), stop_token, frame_stall_timeout); +} + +VideoCaptureMetrics capture_monitor_covering_window_wgc_video( + HWND window, const std::chrono::seconds duration, + const std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler, const std::stop_token stop_token, + const std::chrono::milliseconds frame_stall_timeout) { + if (!IsWindow(window)) { + throw std::invalid_argument("The selected window no longer exists"); + } + const HMONITOR monitor = MonitorFromWindow(window, MONITOR_DEFAULTTONULL); + if (!monitor) { + throw std::runtime_error( + "The monitor-covering window is not attached to a monitor"); + } + return capture_monitor_wgc_video_impl( + monitor, window, duration, requested_frames_per_second, + std::move(frame_handler), stop_token, frame_stall_timeout); +} + +VideoCaptureMetrics capture_monitor_covering_window_dxgi_video( + HWND window, const std::chrono::seconds duration, + const std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler, const std::stop_token stop_token) { + if (!IsWindow(window)) { + throw std::invalid_argument("The selected window no longer exists"); + } + if (duration.count() <= 0 || requested_frames_per_second == 0) { + throw std::invalid_argument( + "Capture duration and frame rate must be positive"); + } + + const auto wall_start = std::chrono::steady_clock::now(); + const std::uint64_t starting_cpu_ticks = process_cpu_ticks(); + VideoCaptureMetrics metrics; + std::optional first_timestamp; + const auto frame_interval = + std::chrono::duration_cast( + std::chrono::duration(1.0 / requested_frames_per_second)); + auto next_frame_at = wall_start; + + try { + const HMONITOR monitor = MonitorFromWindow(window, MONITOR_DEFAULTTONULL); + if (!monitor) { + throw std::runtime_error( + "The monitor-covering window is not attached to a monitor"); + } + auto resources = create_duplication_resources(monitor); + const auto deadline = wall_start + duration; + + while (std::chrono::steady_clock::now() < deadline) { + if (stop_token.stop_requested()) { + metrics.stop_requested = true; + break; + } + if (!IsWindow(window)) { + metrics.source_closed = true; + break; + } + if (!is_foreground_monitor_covering_window(window) || + MonitorFromWindow(window, MONITOR_DEFAULTTONULL) != monitor) { + metrics.presentation_changed = true; + break; + } + + DXGI_OUTDUPL_FRAME_INFO frame_information{}; + winrt::com_ptr desktop_resource; + const HRESULT acquire_result = resources.duplication->AcquireNextFrame( + 100, &frame_information, desktop_resource.put()); + if (acquire_result == DXGI_ERROR_WAIT_TIMEOUT) { + continue; + } + if (acquire_result == DXGI_ERROR_ACCESS_LOST) { + metrics.frame_stalled = true; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + break; + } + winrt::check_hresult(acquire_result); + + try { + const auto now = std::chrono::steady_clock::now(); + if (now >= next_frame_at) { + const auto region = + window_region_on_output(window, resources.output_description); + if (!region) { + metrics.presentation_changed = true; + } else { + const auto texture = desktop_resource.as(); + D3D11_TEXTURE2D_DESC texture_description{}; + texture->GetDesc(&texture_description); + const std::int64_t timestamp_100ns = + std::chrono::duration_cast>>( + now.time_since_epoch()) + .count(); + auto frame = copy_texture_gpu( + *resources.d3d_device, *resources.d3d_context, *texture, + texture_description, timestamp_100ns, *region); + metrics.frames += 1; + metrics.width = frame.width; + metrics.height = frame.height; + metrics.last_timestamp_100ns = timestamp_100ns; + if (!first_timestamp) { + first_timestamp = timestamp_100ns; + metrics.first_timestamp_100ns = timestamp_100ns; + } + if (frame_handler) { + frame_handler(std::move(frame)); + } + next_frame_at = now + frame_interval; + } + } + } catch (...) { + static_cast(resources.duplication->ReleaseFrame()); + throw; + } + winrt::check_hresult(resources.duplication->ReleaseFrame()); + if (metrics.presentation_changed) { + break; + } + } + } catch (const winrt::hresult_error &error) { + if (error.code().value == + static_cast(DXGI_ERROR_ACCESS_LOST)) { + // Windows invalidates every duplication interface when the producer of + // the desktop image changes (for example, entering or leaving direct + // presentation). Return a recoverable stall so the publisher destroys + // these resources and recreates the interface for the new producer. + metrics.frame_stalled = true; + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } else { + metrics.error_code = error.code().value; + metrics.error = error.message().c_str(); + } + } catch (const std::exception &error) { + metrics.error = winrt::to_hstring(error.what()).c_str(); + } + + if (first_timestamp && metrics.frames >= 2 && + metrics.last_timestamp_100ns > *first_timestamp) { + const std::int64_t span = metrics.last_timestamp_100ns - *first_timestamp; + metrics.timestamp_span_seconds = static_cast(span) / 10'000'000.0; + metrics.observed_frames_per_second = + static_cast(metrics.frames - 1) / + metrics.timestamp_span_seconds; + } + record_process_metrics(metrics, std::chrono::steady_clock::now() - wall_start, + starting_cpu_ticks); + return metrics; +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/video_capture.h b/apps/desktop/native/windows-capture-probe/src/video_capture.h new file mode 100644 index 0000000000..754a363f4b --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/video_capture.h @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace chatto::capture { + +struct LiveCaptureStatus; + +struct VideoFrameData { + std::uint32_t width; + std::uint32_t height; + std::int64_t timestamp_100ns; + double gpu_copy_submit_duration_ms; + winrt::com_ptr bgra_texture; +}; + +using VideoFrameHandler = std::function; + +[[nodiscard]] std::pair +window_capture_size(HWND window); + +struct VideoCaptureMetrics { + std::uint64_t frames = 0; + std::uint64_t inferred_gaps = 0; + std::uint64_t resizes = 0; + std::uint64_t sampled_frames = 0; + std::uint64_t changed_samples = 0; + std::uint64_t black_samples = 0; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::int64_t first_timestamp_100ns = 0; + std::int64_t last_timestamp_100ns = 0; + double timestamp_span_seconds = 0; + double observed_frames_per_second = 0; + double longest_frame_interval_ms = 0; + double sampled_luminance_mean = 0; + std::uint8_t sampled_luminance_min = 0; + std::uint8_t sampled_luminance_max = 0; + double wall_duration_seconds = 0; + double process_cpu_seconds = 0; + double process_cpu_single_core_percent = 0; + std::uint64_t peak_working_set_bytes = 0; + bool source_closed = false; + bool frame_stalled = false; + bool presentation_changed = false; + bool stop_requested = false; + std::int32_t error_code = 0; + std::wstring error; +}; + +/** Whether the selected foreground window currently covers its monitor. */ +[[nodiscard]] bool is_foreground_monitor_covering_window(HWND window); + +[[nodiscard]] VideoCaptureMetrics capture_window_video( + HWND window, std::chrono::seconds duration, + std::uint32_t requested_frames_per_second, bool show_preview, + std::shared_ptr live_status = {}, + VideoFrameHandler frame_handler = {}, std::stop_token stop_token = {}, + std::chrono::milliseconds frame_stall_timeout = {}, + bool switch_on_monitor_covering_presentation = false); + +/** + * Capture a foreground monitor-covering window through monitor WGC. + * + * The call returns with `presentation_changed` when the window leaves that + * presentation mode, allowing the caller to resume privacy-preserving WGC. + */ +[[nodiscard]] VideoCaptureMetrics capture_monitor_covering_window_wgc_video( + HWND window, std::chrono::seconds duration, + std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler = {}, std::stop_token stop_token = {}, + std::chrono::milliseconds frame_stall_timeout = {}); + +/** Capture an explicitly selected monitor through Windows Graphics Capture. */ +[[nodiscard]] VideoCaptureMetrics +capture_monitor_wgc_video(HMONITOR monitor, std::chrono::seconds duration, + std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler = {}, + std::stop_token stop_token = {}, + std::chrono::milliseconds frame_stall_timeout = {}); + +/** Capture a foreground monitor-covering window through Desktop Duplication. */ +[[nodiscard]] VideoCaptureMetrics capture_monitor_covering_window_dxgi_video( + HWND window, std::chrono::seconds duration, + std::uint32_t requested_frames_per_second, + VideoFrameHandler frame_handler = {}, std::stop_token stop_token = {}); + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/video_frame_scaler.h b/apps/desktop/native/windows-capture-probe/src/video_frame_scaler.h new file mode 100644 index 0000000000..04e634062b --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/video_frame_scaler.h @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace chatto::capture { + +/** Scale a tightly packed BGRA frame to the track's stable output dimensions. + */ +[[nodiscard]] inline std::vector scale_bgra_frame( + std::vector pixels, const std::uint32_t source_width, + const std::uint32_t source_height, const std::uint32_t output_width, + const std::uint32_t output_height) { + const auto source_bytes = + static_cast(source_width) * source_height * 4; + if (source_width == 0 || source_height == 0 || output_width == 0 || + output_height == 0 || pixels.size() != source_bytes) { + throw std::invalid_argument("The BGRA frame dimensions are invalid"); + } + if (source_width == output_width && source_height == output_height) { + return pixels; + } + + std::vector output(static_cast(output_width) * + output_height * 4); + for (std::uint32_t y = 0; y < output_height; ++y) { + const auto source_y = static_cast( + static_cast(y) * source_height / output_height); + for (std::uint32_t x = 0; x < output_width; ++x) { + const auto source_x = static_cast( + static_cast(x) * source_width / output_width); + const auto source_offset = + (static_cast(source_y) * source_width + source_x) * 4; + const auto output_offset = + (static_cast(y) * output_width + x) * 4; + std::copy_n(pixels.data() + source_offset, 4, + output.data() + output_offset); + } + } + return output; +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/window_sources.cpp b/apps/desktop/native/windows-capture-probe/src/window_sources.cpp new file mode 100644 index 0000000000..2a5ccd197b --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/window_sources.cpp @@ -0,0 +1,259 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "window_sources.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace chatto::capture { +namespace { + +constexpr std::uint32_t kMinimumWidth = 320; +constexpr std::uint32_t kMinimumHeight = 180; + +[[nodiscard]] std::wstring window_title(HWND window) { + const int length = GetWindowTextLengthW(window); + if (length <= 0) { + return {}; + } + + std::wstring title(static_cast(length) + 1, L'\0'); + const int copied = GetWindowTextW(window, title.data(), length + 1); + if (copied <= 0) { + return {}; + } + title.resize(static_cast(copied)); + return title; +} + +[[nodiscard]] std::wstring process_image_path(std::uint32_t process_id) { + const HANDLE process = + OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, process_id); + if (process == nullptr) { + return {}; + } + + std::wstring path(32'768, L'\0'); + DWORD size = static_cast(path.size()); + const BOOL queried = + QueryFullProcessImageNameW(process, 0, path.data(), &size); + CloseHandle(process); + if (!queried || size == 0) { + return {}; + } + + path.resize(size); + return path; +} + +[[nodiscard]] std::wstring application_identifier(const std::wstring &path) { + if (path.empty()) { + return {}; + } + std::wstring normalized = path; + std::transform(normalized.begin(), normalized.end(), normalized.begin(), + towlower); + std::array digest{}; + BCRYPT_ALG_HANDLE algorithm = nullptr; + NTSTATUS status = BCryptOpenAlgorithmProvider( + &algorithm, BCRYPT_SHA256_ALGORITHM, nullptr, 0); + if (status < 0) { + return {}; + } + status = BCryptHash(algorithm, nullptr, 0, + reinterpret_cast(normalized.data()), + static_cast(normalized.size() * sizeof(wchar_t)), + digest.data(), static_cast(digest.size())); + BCryptCloseAlgorithmProvider(algorithm, 0); + if (status < 0) { + return {}; + } + std::wostringstream encoded; + encoded << L"windows-sha256:" << std::hex << std::setfill(L'0'); + for (const auto byte : digest) { + encoded << std::setw(2) << static_cast(byte); + } + return encoded.str(); +} + +[[nodiscard]] bool is_cloaked(HWND window) { + DWORD cloaked = 0; + return SUCCEEDED(DwmGetWindowAttribute(window, DWMWA_CLOAKED, &cloaked, + sizeof(cloaked))) && + cloaked != 0; +} + +[[nodiscard]] RECT frame_bounds(HWND window) { + RECT bounds{}; + if (SUCCEEDED(DwmGetWindowAttribute(window, DWMWA_EXTENDED_FRAME_BOUNDS, + &bounds, sizeof(bounds)))) { + return bounds; + } + + GetWindowRect(window, &bounds); + return bounds; +} + +BOOL CALLBACK collect_window(HWND window, LPARAM parameter) { + auto &sources = *reinterpret_cast *>(parameter); + const RECT bounds = frame_bounds(window); + const auto width = + static_cast(std::max(0L, bounds.right - bounds.left)); + const auto height = + static_cast(std::max(0L, bounds.bottom - bounds.top)); + + if (!is_candidate_window(IsWindowVisible(window) != FALSE, is_cloaked(window), + GetWindow(window, GW_OWNER) != nullptr, width, + height)) { + return TRUE; + } + + std::uint32_t process_id = 0; + GetWindowThreadProcessId(window, reinterpret_cast(&process_id)); + const std::wstring title = window_title(window); + const std::wstring image_path = process_image_path(process_id); + const std::wstring identifier = application_identifier(image_path); + if (process_id == 0 || title.empty() || image_path.empty() || + identifier.empty()) { + return TRUE; + } + + sources.push_back(WindowSource{ + .handle = window, + .process_id = process_id, + .application_name = + std::filesystem::path(image_path).filename().wstring(), + .application_identifier = identifier, + .title = title, + .width = width, + .height = height, + }); + return TRUE; +} + +BOOL CALLBACK collect_display(HMONITOR monitor, HDC, RECT *bounds, + LPARAM parameter) { + auto &sources = *reinterpret_cast *>(parameter); + MONITORINFO information{}; + information.cbSize = sizeof(information); + if (!GetMonitorInfoW(monitor, &information)) { + return TRUE; + } + const auto width = static_cast( + std::max(0L, bounds->right - bounds->left)); + const auto height = static_cast( + std::max(0L, bounds->bottom - bounds->top)); + if (width == 0 || height == 0) { + return TRUE; + } + sources.push_back({ + .handle = monitor, + .display_index = 0, + .is_main_display = + (information.dwFlags & MONITORINFOF_PRIMARY) != 0, + .width = width, + .height = height, + }); + return TRUE; +} + +} // namespace + +bool is_candidate_window(const bool visible, const bool cloaked, + const bool owned, const std::uint32_t width, + const std::uint32_t height) { + return visible && !cloaked && !owned && width >= kMinimumWidth && + height >= kMinimumHeight; +} + +std::vector enumerate_window_sources() { + std::vector sources; + EnumWindows(collect_window, reinterpret_cast(&sources)); + return sources; +} + +std::vector enumerate_display_sources() { + std::vector sources; + EnumDisplayMonitors(nullptr, nullptr, collect_display, + reinterpret_cast(&sources)); + std::stable_sort(sources.begin(), sources.end(), + [](const DisplaySource &left, const DisplaySource &right) { + return left.is_main_display && !right.is_main_display; + }); + for (std::size_t index = 0; index < sources.size(); ++index) { + sources[index].display_index = static_cast(index + 1); + } + return sources; +} + +bool is_display_capture_candidate(HMONITOR monitor) { + if (!monitor) { + return false; + } + MONITORINFO information{}; + information.cbSize = sizeof(information); + return GetMonitorInfoW(monitor, &information) != FALSE; +} + +bool is_window_capture_candidate(HWND window) { + if (!IsWindow(window)) { + return false; + } + const RECT bounds = frame_bounds(window); + const auto width = + static_cast(std::max(0L, bounds.right - bounds.left)); + const auto height = + static_cast(std::max(0L, bounds.bottom - bounds.top)); + return is_candidate_window( + IsWindowVisible(window) != FALSE, is_cloaked(window), + GetWindow(window, GW_OWNER) != nullptr, width, height); +} + +bool window_matches_application(HWND window, + const std::wstring &expected_identifier) { + if (!IsWindow(window) || expected_identifier.empty()) { + return false; + } + DWORD process_id = 0; + GetWindowThreadProcessId(window, &process_id); + return process_id != 0 && + application_identifier(process_image_path(process_id)) == + expected_identifier; +} + +std::optional +select_replacement_window_source(const std::vector &sources, + const std::wstring &expected_identifier, + const std::uint32_t preferred_process_id, + HWND stale_window) { + const WindowSource *selected = nullptr; + for (const auto &source : sources) { + if (source.handle == stale_window || + source.application_identifier != expected_identifier) { + continue; + } + if (selected == nullptr || + (source.process_id == preferred_process_id && + selected->process_id != preferred_process_id) || + (source.process_id == selected->process_id && + static_cast(source.width) * source.height > + static_cast(selected->width) * selected->height)) { + selected = &source; + } + } + if (selected == nullptr) { + return std::nullopt; + } + return *selected; +} + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/src/window_sources.h b/apps/desktop/native/windows-capture-probe/src/window_sources.h new file mode 100644 index 0000000000..323790b352 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/src/window_sources.h @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +#include + +namespace chatto::capture { + +struct WindowSource { + HWND handle; + std::uint32_t process_id; + std::wstring application_name; + std::wstring application_identifier; + std::wstring title; + std::uint32_t width; + std::uint32_t height; +}; + +struct DisplaySource { + HMONITOR handle; + std::uint32_t display_index; + bool is_main_display; + std::uint32_t width; + std::uint32_t height; +}; + +[[nodiscard]] bool is_candidate_window(bool visible, bool cloaked, bool owned, + std::uint32_t width, + std::uint32_t height); + +[[nodiscard]] std::vector enumerate_window_sources(); + +/** Enumerate active physical and virtual desktop monitors for display capture. */ +[[nodiscard]] std::vector enumerate_display_sources(); + +/** Whether a temporary monitor handle still identifies an active display. */ +[[nodiscard]] bool is_display_capture_candidate(HMONITOR monitor); + +/** Whether an existing HWND still satisfies the window-capture source policy. + */ +[[nodiscard]] bool is_window_capture_candidate(HWND window); + +[[nodiscard]] bool +window_matches_application(HWND window, + const std::wstring &expected_identifier); + +/** Choose the best replacement capture window without reusing a stale handle. + */ +[[nodiscard]] std::optional +select_replacement_window_source(const std::vector &sources, + const std::wstring &expected_identifier, + std::uint32_t preferred_process_id, + HWND stale_window); + +} // namespace chatto::capture diff --git a/apps/desktop/native/windows-capture-probe/tests/h264_encoder_test.cpp b/apps/desktop/native/windows-capture-probe/tests/h264_encoder_test.cpp new file mode 100644 index 0000000000..5c6022e8af --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/tests/h264_encoder_test.cpp @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include "h264_encoder.h" + +#include +#include +#include +#include +#include +#include + +using chatto::capture::bgra_to_nv12; +using chatto::capture::h264_access_unit_is_annex_b; +using chatto::capture::h264_access_unit_is_key_frame; +using chatto::capture::h264_access_unit_profile_level; + +int main(const int argument_count, char **arguments) { + const std::vector black_bgra(2 * 2 * 4, 0); + const auto black_nv12 = bgra_to_nv12(black_bgra, 2, 2); + assert(black_nv12.size() == 6); + assert(black_nv12[0] == 16); + assert(black_nv12[1] == 16); + assert(black_nv12[2] == 16); + assert(black_nv12[3] == 16); + assert(black_nv12[4] == 128); + assert(black_nv12[5] == 128); + + bool rejected_odd_dimensions = false; + try { + static_cast(bgra_to_nv12(black_bgra, 1, 4)); + } catch (const std::invalid_argument &) { + rejected_odd_dimensions = true; + } + assert(rejected_odd_dimensions); + + const std::vector idr{ + 0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xe0, + 0x1f, 0x00, 0x00, 0x01, 0x65, 0xaa, + }; + assert(h264_access_unit_is_key_frame(idr)); + assert(h264_access_unit_is_annex_b(idr)); + const auto profile_level = h264_access_unit_profile_level(idr); + assert(profile_level.has_value()); + assert(profile_level->profile_idc == 0x42); + assert(profile_level->profile_iop == 0xe0); + assert(profile_level->level_idc == 0x1f); + + const std::vector delta{ + 0x00, 0x00, 0x00, 0x01, 0x41, 0xaa, + }; + assert(!h264_access_unit_is_key_frame(delta)); + assert(h264_access_unit_is_annex_b(delta)); + + if (argument_count > 1 && std::string_view(arguments[1]) == "--hardware") { + try { + constexpr std::uint32_t width = 1280; + constexpr std::uint32_t height = 720; + auto encoder = chatto::capture::create_hardware_h264_encoder( + width, height, 60, 8'000'000); + std::vector frame( + static_cast(width) * height * 4, 0); + std::size_t access_units = 0; + std::size_t key_frames = 0; + std::size_t annex_b_access_units = 0; + std::optional hardware_profile_level; + for (std::int64_t index = 0; index < 10; ++index) { + if (index == 5) { + encoder->set_target_bitrate(4'000'000); + assert(encoder->target_bitrate_bps() == 4'000'000); + } + for (auto &access_unit : + encoder->encode(frame, index * 1'000'000 / 60, index == 0)) { + ++access_units; + key_frames += access_unit.key_frame ? 1U : 0U; + annex_b_access_units += + h264_access_unit_is_annex_b(access_unit.data) ? 1U : 0U; + if (!hardware_profile_level.has_value()) { + hardware_profile_level = + h264_access_unit_profile_level(access_unit.data); + } + } + } + for (auto &access_unit : encoder->finish()) { + ++access_units; + key_frames += access_unit.key_frame ? 1U : 0U; + annex_b_access_units += + h264_access_unit_is_annex_b(access_unit.data) ? 1U : 0U; + if (!hardware_profile_level.has_value()) { + hardware_profile_level = + h264_access_unit_profile_level(access_unit.data); + } + } + std::cout << "hardware_encoder=" << encoder->implementation_name() + << " access_units=" << access_units + << " key_frames=" << key_frames + << " annex_b_access_units=" << annex_b_access_units; + if (hardware_profile_level.has_value()) { + std::cout << " profile_idc=" + << static_cast(hardware_profile_level->profile_idc) + << " profile_iop=" + << static_cast(hardware_profile_level->profile_iop) + << " level_idc=" + << static_cast(hardware_profile_level->level_idc); + } + std::cout << '\n'; + assert(access_units > 0); + assert(key_frames > 0); + assert(annex_b_access_units == access_units); + } catch (const std::exception &error) { + std::cerr << "hardware encoder smoke test failed: " << error.what() + << '\n'; + return 1; + } + } + return 0; +} diff --git a/apps/desktop/native/windows-capture-probe/tests/latest_frame_queue_test.cpp b/apps/desktop/native/windows-capture-probe/tests/latest_frame_queue_test.cpp new file mode 100644 index 0000000000..fad63e86e5 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/tests/latest_frame_queue_test.cpp @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include "../src/latest_frame_queue.h" +#include "../src/video_frame_scaler.h" + +namespace { + +void expect(const bool condition, const char *message) { + if (!condition) { + std::cerr << "FAILED: " << message << "\n"; + std::exit(1); + } +} + +} // namespace + +int main() { + chatto::capture::LatestFrameQueue queue; + + expect(!queue.push(1), "the first frame is accepted without a drop"); + expect(queue.push(2), "a pending frame is replaced"); + const auto latest = queue.wait_pop(); + expect(latest && *latest == 2, "the consumer receives only the latest frame"); + + expect(!queue.push(3), "the empty queue accepts another frame"); + queue.close(); + const auto final = queue.wait_pop(); + expect(final && *final == 3, "close drains the final pending frame"); + expect(!queue.wait_pop(), "a closed drained queue finishes the consumer"); + expect(!queue.push(4), "a closed queue rejects later frames"); + + std::vector resized_source(4U * 2U * 4U); + for (std::size_t pixel = 0; pixel < resized_source.size() / 4; ++pixel) { + resized_source[pixel * 4] = static_cast(pixel); + } + const auto resized = + chatto::capture::scale_bgra_frame(std::move(resized_source), 4, 2, 2, 1); + expect(resized.size() == 2U * 4U, + "a resized source produces the stable track dimensions"); + expect(resized[0] == 0 && resized[4] == 2, + "scaling samples across the complete resized source frame"); + + std::cout << "latest frame queue tests passed\n"; + return 0; +} diff --git a/apps/desktop/native/windows-capture-probe/tests/window_sources_test.cpp b/apps/desktop/native/windows-capture-probe/tests/window_sources_test.cpp new file mode 100644 index 0000000000..51604c3347 --- /dev/null +++ b/apps/desktop/native/windows-capture-probe/tests/window_sources_test.cpp @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include "../src/window_sources.h" + +namespace { + +void expect(const bool condition, const char *message) { + if (!condition) { + std::cerr << "FAILED: " << message << "\n"; + std::exit(1); + } +} + +} // namespace + +int main() { + using chatto::capture::is_candidate_window; + + expect(is_candidate_window(true, false, false, 320, 180), + "minimum visible top-level window is included"); + expect(!is_candidate_window(false, false, false, 1920, 1080), + "hidden window is excluded"); + expect(!is_candidate_window(true, true, false, 1920, 1080), + "cloaked window is excluded"); + expect(!is_candidate_window(true, false, true, 1920, 1080), + "owned window is excluded"); + expect(!is_candidate_window(true, false, false, 319, 1080), + "narrow implementation window is excluded"); + expect(!is_candidate_window(true, false, false, 1920, 179), + "short implementation window is excluded"); + + const std::wstring expected_identifier = L"windows-sha256:game"; + const auto first_handle = reinterpret_cast(std::uintptr_t{1}); + const auto stale_handle = reinterpret_cast(std::uintptr_t{2}); + const auto preferred_handle = reinterpret_cast(std::uintptr_t{3}); + const std::vector sources{ + {.handle = first_handle, + .process_id = 7, + .application_identifier = expected_identifier, + .width = 1920, + .height = 1080}, + {.handle = stale_handle, + .process_id = 42, + .application_identifier = expected_identifier, + .width = 3840, + .height = 2160}, + {.handle = preferred_handle, + .process_id = 42, + .application_identifier = expected_identifier, + .width = 1280, + .height = 720}, + }; + const auto replacement = chatto::capture::select_replacement_window_source( + sources, expected_identifier, 42, stale_handle); + expect(replacement && replacement->handle == preferred_handle, + "replacement prefers the original process and excludes stale handle"); + + std::cout << "window source tests passed\n"; + return 0; +} diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 84393ba65d..41bd5db719 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -7,7 +7,7 @@ "main": "main.mjs", "scripts": { "build": "install-electron && node scripts/build.mjs", - "check": "node --check main.mjs && node --check preload.cjs && node --check frontend_protocol.mjs && node --check game_capture.mjs && node --check scripts/build.mjs && node --check scripts/macos-capture-helper.mjs && node --test", + "check": "node --check main.mjs && node --check preload.cjs && node --check frontend_protocol.mjs && node --check game_capture.mjs && node --check scripts/build.mjs && node --check scripts/macos-capture-helper.mjs && node --check scripts/windows-capture-helper.mjs && node --test", "dev": "electron ." }, "devDependencies": { diff --git a/apps/desktop/scripts/build.mjs b/apps/desktop/scripts/build.mjs index 360f9dea17..14b67e3a34 100644 --- a/apps/desktop/scripts/build.mjs +++ b/apps/desktop/scripts/build.mjs @@ -6,6 +6,7 @@ import packageJson from "../package.json" with { type: "json" }; import { pruneElectronLocales } from "./locales.mjs"; import { embedMacOSCaptureHelper } from "./macos-capture-helper.mjs"; import { macOSVersions, releaseBuildVersion } from "./version.mjs"; +import { embedWindowsCaptureHelper } from "./windows-capture-helper.mjs"; const desktopRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -17,7 +18,7 @@ const packagerOut = path.join(distRoot, ".packager"); const platform = process.platform; const electronChecksum = process.env.CHATTO_ELECTRON_CHECKSUM; const electronArchiveName = `electron-v${packageJson.devDependencies.electron}-${platform}-${process.arch}.zip`; -const embedCaptureHelper = platform === "darwin"; +const embedCaptureHelper = platform === "darwin" || platform === "win32"; const macOSSignIdentity = platform === "darwin" ? (process.env.CHATTO_MACOS_SIGN_IDENTITY ?? "-") @@ -72,8 +73,12 @@ const [bundleRoot] = await packager({ afterPrune: embedCaptureHelper ? [ async ({ buildPath }) => { - const appBundle = path.resolve(buildPath, "../../.."); - await embedMacOSCaptureHelper(appBundle, macVersions); + if (platform === "darwin") { + const appBundle = path.resolve(buildPath, "../../.."); + await embedMacOSCaptureHelper(appBundle, macVersions); + } else { + await embedWindowsCaptureHelper(path.dirname(buildPath)); + } }, ] : undefined, diff --git a/apps/desktop/scripts/windows-capture-helper.mjs b/apps/desktop/scripts/windows-capture-helper.mjs new file mode 100644 index 0000000000..18fb1353ea --- /dev/null +++ b/apps/desktop/scripts/windows-capture-helper.mjs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: 2026 ChattoCorp GmbH +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { copyFile, mkdir, rm } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const windowsCaptureHelperExecutable = + "chatto-windows-capture-probe.exe"; + +const scriptsRoot = path.dirname(fileURLToPath(import.meta.url)); +const probeRoot = path.resolve(scriptsRoot, "../native/windows-capture-probe"); +const visualCppRuntimeFiles = [ + "msvcp140.dll", + "msvcp140_atomic_wait.dll", + "vcruntime140.dll", + "vcruntime140_1.dll", +]; + +/** Build and embed the pinned Windows capture helper and its LiveKit runtime. */ +export async function embedWindowsCaptureHelper(resourcesDirectory) { + if (process.platform !== "win32" || process.arch !== "x64") { + throw new Error("The Windows capture helper requires Windows x64."); + } + const buildDirectory = path.join(probeRoot, "build-package"); + const cmake = findCmakeExecutable(); + execFileSync( + cmake, + ["-S", probeRoot, "-B", buildDirectory, "-A", "x64"], + { stdio: "inherit" }, + ); + execFileSync( + cmake, + [ + "--build", + buildDirectory, + "--config", + "Release", + "--target", + windowsCaptureHelperExecutable.replace(/\.exe$/, ""), + ], + { stdio: "inherit" }, + ); + + const destination = path.join(resourcesDirectory, "windows-capture"); + await rm(destination, { recursive: true, force: true }); + await mkdir(destination, { recursive: true }); + for (const file of [ + windowsCaptureHelperExecutable, + "livekit.dll", + "livekit_ffi.dll", + ]) { + await copyFile( + path.join(buildDirectory, "Release", file), + path.join(destination, file), + ); + } + const runtimeDirectory = findVisualCppRuntimeDirectory(); + for (const file of visualCppRuntimeFiles) { + await copyFile( + path.join(runtimeDirectory, file), + path.join(destination, file), + ); + } + return destination; +} + +function findCmakeExecutable() { + if (process.env.CMAKE) { + return process.env.CMAKE; + } + try { + const matches = execFileSync("where.exe", ["cmake"], { + encoding: "utf8", + windowsHide: true, + }) + .split(/\r?\n/) + .filter(Boolean); + if (matches.length > 0) { + return matches[0]; + } + } catch { + // Fall through to the CMake bundled with Visual Studio Build Tools. + } + return path.join( + findVisualStudioInstallation(), + "Common7", + "IDE", + "CommonExtensions", + "Microsoft", + "CMake", + "CMake", + "bin", + "cmake.exe", + ); +} + +function findVisualCppRuntimeDirectory() { + const vswhere = findVisualStudioInstaller(); + const matches = execFileSync( + vswhere, + [ + "-latest", + "-products", + "*", + "-find", + "VC\\Redist\\MSVC\\**\\x64\\Microsoft.VC143.CRT\\msvcp140.dll", + ], + { encoding: "utf8" }, + ) + .split(/\r?\n/) + .filter((entry) => entry && !entry.toLowerCase().includes("\\onecore\\")); + if (matches.length === 0) { + throw new Error("The Visual C++ x64 redistributable files are unavailable."); + } + return path.dirname(matches[0]); +} + +function findVisualStudioInstallation() { + const installation = execFileSync( + findVisualStudioInstaller(), + [ + "-latest", + "-products", + "*", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + ], + { encoding: "utf8" }, + ).trim(); + if (!installation) { + throw new Error("A Visual Studio installation with C++ tools is required."); + } + return installation; +} + +function findVisualStudioInstaller() { + const programFilesX86 = process.env["ProgramFiles(x86)"]; + if (!programFilesX86) { + throw new Error("The Visual Studio installer directory is unavailable."); + } + return path.join( + programFilesX86, + "Microsoft Visual Studio", + "Installer", + "vswhere.exe", + ); +} diff --git a/apps/docs-website/src/content/docs/guides/infrastructure/voice-calls.mdx b/apps/docs-website/src/content/docs/guides/infrastructure/voice-calls.mdx index a9afb6cc95..d0db1d2681 100644 --- a/apps/docs-website/src/content/docs/guides/infrastructure/voice-calls.mdx +++ b/apps/docs-website/src/content/docs/guides/infrastructure/voice-calls.mdx @@ -21,13 +21,17 @@ The first person to select **Start call** and join creates the call and a fresh Screen sharing can include audio from a browser tab when the browser supports it. In Chrome, select a browser tab and enable **Share tab audio** in the browser picker. Chatto does not request whole-system audio because that could capture remote call playback and feed it back into the room. Chatto Desktop uses the same screen-share button but provides its own chooser -on macOS 15 and newer. It shows previews of application windows and complete -displays. Application-window sharing includes that application's audio; -complete-display sharing is video-only so call audio cannot be captured and -echoed back to other participants. The first use asks for macOS **Screen & -System Audio Recording** permission and may require Chatto Desktop to be -restarted after access is granted. In ordinary web browsers, the button keeps -using the browser's window/tab/screen chooser. +on macOS 15 and newer and supported Windows x64 systems. It shows previews of +application windows and also offers complete displays. Application-window +sharing includes that application's audio. Complete-display sharing is +video-only so call audio cannot be captured and echoed back to other +participants. On Windows, application capture follows the selected game into +monitor-covering borderless presentation. The sender's Windows preview is a +local mirror of the encoded share, so it does not consume LiveKit downlink or +depend on a network round trip. On macOS, the first use asks for **Screen & System Audio +Recording** permission and may require Chatto Desktop to be restarted after +access is granted. In ordinary web browsers, the button keeps using the +browser's window/tab/screen chooser. Chatto records call start/join/leave/end facts durably and reconciles active participants against LiveKit webhooks, so call indicators recover after server restarts. These call facts update the active call UI without appearing as room timeline messages. LiveKit media is end-to-end encrypted by current Chatto clients; older clients that do not support this call encryption cannot decode encrypted media. diff --git a/apps/frontend/src/lib/components/voice/NativeScreenSharePreview.svelte b/apps/frontend/src/lib/components/voice/NativeScreenSharePreview.svelte new file mode 100644 index 0000000000..3120d88379 --- /dev/null +++ b/apps/frontend/src/lib/components/voice/NativeScreenSharePreview.svelte @@ -0,0 +1,125 @@ + + + +
+ +
diff --git a/apps/frontend/src/lib/components/voice/ScreenShareControlButton.svelte b/apps/frontend/src/lib/components/voice/ScreenShareControlButton.svelte index 71ec77c5cb..8170837513 100644 --- a/apps/frontend/src/lib/components/voice/ScreenShareControlButton.svelte +++ b/apps/frontend/src/lib/components/voice/ScreenShareControlButton.svelte @@ -106,8 +106,9 @@ to the browser's `getDisplayMedia` picker through LiveKit. : m('voice.display_number', { number: source.displayIndex }); try { await voiceCallState.startNativeScreenShare(source.id, sourceName); - } catch { + } catch (error) { if (!serverScope.isCurrent()) return; + console.error('Failed to start native screen sharing:', error); toast.error(m('voice.screen_share_failed')); } } diff --git a/apps/frontend/src/lib/components/voice/VideoThumbnail.svelte b/apps/frontend/src/lib/components/voice/VideoThumbnail.svelte index 5184dc013d..0b40a5fded 100644 --- a/apps/frontend/src/lib/components/voice/VideoThumbnail.svelte +++ b/apps/frontend/src/lib/components/voice/VideoThumbnail.svelte @@ -19,10 +19,11 @@ resolution to request for sidebar-width tiles. - `showIdentityOverlay` - Whether to show the avatar overlay - `fit` - How the video track should fit the tile. Camera thumbnails default to `cover`; screen shares should use `contain` to avoid cropping shared content. - `fill` - Whether the video should fill its parent's height instead of using thumbnail aspect-ratio sizing. +- `preferFullQuality` - Render a single-layer screen share without adaptive-stream tile sizing. --> diff --git a/apps/frontend/src/lib/components/voice/VoiceCallPanel.svelte b/apps/frontend/src/lib/components/voice/VoiceCallPanel.svelte index d7caa7bff4..846383a7ff 100644 --- a/apps/frontend/src/lib/components/voice/VoiceCallPanel.svelte +++ b/apps/frontend/src/lib/components/voice/VoiceCallPanel.svelte @@ -26,6 +26,7 @@ Room sidebar panel for voice/video calls. import UserAvatar from '$lib/components/UserAvatar.svelte'; import VideoThumbnail from './VideoThumbnail.svelte'; + import NativeScreenSharePreview from './NativeScreenSharePreview.svelte'; import AudioDeviceMenu from './AudioDeviceMenu.svelte'; import VoiceCallControlButton from './VoiceCallControlButton.svelte'; import ScreenShareControlButton from './ScreenShareControlButton.svelte'; @@ -35,6 +36,7 @@ Room sidebar panel for voice/video calls. import { getVoiceCallJoinErrorMessage } from '$lib/state/server/voiceCall.svelte'; import type { Track } from 'livekit-client'; import type { Attachment } from 'svelte/attachments'; + import type { NativeScreenSharePreview as NativeScreenSharePreviewSource } from '$lib/desktop/nativeScreenSharePublisher'; import { startDMWith } from '$lib/dm/startDM'; import { toast } from '$lib/ui/toast'; @@ -74,6 +76,8 @@ Room sidebar panel for voice/video calls. videoTrack: Track | null; isScreenShareEnabled: boolean; screenShareTrack: Track | null; + nativeScreenSharePreview: NativeScreenSharePreviewSource | null; + screenShareSimulcasted: boolean; }; let participants: DisplayParticipant[] = $derived.by(() => { @@ -95,7 +99,9 @@ Room sidebar panel for voice/video calls. isCameraEnabled: p.isCameraEnabled, videoTrack: p.videoTrack, isScreenShareEnabled: p.isScreenShareEnabled, - screenShareTrack: p.screenShareTrack + screenShareTrack: p.screenShareTrack, + nativeScreenSharePreview: p.nativeScreenSharePreview, + screenShareSimulcasted: p.screenShareSimulcasted })); } @@ -116,7 +122,9 @@ Room sidebar panel for voice/video calls. isCameraEnabled: false, videoTrack: null, isScreenShareEnabled: false, - screenShareTrack: null + screenShareTrack: null, + nativeScreenSharePreview: null, + screenShareSimulcasted: false })); }); @@ -128,7 +136,9 @@ Room sidebar panel for voice/video calls. }) ); let screenShareParticipants = $derived( - sortedParticipants.filter((p) => p.isScreenShareEnabled && p.screenShareTrack) + sortedParticipants.filter( + (p) => p.isScreenShareEnabled && (p.screenShareTrack || p.nativeScreenSharePreview) + ) ); let videoParticipants = $derived( sortedParticipants.filter((p) => p.isCameraEnabled && p.videoTrack) @@ -183,7 +193,10 @@ Room sidebar panel for voice/video calls. } function hasScreenShare(participant: DisplayParticipant) { - return participant.isScreenShareEnabled && participant.screenShareTrack; + return ( + participant.isScreenShareEnabled && + (participant.screenShareTrack || participant.nativeScreenSharePreview) + ); } function hasConnectionWarning(participant: DisplayParticipant) { @@ -492,13 +505,21 @@ Room sidebar panel for voice/video calls. class={callTileMediaButtonClass} onclick={(e) => showUserMenu(participant, e)} > - + {#if participant.nativeScreenSharePreview} + + {:else} + + {/if} {/snippet} @@ -535,14 +556,23 @@ Room sidebar panel for voice/video calls. onclick={(e) => showUserMenu(participant, e)} > {#if isScreen} - + {#if participant.nativeScreenSharePreview} + + {:else} + + {/if} {:else if isVideo} { hostChannel.port1.onmessage = (event) => { if (event.data?.kind !== 'stop') return; stopped(); + hostChannel.port1.postMessage({ kind: 'stopping' }); hostChannel.port1.postMessage({ kind: 'ended' }); }; hostChannel.port1.start(); @@ -30,7 +31,8 @@ describe('NativeScreenSharePublisherSession', () => { kind: 'started', width: 1920, height: 1080, - frameRate: 60 + frameRate: 60, + localPreviewAvailable: true }); }); return 'publisher-1'; @@ -44,6 +46,20 @@ describe('NativeScreenSharePublisherSession', () => { token: 'publisher-token', e2eeKey: 'shared-e2ee-key' }); + const previewFrame = vi.fn(); + session.preview!.subscribe(previewFrame); + hostChannel.port1.postMessage({ + kind: 'preview-frame', + timestampUs: 123_456, + keyFrame: true, + data: Uint8Array.from([0, 0, 0, 1, 0x65]) + }); + expect(session.preview).toMatchObject({ width: 1920, height: 1080, frameRate: 60 }); + await vi.waitFor(() => + expect(previewFrame).toHaveBeenCalledWith( + expect.objectContaining({ timestampUs: 123_456, keyFrame: true }) + ) + ); await session.stop(); diff --git a/apps/frontend/src/lib/desktop/nativeScreenSharePublisher.ts b/apps/frontend/src/lib/desktop/nativeScreenSharePublisher.ts index d096a5fbce..f7586bfcf0 100644 --- a/apps/frontend/src/lib/desktop/nativeScreenSharePublisher.ts +++ b/apps/frontend/src/lib/desktop/nativeScreenSharePublisher.ts @@ -8,29 +8,136 @@ const publisherStartTimeoutMs = 20_000; const publisherStopTimeoutMs = 5_000; type PublisherMessage = - | { kind: 'started'; width: number; height: number; frameRate: number } + | { + kind: 'started'; + width: number; + height: number; + frameRate: number; + localPreviewAvailable?: boolean; + } + | { kind: 'preview-frame'; timestampUs: number; keyFrame: boolean; data: Uint8Array } + | { + kind: 'metrics'; + submittedFrames: number; + publishedFrames: number; + droppedFrames: number; + captureFps: number; + publishFps: number; + averageReadbackMs: number; + averageScaleMs: number; + averagePublishMs: number; + averageHardwareEncodeMs: number; + averageGpuCopySubmitMs: number; + averageGpuConversionSubmitMs: number; + averageEncoderSubmitMs: number; + averageBitstreamWaitMs: number; + hardwareEncoderImplementation: string; + requestedEncoderBitrate: number; + appliedEncoderBitrate: number; + actualHardwareBitrate: number; + encoderRateControlMode: number; + requestedEncoderFps: number; + hardwareEncodedFrames: number; + hardwareEncodedBytes: number; + hardwareKeyFrames: number; + hardwareEncodedWidth: number; + hardwareEncodedHeight: number; + encoderResolutionChanges: number; + lastPublishMs: number; + sourceWidth: number; + sourceHeight: number; + dimensionChanges: number; + captureBackend: 'wgc-window' | 'wgc-monitor' | 'dxgi-display'; + rtcStatsAvailable: boolean; + outboundStreams: number; + activeOutboundStreams: number; + minimumActiveOutboundFps: number; + maximumActiveOutboundFps: number; + framesEncoded: number; + framesSent: number; + bytesSent: number; + retransmittedPacketsSent: number; + retransmittedBytesSent: number; + nackCount: number; + pliCount: number; + targetBitrate: number; + averageEncodeMs: number; + encodedWidth: number; + encodedHeight: number; + averageQp: number; + encoderImplementation: string; + cpuLimitedStreams: number; + bandwidthLimitedStreams: number; + powerEfficientStreams: number; + remoteInboundStatsAvailable: boolean; + remotePacketsLost: number; + remoteJitterSeconds: number; + remoteFractionLost: number; + remoteRoundTripTimeMs: number; + candidatePairStatsAvailable: boolean; + availableOutgoingBitrate: number; + currentRoundTripTimeMs: number; + packetsDiscardedOnSend: number; + bytesDiscardedOnSend: number; + } + | { kind: 'stopping' } | { kind: 'error'; message?: string } | { kind: 'ended' }; +export type NativeScreenSharePreviewFrame = { + timestampUs: number; + keyFrame: boolean; + data: Uint8Array; +}; + +export type NativeScreenSharePreview = { + width: number; + height: number; + frameRate: number; + subscribe(listener: (frame: NativeScreenSharePreviewFrame) => void): () => void; +}; + /** Control handle for a native helper that publishes screen-share media to LiveKit. */ export class NativeScreenSharePublisherSession { onEnded: ((error?: Error) => void) | null = null; readonly #port: MessagePort; + readonly preview: NativeScreenSharePreview | null; + readonly #previewListeners = new Set<(frame: NativeScreenSharePreviewFrame) => void>(); #finished = false; #stopPromise: Promise | null = null; #resolveStop: (() => void) | null = null; #rejectStop: ((error: Error) => void) | null = null; #stopTimeout: number | null = null; - private constructor(port: MessagePort) { + private constructor( + port: MessagePort, + started: Extract + ) { this.#port = port; + this.preview = started.localPreviewAvailable + ? { + width: started.width, + height: started.height, + frameRate: started.frameRate, + subscribe: (listener) => { + this.#previewListeners.add(listener); + return () => this.#previewListeners.delete(listener); + } + } + : null; port.onmessage = (event: MessageEvent) => { const message = event.data; if (message.kind === 'error') { this.finish(new Error(message.message || 'Native screen sharing stopped unexpectedly.')); } else if (message.kind === 'ended') { this.finish(); + } else if (message.kind === 'metrics') { + console.info('[Chatto Desktop] Native screen-share publisher metrics', message); + } else if (message.kind === 'stopping') { + console.info('[Chatto Desktop] Native screen-share publisher stopping'); + } else if (message.kind === 'preview-frame') { + for (const listener of this.#previewListeners) listener(message); } }; port.onmessageerror = () => @@ -43,8 +150,8 @@ export class NativeScreenSharePublisherSession { request: NativeScreenSharePublisherRequest ): Promise { const port = await requestPublisherPort(request); - await waitForPublisherStarted(port); - return new NativeScreenSharePublisherSession(port); + const started = await waitForPublisherStarted(port); + return new NativeScreenSharePublisherSession(port, started); } /** Ask Desktop to stop publishing and wait for the helper to exit. */ @@ -68,6 +175,7 @@ export class NativeScreenSharePublisherSession { this.#finished = true; if (this.#stopTimeout !== null) window.clearTimeout(this.#stopTimeout); this.#port.close(); + this.#previewListeners.clear(); if (this.#stopPromise) { if (error) this.#rejectStop?.(error); else this.#resolveStop?.(); @@ -117,7 +225,9 @@ function requestPublisherPort(request: NativeScreenSharePublisherRequest): Promi }); } -function waitForPublisherStarted(port: MessagePort): Promise { +function waitForPublisherStarted( + port: MessagePort +): Promise> { return new Promise((resolve, reject) => { const timeout = window.setTimeout(() => { cleanup(); @@ -128,7 +238,7 @@ function waitForPublisherStarted(port: MessagePort): Promise { const message = event.data; if (message.kind === 'started') { cleanup(); - resolve(); + resolve(message); } else if (message.kind === 'error') { cleanup(); port.close(); diff --git a/apps/frontend/src/lib/state/server/voiceCall.svelte.spec.ts b/apps/frontend/src/lib/state/server/voiceCall.svelte.spec.ts index 5b5f206568..7fb6dc569b 100644 --- a/apps/frontend/src/lib/state/server/voiceCall.svelte.spec.ts +++ b/apps/frontend/src/lib/state/server/voiceCall.svelte.spec.ts @@ -584,8 +584,10 @@ describe('VoiceCallState', () => { }); it('publishes camera and one native screen share under the local participant', async () => { + const preview = { width: 1920, height: 1080, frameRate: 60, subscribe: vi.fn() }; const session = { stop: vi.fn(), + preview, onEnded: null as ((error?: Error) => void) | null }; gameCaptureMocks.start.mockResolvedValue(session); @@ -606,7 +608,9 @@ describe('VoiceCallState', () => { expect(state.nativeScreenShareSourceName).toBe('Moonring'); expect(state.participants[0]).toMatchObject({ isCameraEnabled: true, - isScreenShareEnabled: true + isScreenShareEnabled: true, + screenShareTrack: null, + nativeScreenSharePreview: preview }); await state.toggleScreenShare(); @@ -1174,7 +1178,8 @@ describe('VoiceCallState', () => { expect(state.participants[1]).toMatchObject({ login: 'remote', isScreenShareEnabled: true, - screenShareTrack: gameVideoTrack + screenShareTrack: gameVideoTrack, + screenShareSimulcasted: false }); state.toggleParticipantLocalMute('remote-user'); @@ -1199,10 +1204,38 @@ describe('VoiceCallState', () => { await state.join('wss://livekit.example.test', 'R1'); const gameAudio = { kind: 'audio', attach: vi.fn(), detach: vi.fn() }; - roomEventHandlers.get('TrackSubscribed')?.(gameAudio, {}, companion); + const publication = { setSubscribed: vi.fn() }; + roomEventHandlers.get('TrackSubscribed')?.(gameAudio, publication, companion); expect(gameAudio.attach).not.toHaveBeenCalled(); + expect(publication.setSubscribed).toHaveBeenCalledWith(false); expect(companion.setVolume).toHaveBeenCalledWith(0, 'microphone'); expect(companion.setVolume).toHaveBeenCalledWith(0, 'screen_share_audio'); }); + + it('requests full frame rate for a non-simulcast native screen share', async () => { + const companion = { + identity: 'publisher-1', + name: 'Remote User', + metadata: '{"publisherKind":"game_share","ownerIdentity":"remote-user"}', + connectionQuality: 'good', + isSpeaking: false, + audioLevel: 0, + setVolume: vi.fn(), + trackPublications: new Map(), + getTrackPublications: vi.fn(() => []) + }; + mockRemoteParticipants.set('publisher-1', companion); + const state = new VoiceCallState(createVoiceCallClient()); + await state.join('wss://livekit.example.test', 'R1'); + const gameVideo = { kind: 'video', source: 'screen_share' }; + const publication = { + simulcasted: false, + setVideoFPS: vi.fn() + }; + + roomEventHandlers.get('TrackSubscribed')?.(gameVideo, publication, companion); + + expect(publication.setVideoFPS).toHaveBeenCalledWith(60); + }); }); diff --git a/apps/frontend/src/lib/state/server/voiceCall.svelte.ts b/apps/frontend/src/lib/state/server/voiceCall.svelte.ts index 0a6aa4f51e..f63c0e9ca3 100644 --- a/apps/frontend/src/lib/state/server/voiceCall.svelte.ts +++ b/apps/frontend/src/lib/state/server/voiceCall.svelte.ts @@ -19,6 +19,7 @@ import { playCallSound } from '$lib/audio/callSounds'; import { m } from '$lib/i18n/messages'; import type { VoiceCallAPI } from '$lib/api-client/voiceCalls'; import { NativeScreenSharePublisherSession } from '$lib/desktop/nativeScreenSharePublisher'; +import type { NativeScreenSharePreview } from '$lib/desktop/nativeScreenSharePublisher'; export type CallParticipantInfo = { identity: string; @@ -32,6 +33,8 @@ export type CallParticipantInfo = { videoTrack: Track | null; isScreenShareEnabled: boolean; screenShareTrack: Track | null; + nativeScreenSharePreview: NativeScreenSharePreview | null; + screenShareSimulcasted: boolean; isLocallyMuted: boolean; }; @@ -55,6 +58,8 @@ type LiveKitModule = typeof import('livekit-client'); const RECENTLY_DISCONNECTED_CALL_SOUND_MS = 5_000; const MEDIA_DEVICE_TOAST_DEDUPLICATION_MS = 1_500; +const NATIVE_SCREEN_SHARE_STATS_INTERVAL_MS = 2_000; +const NATIVE_SCREEN_SHARE_MAX_RECEIVE_FPS = 60; let liveKitModule: LiveKitModule | null = null; let liveKitModulePromise: Promise | null = null; @@ -240,6 +245,12 @@ export class VoiceCallState { private nativeScreenShareSession: NativeScreenSharePublisherSession | null = null; private e2eeWorker: Worker | null = null; private audioLevelInterval: ReturnType | null = null; + private nativeScreenShareStatsInterval: ReturnType | null = null; + private nativeScreenShareStatsReportInFlight = false; + private nativeScreenSharePreviousStats = new Map< + string, + { timestamp: number; framesReceived: number; framesDecoded: number; bytesReceived: number } + >(); private suppressDisconnectToast = false; private explicitMediaDeviceOperationDepth = 0; private lastMediaDeviceToast: { @@ -780,6 +791,7 @@ export class VoiceCallState { this.isNativeScreenShareEnabled = true; this.nativeScreenShareSourceName = sourceName; this.isScreenShareEnabled = true; + this.suppressLocalCompanionSubscriptions(); this.updateParticipants(); } @@ -963,6 +975,7 @@ export class VoiceCallState { const { RoomEvent, Track } = getLoadedLiveKit(); this.room.on(RoomEvent.ParticipantConnected, () => { + this.suppressLocalCompanionSubscriptions(); this.updateParticipants(); }); @@ -1007,13 +1020,33 @@ export class VoiceCallState { RoomEvent.TrackSubscribed, ( track: RemoteTrack, - _publication: RemoteTrackPublication, + publication: RemoteTrackPublication, participant: RemoteParticipant ) => { + const isLocalCompanion = this.isLocalCompanionPublisher(participant); + if ( + isLocalCompanion && + (this.hasLocalNativePreview() || track.kind === Track.Kind.Audio) + ) { + track.detach(); + publication.setSubscribed(false); + this.updateParticipants(); + return; + } if (track.kind === Track.Kind.Audio) { - if (!this.isLocalCompanionPublisher(participant)) track.attach(); + track.attach(); this.applyAllParticipantAudioVolumes(); } + if ( + track.source === Track.Source.ScreenShare && + isCompanionPublisher(participant) && + publication.simulcasted !== true + ) { + // The native helper publishes one full-cadence layer. Request a high + // receive ceiling explicitly so LiveKit does not treat a compact UI + // tile like a conventional 30 fps browser screen share. + publication.setVideoFPS(NATIVE_SCREEN_SHARE_MAX_RECEIVE_FPS); + } this.updateParticipants(); } ); @@ -1028,6 +1061,7 @@ export class VoiceCallState { // Track published/unpublished — catches camera enable/disable by remote participants this.room.on(RoomEvent.TrackPublished, () => { + this.suppressLocalCompanionSubscriptions(); this.updateParticipants(); }); @@ -1048,6 +1082,98 @@ export class VoiceCallState { this.audioLevelInterval = setInterval(() => { this.updateAudioLevels(); }, 60); + this.nativeScreenShareStatsInterval = setInterval(() => { + void this.reportNativeScreenShareReceiverStats(); + }, NATIVE_SCREEN_SHARE_STATS_INTERVAL_MS); + } + + private async reportNativeScreenShareReceiverStats(): Promise { + if (!this.room || this.nativeScreenShareStatsReportInFlight) return; + this.nativeScreenShareStatsReportInFlight = true; + try { + const { Track } = getLoadedLiveKit(); + for (const participant of this.room.remoteParticipants.values()) { + if (!isCompanionPublisher(participant)) continue; + for (const publication of participant.trackPublications.values()) { + const track = publication.track; + if ( + publication.source !== Track.Source.ScreenShare || + !track || + track.kind !== Track.Kind.Video + ) { + continue; + } + const report = await (track as RemoteTrack).getRTCStatsReport(); + if (!report) continue; + report.forEach((entry) => { + if (entry.type !== 'inbound-rtp' || entry.kind !== 'video') return; + const stats = entry as RTCInboundRtpStreamStats & { + decoderImplementation?: string; + framesDecoded?: number; + framesDropped?: number; + framesPerSecond?: number; + framesReceived?: number; + freezeCount?: number; + jitterBufferDelay?: number; + jitterBufferEmittedCount?: number; + keyFramesDecoded?: number; + totalDecodeTime?: number; + totalFreezesDuration?: number; + }; + const framesReceived = stats.framesReceived ?? 0; + const framesDecoded = stats.framesDecoded ?? 0; + const bytesReceived = stats.bytesReceived ?? 0; + const previous = this.nativeScreenSharePreviousStats.get(entry.id); + const elapsedSeconds = previous ? (stats.timestamp - previous.timestamp) / 1_000 : 0; + const intervalReceiveFps = + previous && elapsedSeconds > 0 + ? (framesReceived - previous.framesReceived) / elapsedSeconds + : 0; + const intervalDecodeFps = + previous && elapsedSeconds > 0 + ? (framesDecoded - previous.framesDecoded) / elapsedSeconds + : 0; + const intervalBitrate = + previous && elapsedSeconds > 0 + ? ((bytesReceived - previous.bytesReceived) * 8) / elapsedSeconds + : 0; + this.nativeScreenSharePreviousStats.set(entry.id, { + timestamp: stats.timestamp, + framesReceived, + framesDecoded, + bytesReceived + }); + console.info('[Chatto] Native screen-share receiver metrics', { + intervalReceiveFps, + intervalDecodeFps, + intervalBitrate, + browserFramesPerSecond: stats.framesPerSecond ?? 0, + framesReceived, + framesDecoded, + framesDropped: stats.framesDropped ?? 0, + keyFramesDecoded: stats.keyFramesDecoded ?? 0, + packetsLost: stats.packetsLost ?? 0, + jitterSeconds: stats.jitter ?? 0, + averageJitterBufferMs: + stats.jitterBufferDelay && stats.jitterBufferEmittedCount + ? (stats.jitterBufferDelay / stats.jitterBufferEmittedCount) * 1_000 + : 0, + averageDecodeMs: + stats.totalDecodeTime && framesDecoded + ? (stats.totalDecodeTime / framesDecoded) * 1_000 + : 0, + freezeCount: stats.freezeCount ?? 0, + totalFreezesDuration: stats.totalFreezesDuration ?? 0, + decoderImplementation: stats.decoderImplementation ?? '' + }); + }); + } + } + } catch (error) { + console.warn('[Chatto] Could not read native screen-share receiver metrics', error); + } finally { + this.nativeScreenShareStatsReportInFlight = false; + } } private updateParticipants(): void { @@ -1077,9 +1203,12 @@ export class VoiceCallState { const companion = companionPublishers.find( (candidate) => parseParticipantMetadata(candidate.metadata).ownerIdentity === p.identity ); - const screenShareTrack = - getParticipantScreenShareTrack(p) ?? - (companion ? getParticipantScreenShareTrack(companion) : null); + const useLocalNativePreview = + isLocal && this.nativeScreenShareSession?.preview != null; + const screenSharePublication = + getParticipantScreenSharePublication(p) ?? + (!useLocalNativePreview && companion ? getParticipantScreenSharePublication(companion) : null); + const screenShareTrack = screenSharePublication?.track ?? null; return { identity: p.identity, name: p.name ?? p.identity, @@ -1093,11 +1222,29 @@ export class VoiceCallState { isScreenShareEnabled: screenShareTrack !== null || (isLocal && this.isNativeScreenShareEnabled), screenShareTrack, + nativeScreenSharePreview: useLocalNativePreview + ? this.nativeScreenShareSession?.preview ?? null + : null, + screenShareSimulcasted: screenSharePublication?.simulcasted === true, isLocallyMuted: !isLocal && this.isParticipantLocallyMuted(p.identity) }; }); } + private suppressLocalCompanionSubscriptions(): void { + if (!this.room || !this.hasLocalNativePreview()) return; + for (const participant of this.room.remoteParticipants.values()) { + if (!this.isLocalCompanionPublisher(participant)) continue; + for (const publication of participant.trackPublications.values()) { + publication.setSubscribed(false); + } + } + } + + private hasLocalNativePreview(): boolean { + return this.nativeScreenShareSession?.preview != null; + } + private applyAllParticipantAudioVolumes(): void { if (!this.room) return; for (const participant of this.room.remoteParticipants.values()) { @@ -1231,6 +1378,12 @@ export class VoiceCallState { clearInterval(this.audioLevelInterval); this.audioLevelInterval = null; } + if (this.nativeScreenShareStatsInterval) { + clearInterval(this.nativeScreenShareStatsInterval); + this.nativeScreenShareStatsInterval = null; + } + this.nativeScreenShareStatsReportInFlight = false; + this.nativeScreenSharePreviousStats.clear(); this.teardownLocalAudioAnalyser(); if (this.room) { // Detach all remote audio tracks to clean up