diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt new file mode 100644 index 00000000..96834a9b --- /dev/null +++ b/benchmark/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.16) +project(obs-bgremoval-bench LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(OpenCV REQUIRED COMPONENTS core) +find_package(Threads REQUIRED) +find_package(onnxruntime CONFIG QUIET) + +add_executable(bench bench.cpp) +target_include_directories(bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src) +target_link_libraries(bench PRIVATE ${OpenCV_LIBS} Threads::Threads) + +if(onnxruntime_FOUND) + message(STATUS "ONNX Runtime found - CPU utilization test enabled") + target_compile_definitions(bench PRIVATE HAS_ONNXRUNTIME=1) + target_link_libraries(bench PRIVATE onnxruntime::onnxruntime) +else() + message(STATUS "ONNX Runtime not found - CPU utilization test disabled") +endif() diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000..3d5e6d90 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,61 @@ +# Benchmark + +Micro-benchmark for the `obs-backgroundremoval` pre-inference pipeline. +It exercises the same code path the plugin runs in `video_tick` before +model inference: cloning the input frame and running a PSNR similarity +check on full-size 1920x1080 BGRA frames. + +## Prerequisites + +- A **C++17** compiler +- **CMake** 3.16+ +- **OpenCV** (core) +- **ONNX Runtime** (optional) — enables the CPU utilisation test + +## Build + +Run from the repository root: + +```bash +cmake -B build benchmark +cmake --build build +``` + +If ONNX Runtime is found by CMake, the CPU utilisation test is compiled in +automatically. + +## Usage + +``` +./build/bench [-n ] [-m ] +``` + +| Flag | Default | Description | +|------|---------|-------------| +| `-n` | 660 | Number of synthetic frames to process | +| `-m` | *(none)* | Path to an ONNX model (e.g. `benchmark/tiny.onnx`). Enables the ORT CPU utilisation test (Linux only). | + +## What it measures + +- **Per-frame timing** — mean, median, and p95 latency in microseconds. +- **Hardware perf counters** (Linux only) — cache references, cache misses, + LLC store misses via `perf_event_open`. +- **ORT CPU utilisation** (requires `-m`) — runs 90 frames at 30 fps with a + real ONNX Runtime session and reports wall time vs CPU time, showing the + effect of thread-pool spin-waiting. + +## Example output + +``` +Generated 60 synthetic frames (1920x1080 BGRA) + +=== obs-backgroundremoval benchmark === +Frames: 60 @ 1920x1080 BGRA (7.9 MB/frame) + mean: 2309.6 us/frame + median: 2012.1 us/frame + p95: 4637.1 us/frame + total: 138.6 ms (60 frames) + cache-misses: 29067554 (84.0% of 34591781 refs) + LLC-store-misses: 8461194 (86.8% of 9749379 stores) +=== +``` diff --git a/benchmark/bench.cpp b/benchmark/bench.cpp new file mode 100644 index 00000000..c4c5e28c --- /dev/null +++ b/benchmark/bench.cpp @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: 2026 Xavier Ruiz +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#include +#include +#include +#endif + +#ifndef _WIN32 +#include +#endif + +#ifdef HAS_ONNXRUNTIME +#include +#endif + +#include "pipeline-helpers.h" + +using Clock = std::chrono::high_resolution_clock; +using std::chrono::duration; + +// ─── Hardware performance counters (Linux only) ────────────────────── + +struct PerfCounter { + int fd = -1; + const char *name; + +#ifdef __linux__ + PerfCounter(uint32_t type, uint64_t config, const char *label) : name(label) + { + struct perf_event_attr pe = {}; + pe.type = type; + pe.size = sizeof(pe); + pe.config = config; + pe.disabled = 1; + pe.exclude_kernel = 1; + pe.exclude_hv = 1; + fd = (int)syscall(__NR_perf_event_open, &pe, 0, -1, -1, 0); + } + ~PerfCounter() + { + if (fd >= 0) + close(fd); + } + void reset() + { + if (fd >= 0) + ioctl(fd, PERF_EVENT_IOC_RESET, 0); + } + void enable() + { + if (fd >= 0) + ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); + } + void disable() + { + if (fd >= 0) + ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); + } + long long read_count() + { + if (fd < 0) + return -1; + long long count = 0; + if (::read(fd, &count, sizeof(count)) != sizeof(count)) + return -1; + return count; + } +#else + PerfCounter(uint32_t, uint64_t, const char *label) : name(label) {} + ~PerfCounter() {} + void reset() {} + void enable() {} + void disable() {} + long long read_count() { return -1; } +#endif +}; + +// ─── CPU time measurement ──────────────────────────────────────────── + +#ifndef _WIN32 +static double get_cpu_seconds() +{ + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); + return (ru.ru_utime.tv_sec + ru.ru_utime.tv_usec / 1e6) + (ru.ru_stime.tv_sec + ru.ru_stime.tv_usec / 1e6); +} +#endif + +// ─── Pipeline benchmark ────────────────────────────────────────────── + +// Exercises the same pre-inference pipeline the plugin uses in video_tick: +// 1. Clone the input frame (simulates inputBGRA.clone()) +// 2. pipeline::check_similarity (PSNR skip check on full-size frame) +static double process_frame(cv::Mat &lastImage, std::mutex &mtx, const cv::Mat &frame) +{ + auto t0 = Clock::now(); + + // Step 1: Clone under lock (simulates video_tick acquiring inputBGRA) + cv::Mat cloned; + { + std::lock_guard lock(mtx); + cloned = frame.clone(); + } + + // Step 2: Similarity check on the full-size frame + pipeline::check_similarity(cloned, lastImage, 35.0); + + auto t1 = Clock::now(); + return duration(t1 - t0).count(); +} + +// Generate synthetic 1920x1080 BGRA frames. +// Most frames differ by small noise (exercises PSNR skip path); +// every 30th frame is a "scene change" (exercises the non-skip path). +static std::vector generate_frames(int count) +{ + const int W = 1920, H = 1080; + std::vector frames; + frames.reserve(count); + + cv::RNG rng(42); + cv::Mat base(H, W, CV_8UC4); + rng.fill(base, cv::RNG::UNIFORM, 0, 256); + + for (int i = 0; i < count; i++) { + if (i > 0 && i % 30 == 0) { + rng.fill(base, cv::RNG::UNIFORM, 0, 256); + } else if (i > 0) { + cv::Mat noise(H, W, CV_8UC4); + rng.fill(noise, cv::RNG::UNIFORM, 0, 4); + cv::add(base, noise, base); + } + frames.push_back(base.clone()); + } + printf("Generated %d synthetic frames (%dx%d BGRA)\n", count, W, H); + return frames; +} + +int main(int argc, char **argv) +{ + std::string model_path; + int max_frames = 660; + + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + if (arg == "-n" && i + 1 < argc) + max_frames = std::stoi(argv[++i]); + else if (arg == "-m" && i + 1 < argc) + model_path = argv[++i]; + } + + auto frames = generate_frames(max_frames); + + size_t frame_bytes = frames[0].total() * frames[0].elemSize(); + + // Set up hardware performance counters (Linux perf_event_open) +#ifdef __linux__ + PerfCounter cache_refs(PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, "cache-references"); + PerfCounter cache_misses(PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, "cache-misses"); + PerfCounter llc_stores(PERF_TYPE_HW_CACHE, + PERF_COUNT_HW_CACHE_LL | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | + (PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16), + "LLC-stores"); + PerfCounter llc_store_misses(PERF_TYPE_HW_CACHE, + PERF_COUNT_HW_CACHE_LL | (PERF_COUNT_HW_CACHE_OP_WRITE << 8) | + (PERF_COUNT_HW_CACHE_RESULT_MISS << 16), + "LLC-store-misses"); + bool has_perf = (cache_misses.fd >= 0); +#else + PerfCounter cache_refs(0, 0, "cache-references"); + PerfCounter cache_misses(0, 0, "cache-misses"); + PerfCounter llc_stores(0, 0, "LLC-stores"); + PerfCounter llc_store_misses(0, 0, "LLC-store-misses"); + bool has_perf = false; +#endif + + // Warmup (10 frames) + { + std::mutex mtx; + cv::Mat lastImage; + for (int i = 0; i < 10 && i < (int)frames.size(); i++) + process_frame(lastImage, mtx, frames[i]); + } + + // Reset and enable counters before benchmark + cache_refs.reset(); + cache_misses.reset(); + llc_stores.reset(); + llc_store_misses.reset(); + cache_refs.enable(); + cache_misses.enable(); + llc_stores.enable(); + llc_store_misses.enable(); + + // Benchmark + std::vector samples; + { + std::mutex mtx; + cv::Mat lastImage; + for (size_t i = 0; i < frames.size(); i++) { + double us = process_frame(lastImage, mtx, frames[i]); + samples.push_back(us); + } + } + + // Disable counters and read values + cache_refs.disable(); + cache_misses.disable(); + llc_stores.disable(); + llc_store_misses.disable(); + + long long cr = cache_refs.read_count(); + long long cm = cache_misses.read_count(); + long long ls = llc_stores.read_count(); + long long lm = llc_store_misses.read_count(); + + std::sort(samples.begin(), samples.end()); + double sum = std::accumulate(samples.begin(), samples.end(), 0.0); + double mean = sum / samples.size(); + double median = samples[samples.size() / 2]; + double p95 = samples[(size_t)(samples.size() * 0.95)]; + + printf("\n=== obs-backgroundremoval benchmark ===\n"); + printf("Frames: %zu @ %dx%d BGRA (%.1f MB/frame)\n", samples.size(), frames[0].cols, frames[0].rows, + frame_bytes / (1024.0 * 1024.0)); + printf(" mean: %.1f us/frame\n", mean); + printf(" median: %.1f us/frame\n", median); + printf(" p95: %.1f us/frame\n", p95); + printf(" total: %.1f ms (%zu frames)\n", sum / 1000.0, samples.size()); + if (has_perf) { + printf(" cache-misses: %lld (%.1f%% of %lld refs)\n", cm, cr > 0 ? 100.0 * cm / cr : 0.0, cr); + printf(" LLC-store-misses: %lld (%.1f%% of %lld stores)\n", lm, ls > 0 ? 100.0 * lm / ls : 0.0, ls); + } + // CPU utilization test: real ORT thread pool with 30fps pacing +#if defined(HAS_ONNXRUNTIME) && !defined(_WIN32) + if (!model_path.empty()) { + const int cpu_frames = 90; // 3 seconds at 30fps + const auto interval = std::chrono::microseconds(33333); + const int ort_threads = 4; + const bool ort_spinning = false; + + // Suppress OpenCV thread pool for this test so only ORT + // threads contribute to CPU utilization. + cv::setNumThreads(1); + + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "bench"); + Ort::SessionOptions opts; + opts.SetIntraOpNumThreads(ort_threads); + opts.SetInterOpNumThreads(ort_threads); + opts.AddConfigEntry("session.intra_op.allow_spinning", ort_spinning ? "1" : "0"); + opts.AddConfigEntry("session.inter_op.allow_spinning", ort_spinning ? "1" : "0"); + + // Count process threads via /proc/self/status + auto count_threads = []() -> int { + FILE *f = fopen("/proc/self/status", "r"); + if (!f) + return -1; + char line[256]; + while (fgets(line, sizeof(line), f)) { + int n; + if (sscanf(line, "Threads: %d", &n) == 1) { + fclose(f); + return n; + } + } + fclose(f); + return -1; + }; + + int threads_before = count_threads(); + + // Create session — thread pool is lazily initialized. + Ort::Session session(env, model_path.c_str(), opts); + + // Set up inference inputs for the tiny Identity model. + // Running inference each frame keeps ORT's thread pool + // active — without periodic calls, spinning threads hit + // a timeout and go to sleep, hiding the CPU cost. + std::vector input_data = {1.0f}; + std::vector input_shape = {1}; + auto mem_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + auto input_tensor = + Ort::Value::CreateTensor(mem_info, input_data.data(), 1, input_shape.data(), 1); + const char *input_names[] = {"x"}; + const char *output_names[] = {"y"}; + + // Warmup inference to create thread pool + session.Run(Ort::RunOptions{nullptr}, input_names, &input_tensor, 1, output_names, 1); + + int threads_after = count_threads(); + + double cpu0 = get_cpu_seconds(); + auto wall0 = Clock::now(); + + std::mutex mtx; + cv::Mat lastImage; + + for (int i = 0; i < cpu_frames; i++) { + auto t = Clock::now(); + process_frame(lastImage, mtx, frames[i % frames.size()]); + // Run inference to keep ORT thread pool spinning + session.Run(Ort::RunOptions{nullptr}, input_names, &input_tensor, 1, output_names, 1); + std::this_thread::sleep_until(t + interval); + } + + auto wall1 = Clock::now(); + double cpu1 = get_cpu_seconds(); + + double wall_s = duration(wall1 - wall0).count(); + double cpu_s = cpu1 - cpu0; + + printf("--- CPU utilization (30fps, %d frames, %d ORT threads) ---\n", cpu_frames, ort_threads); + printf(" spinning: %s\n", ort_spinning ? "on" : "off"); + printf(" threads: %d -> %d (ORT created %d)\n", threads_before, threads_after, + threads_after - threads_before); + printf(" wall: %.2f s\n", wall_s); + printf(" cpu: %.2f s (%.0f%%, %.1f cores)\n", cpu_s, 100.0 * cpu_s / wall_s, cpu_s / wall_s); + } +#endif + + printf("===\n"); + + return 0; +} diff --git a/benchmark/tiny.onnx b/benchmark/tiny.onnx new file mode 100644 index 00000000..461943ce Binary files /dev/null and b/benchmark/tiny.onnx differ diff --git a/src/background-filter.cpp b/src/background-filter.cpp index f1652b0a..dd67fb87 100644 --- a/src/background-filter.cpp +++ b/src/background-filter.cpp @@ -31,6 +31,7 @@ #include "models/ModelPPHumanSeg.hpp" #include "models/ModelTCMonoDepth.hpp" #include "FilterData.hpp" +#include "pipeline-helpers.h" #include "ort-utils/ort-session-utils.hpp" #include "obs-utils/obs-utils.hpp" #include "consts.h" @@ -544,16 +545,8 @@ void background_filter_video_tick(void *data, float seconds) } if (tf->enableImageSimilarity) { - if (!tf->lastImageBGRA.empty() && !imageBGRA.empty() && tf->lastImageBGRA.size() == imageBGRA.size()) { - // calculate PSNR - double psnr = cv::PSNR(tf->lastImageBGRA, imageBGRA); - - if (psnr > tf->imageSimilarityThreshold) { - // The image is almost the same as the previous one. Skip processing. - return; - } - } - tf->lastImageBGRA = imageBGRA.clone(); + if (pipeline::check_similarity(imageBGRA, tf->lastImageBGRA, tf->imageSimilarityThreshold)) + return; } if (tf->backgroundMask.empty()) { diff --git a/src/ort-utils/ort-session-utils.cpp b/src/ort-utils/ort-session-utils.cpp index 5e43ff30..73dc3e40 100644 --- a/src/ort-utils/ort-session-utils.cpp +++ b/src/ort-utils/ort-session-utils.cpp @@ -32,6 +32,8 @@ int createOrtSession(filter_data *tf) Ort::SessionOptions sessionOptions; sessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_ALL); + sessionOptions.AddConfigEntry("session.intra_op.allow_spinning", "0"); + sessionOptions.AddConfigEntry("session.inter_op.allow_spinning", "0"); if (tf->useGPU != USEGPU_CPU) { sessionOptions.DisableMemPattern(); sessionOptions.SetExecutionMode(ExecutionMode::ORT_SEQUENTIAL); diff --git a/src/pipeline-helpers.h b/src/pipeline-helpers.h new file mode 100644 index 00000000..2087df25 --- /dev/null +++ b/src/pipeline-helpers.h @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Xavier Ruiz +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +namespace pipeline { + +// Similarity check: skip processing when the frame is nearly identical +// to the previous one (PSNR above threshold). +inline bool check_similarity(const cv::Mat ¤t, cv::Mat &lastImage, double threshold) +{ + if (!lastImage.empty() && lastImage.size() == current.size()) { + if (cv::PSNR(lastImage, current) > threshold) + return true; + } + current.copyTo(lastImage); + return false; +} + +} // namespace pipeline