From f8be2d157484f1e8b3c856bc2923751ee8b2eaec Mon Sep 17 00:00:00 2001 From: Yannik Schroeder Date: Fri, 24 Jul 2026 17:34:36 +0200 Subject: [PATCH 1/5] first mediabunny video attempt --- package-lock.json | 37 ++++++++- package.json | 1 + .../components/videoScreen/videoPlayback.vue | 79 ++++++++++++++++++- 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4614e0b2eb..fd211428eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,9 @@ { - "name": "core", + "name": "biigle", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "core", "license": "GPL-3.0-only", "dependencies": { "@biigle/ol": "^9.2.4", @@ -19,6 +18,7 @@ "geotiff": "^2.1.3", "jsts": "^2.11.0", "magic-wand-tool": "^1.1.4", + "mediabunny": "^1.51.0", "mitt": "^3.0.1", "onnxruntime-web": "^1.20.1", "polymorph-js": "^0.2.4", @@ -1185,6 +1185,21 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/dom-mediacapture-transform": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.12.tgz", + "integrity": "sha512-d7/QsLRwF864A5mgIM/YrfiglHoYn7zgCcAoJgW404r+2DwnNr7EBbLnCWpmOMgH8y0te73L1AV6H1bmauaWFw==", + "license": "MIT", + "dependencies": { + "@types/dom-webcodecs": "*" + } + }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "license": "MIT" + }, "node_modules/@types/express-serve-static-core": { "version": "4.17.28", "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.28.tgz", @@ -2939,6 +2954,24 @@ "integrity": "sha512-S4rHzCs/bAp7nhQGKeg+McWuqrdyZKpnu8Ahd8AU7NzuLTm/Hh8tkpv1tW91Kmm59foIrXzip1d+P9NDoyxrZA==", "license": "MIT" }, + "node_modules/mediabunny": { + "version": "1.51.0", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.51.0.tgz", + "integrity": "sha512-u327374xU8Ho0gCaMII7fUK8t0PnqkabCox1k8uUwvgvGb9o6YQGZEG2Qr4DTe7nTMpzfL7ukgnHDvDROySZ+Q==", + "license": "MPL-2.0", + "workspaces": [ + ".", + "packages/*" + ], + "dependencies": { + "@types/dom-mediacapture-transform": "^0.1.11", + "@types/dom-webcodecs": "0.1.13" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + } + }, "node_modules/micromatch": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", diff --git a/package.json b/package.json index ec92e6ebc8..bd873e5ec6 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "geotiff": "^2.1.3", "jsts": "^2.11.0", "magic-wand-tool": "^1.1.4", + "mediabunny": "^1.51.0", "mitt": "^3.0.1", "onnxruntime-web": "^1.20.1", "polymorph-js": "^0.2.4", diff --git a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue index 085ab9548a..7237f22954 100644 --- a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue +++ b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue @@ -3,6 +3,7 @@ import CanvasSource from '@/annotations/ol/source/Canvas.js'; import ImageLayer from '@biigle/ol/layer/Image'; import Projection from '@biigle/ol/proj/Projection'; import View from '@biigle/ol/View'; +import { Input, ALL_FORMATS, UrlSource, VideoSampleSink } from 'mediabunny'; /** * Mixin for the videoScreen component that contains logic for the video playback. @@ -26,6 +27,11 @@ export default { // parameter tracking seeking state specific for frame jump, needed because looking for seeking directly leads to error seekingFrame: this.seeking, shouldUseVideoFrameCallback: false, + mediabunnyInput: null, + mediabunnySink: null, + cachedBitmap: null, + cachedTime: null, + useMediabunnyFallback: true, }; }, methods: { @@ -75,11 +81,17 @@ export default { this.updateMapReadyRevision(); }, renderVideo(force) { + const currentTime = this.video.currentTime; // Drop animation frame if the time has not changed. if (force || this.renderCurrentTime !== this.video.currentTime) { this.renderCurrentTime = this.video.currentTime; - this.videoContext.drawImage(this.video, 0, 0, this.videoCanvas.width, this.videoCanvas.height); - this.videoSource.changed(); + + if (!this.video.paused && this.useMediabunnyFallback) { + this.getFrameBitmap(currentTime); + } else { + this.videoContext.drawImage(this.video, 0, 0, this.videoCanvas.width, this.videoCanvas.height); + this.videoSource.changed(); + } } }, startRenderLoop() { @@ -243,6 +255,58 @@ export default { toggleFirefoxFullscreen() { document.body.requestFullscreen(); }, + async initMediabunny() { + if (this.mediabunnyInput) { + this.mediabunnyInput.close(); + this.cachedBitmap?.close(); + } + this.cachedBitmap = null; + this.cachedTime = null; + this.useMediabunnyFallback = true; + + try { + this.mediabunnyInput = new Input({ + source: new UrlSource(this.video.src), + formats: ALL_FORMATS + }); + console.log(this.video.src); + const track = await this.mediabunnyInput.getPrimaryVideoTrack(); + this.mediabunnySink = new VideoSampleSink(track); + } catch (e) { + this.useMediabunnyFallback = false; + throw e; + } + }, + async getFrameBitmap(time) { + if (!this.useMediabunnyFallback || !this.mediabunnySink) { + return; + } + + const roundedTime = Math.round(time * 100) / 100; + if (this.cachedTime === roundedTime && this.cachedBitmap) { + this.drawBitmap(this.cachedBitmap); + return; + } + + if (this.cachedBitmap) { + this.cachedBitmap.close(); + } + + const sample = await this.mediabunnySink.getSample(roundedTime); + const videoFrame = sample.toVideoFrame(); + const bitmap = await createImageBitmap(videoFrame); + sample.close(); + videoFrame.close(); + + this.cachedTime = roundedTime; + this.cachedBitmap = bitmap; + this.drawBitmap(bitmap); + }, + drawBitmap(bitmap) { + this.videoContext.drawImage(bitmap, 0, 0, this.videoCanvas.width, this.videoCanvas.height); + this.videoSource.changed(); + }, + }, watch: { seeking(seeking) { @@ -252,6 +316,17 @@ export default { this.startRenderLoop(); } }, + video: { + immediate: true, + deep: true, + handler() { + console.log(this.video.src); + if (!this.video.src) { + return; + } + this.initMediabunny(); + } + } }, created() { this.videoCanvas = document.createElement('canvas'); From 9ad5c3e2ae6a01cc764c5d2eeeb0b5421469c53c Mon Sep 17 00:00:00 2001 From: Yannik Schroeder Date: Tue, 28 Jul 2026 17:47:12 +0200 Subject: [PATCH 2/5] init mediabunny correctly --- .../js/videos/components/videoScreen/videoPlayback.vue | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue index 7237f22954..8bf783b19c 100644 --- a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue +++ b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue @@ -320,11 +320,9 @@ export default { immediate: true, deep: true, handler() { - console.log(this.video.src); - if (!this.video.src) { - return; - } - this.initMediabunny(); + this.video.addEventListener('loadedmetadata', () => { + this.initMediabunny(); + }); } } }, From 43e2f2646a8e49291539d453772f9ea15cca8efb Mon Sep 17 00:00:00 2001 From: yannik131 Date: Fri, 21 Aug 2026 19:55:24 +0200 Subject: [PATCH 3/5] Use frame before timestamp for largo thumbnails --- app/Jobs/ProcessAnnotatedVideo.php | 103 ++++++++++++++++--- tests/php/Jobs/ProcessAnnotatedVideoTest.php | 2 +- 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/app/Jobs/ProcessAnnotatedVideo.php b/app/Jobs/ProcessAnnotatedVideo.php index 7f5811d91c..6c47a0d4e8 100644 --- a/app/Jobs/ProcessAnnotatedVideo.php +++ b/app/Jobs/ProcessAnnotatedVideo.php @@ -5,25 +5,31 @@ use Biigle\VideoAnnotation; use Biigle\VideoAnnotationLabelFeatureVector; use Biigle\VolumeFile; -use FFMpeg\Coordinate\TimeCode; use FFMpeg\Exception\RuntimeException; use FFMpeg\FFMpeg; +use FFMpeg\FFProbe; use FFMpeg\Media\Video; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Jcupitt\Vips\Exception as VipsException; use Jcupitt\Vips\Image; +use Throwable; class ProcessAnnotatedVideo extends ProcessAnnotatedFile { + /** + * FFProbe instance for extracting the FPS + */ + private ?FFProbe $ffprobe = null; + /** * {@inheritdoc} */ public function handleFile(VolumeFile $file, $path) { - $video = $this->getVideo($path); // The chunk size is rather low because individual video annotations can contain // lots of data (if they are multi-frame annotations from object tracking with // many annotated frames). With a chunk size too large, this could run into out @@ -31,7 +37,7 @@ public function handleFile(VolumeFile $file, $path) // Also (if feature vectors are generated), a PNG is stored for each frame in a // chunk. Large chunks could comsume too much space. $this->getAnnotationQuery($file) - ->chunkById(100, fn ($a) => $this->processAnnotationChunk($a, $video)); + ->chunkById(100, fn ($a) => $this->processAnnotationChunk($a, $path)); } /** @@ -39,7 +45,7 @@ public function handleFile(VolumeFile $file, $path) * * @param Collection $annotations */ - protected function processAnnotationChunk(Collection $annotations, Video $video): void + protected function processAnnotationChunk(Collection $annotations, string $sourcePath): void { $frameFiles = []; @@ -48,7 +54,7 @@ protected function processAnnotationChunk(Collection $annotations, Video $video) $points = $a->points[0] ?? null; $frame = $a->frames[0]; try { - $videoFrame = $this->getVideoFrame($video, $frame); + $videoFrame = $this->getVideoFrame($sourcePath, $frame); } catch (RuntimeException $e) { // FFMpeg can't extract the frame. continue; @@ -128,31 +134,102 @@ protected function getVideo($path) /** * Get a video frame from a specific time as VipsImage object. + * If thumbnail creation only used ffmpeg -ss ... (or a library call + * that would resolve to the same command), ffmpeg would return the frame at or + * after the timestamp. Browsers usually display the one at or before the timestamp. + * To mimic browser behavior, we force ffmpeg to give the frame at or before the + * timestamp by decoding a short window around the timestamp and using the + * `select='lte(t,)'` filter. * - * @param Video $video + * @param string $sourcePath * @param float $time * @param int $trySeek * * @return Image */ - protected function getVideoFrame(Video $video, float $time, int $trySeek = 60) + protected function getVideoFrame(string $sourcePath, float $time, int $trySeek = 60) { + $fps = max($this->getVideoFps($sourcePath), 0.0001); + $window = max(0.05, 3 / $fps); + // Sometimes an annotation is near the end of the video (or exactly at the end). // FFMpeg often returns an empty buffer in this case. If there is an empty frame, // we try to seek backwards one frame until the buffer is not empty or the number // of tries is exceeded. do { - /** @var string */ - $buffer = $video->frame(TimeCode::fromSeconds($time)) - ->save('', false, true); + $seekTime = sprintf('%F', max(0, $time - $window)); + $decodeDuration = sprintf('%F', 2 * $window); + $timeString = sprintf('%F', $time); + + // Use a temporary file with random name for the jpg thumbnail file + // We only return the buffer anyways, the file can be safely deleted later + $tmpFile = tempnam(sys_get_temp_dir(), 'largo_video_frame'); + if ($tmpFile === false) { + throw new RuntimeException('Could not create a temporary file.'); + } + + $outputPath = "{$tmpFile}.jpg"; + File::move($tmpFile, $outputPath); + + try { + Process::forever() + ->run(sprintf( + 'ffmpeg -copyts -ss %s -t %s -i %s -vf "select=\'lte(t\\,%s)\'" -fps_mode passthrough -update 1 -y %s', + $seekTime, + $decodeDuration, + escapeshellarg($sourcePath), + $timeString, + escapeshellarg($outputPath) + )) + ->throw(); + + $buffer = File::get($outputPath); + } catch (Throwable $e) { + $buffer = ''; + } finally { + if (File::exists($outputPath)) { + File::delete($outputPath); + } + } + + if (!empty($buffer)) { + return Image::newFromBuffer($buffer); + } + $trySeek -= 1; // Roughly estimated framerate of 30 fps. With 60 iterations, we seek back up // to 2 s by default (this is based on what was required for edge cases in // 1.5M annotations on 16k videos). - $time = max(0, $time - 0.033333333); - } while (empty($buffer) && $trySeek > 0); + $time = max(0, $time - (1 / 30.0)); + } while ($trySeek > 0); + + throw new RuntimeException('Could not extract a video frame.'); + } + + /** + * Return the average video fps using ffprobe or fall back to 30 FPS if it failed + * @param string $path Path to the video file + * @return float The average FPS of the video + */ + protected function getVideoFps(string $path): float + { + if (!isset($this->ffprobe)) { + $this->ffprobe = FFProbe::create(); + } + + $stream = $this->ffprobe->streams($path)->videos()->first(); + if ($stream === null) { + return 30.0; + } + + // FFProbe returns FPS as a rational like 60/1 (60 FPS) since decimals + // can't represent some values correctly + $rate = $stream->get('avg_frame_rate'); + if (is_string($rate) && preg_match('/^(\d+)\/(\d+)$/', $rate, $matches) && (int) $matches[2] > 0) { + return (float) $matches[1] / (float) $matches[2]; + } - return Image::newFromBuffer($buffer); + return 30.0; } /** diff --git a/tests/php/Jobs/ProcessAnnotatedVideoTest.php b/tests/php/Jobs/ProcessAnnotatedVideoTest.php index 1d91b82605..89c89c8701 100644 --- a/tests/php/Jobs/ProcessAnnotatedVideoTest.php +++ b/tests/php/Jobs/ProcessAnnotatedVideoTest.php @@ -877,7 +877,7 @@ public function getVideo($path) return Mockery::mock(Video::class); } - public function getVideoFrame(Video $video, float $time, int $trySeek = 3) + public function getVideoFrame(string $sourcePath, float $time, int $trySeek = 3) { if ($this->throwGetFrame !== false) { throw new $this->throwGetFrame; From deea556aa77c208ce06e713ed13693ad0830df96 Mon Sep 17 00:00:00 2001 From: yannik131 Date: Sat, 22 Aug 2026 11:27:24 +0200 Subject: [PATCH 4/5] working mediabunny frame grab --- .../components/videoScreen/videoPlayback.vue | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue index 8bf783b19c..80b9494f89 100644 --- a/resources/assets/js/videos/components/videoScreen/videoPlayback.vue +++ b/resources/assets/js/videos/components/videoScreen/videoPlayback.vue @@ -81,18 +81,14 @@ export default { this.updateMapReadyRevision(); }, renderVideo(force) { - const currentTime = this.video.currentTime; // Drop animation frame if the time has not changed. - if (force || this.renderCurrentTime !== this.video.currentTime) { - this.renderCurrentTime = this.video.currentTime; - - if (!this.video.paused && this.useMediabunnyFallback) { - this.getFrameBitmap(currentTime); - } else { - this.videoContext.drawImage(this.video, 0, 0, this.videoCanvas.width, this.videoCanvas.height); - this.videoSource.changed(); - } + if (!force && this.renderCurrentTime === this.video.currentTime) { + return; } + + this.renderCurrentTime = this.video.currentTime; + this.videoContext.drawImage(this.video, 0, 0, this.videoCanvas.width, this.videoCanvas.height); + this.videoSource.changed(); }, startRenderLoop() { let render; @@ -118,6 +114,13 @@ export default { } this.animationFrameId = null; }, + async renderPausedFrame(time = this.video.currentTime) { + if (!this.video.paused || !this.useMediabunnyFallback) { + this.renderVideo(true); + return; + } + await this.getFrameBitmap(time); + }, setPlaying() { this.playing = true; if (!this.animationFrameId) { @@ -150,8 +153,12 @@ export default { // Update the layer (dimensions) if a new video is loaded. this.video.addEventListener('loadedmetadata', this.updateVideoLayer); }, - handleSeeked() { - this.renderVideo(true); + async handleSeeked() { + if (this.video.paused) { + await this.renderPausedFrame(); + } else { + this.renderVideo(true); + } }, // 5 next methods are a workaround to get previous and next frames, adapted from here: https://github.com/angrycoding/requestVideoFrameCallback-prev-next/tree/main async emitPreviousFrame() { @@ -269,7 +276,6 @@ export default { source: new UrlSource(this.video.src), formats: ALL_FORMATS }); - console.log(this.video.src); const track = await this.mediabunnyInput.getPrimaryVideoTrack(); this.mediabunnySink = new VideoSampleSink(track); } catch (e) { @@ -288,16 +294,21 @@ export default { return; } - if (this.cachedBitmap) { - this.cachedBitmap.close(); - } - const sample = await this.mediabunnySink.getSample(roundedTime); const videoFrame = sample.toVideoFrame(); const bitmap = await createImageBitmap(videoFrame); sample.close(); videoFrame.close(); + if (!this.video.paused || Math.round(this.video.currentTime * 100) / 100 !== roundedTime) { + bitmap.close(); + return; + } + + if (this.cachedBitmap) { + this.cachedBitmap.close(); + } + this.cachedTime = roundedTime; this.cachedBitmap = bitmap; this.drawBitmap(bitmap); From 177cf066ea5471c6ea79472d8b27de81f536e3ea Mon Sep 17 00:00:00 2001 From: yannik131 Date: Sat, 22 Aug 2026 17:24:52 +0200 Subject: [PATCH 5/5] Fixed some failing tests --- .../Controllers/Views/Admin/UsersController.php | 6 ++++-- app/Jobs/ProcessAnnotatedVideo.php | 13 +------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/app/Http/Controllers/Views/Admin/UsersController.php b/app/Http/Controllers/Views/Admin/UsersController.php index f34104dfb6..f99a06df73 100644 --- a/app/Http/Controllers/Views/Admin/UsersController.php +++ b/app/Http/Controllers/Views/Admin/UsersController.php @@ -205,7 +205,8 @@ protected function showAnnotations(User $user) $totalAnnotations = (clone $annotationQuery)->distinct()->count('image_annotations.id'); if ($totalAnnotations > 0) { - $relativeAnnotations = $totalAnnotations / ImageAnnotation::estimatedCount(); + $estimatedCount = ImageAnnotation::estimatedCount(); + $relativeAnnotations = $estimatedCount > 0 ? $totalAnnotations / $estimatedCount : 0; $recentImageAnnotations = $annotationQuery->orderBy('image_annotation_labels.created_at', 'desc') ->take(10) @@ -235,7 +236,8 @@ public function showVideos(User $user) $totalVideoAnnotations = (clone $annotationQuery)->distinct()->count('video_annotations.id'); if ($totalVideoAnnotations > 0) { - $relativeVideoAnnotations = $totalVideoAnnotations / VideoAnnotation::estimatedCount(); + $estimatedCount = VideoAnnotation::estimatedCount(); + $relativeVideoAnnotations = $estimatedCount > 0 ? $totalVideoAnnotations / $estimatedCount : 0; $recentVideoAnnotations = $annotationQuery->orderBy('video_annotation_labels.created_at', 'desc') ->take(10) diff --git a/app/Jobs/ProcessAnnotatedVideo.php b/app/Jobs/ProcessAnnotatedVideo.php index d829d65214..1228fec75c 100644 --- a/app/Jobs/ProcessAnnotatedVideo.php +++ b/app/Jobs/ProcessAnnotatedVideo.php @@ -8,7 +8,6 @@ use FFMpeg\Exception\RuntimeException; use FFMpeg\FFMpeg; use FFMpeg\FFProbe; -use FFMpeg\Media\Video; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Support\Facades\File; @@ -39,7 +38,7 @@ public function handleFile(VolumeFile $file, $path) // of memory issues. // Also (if feature vectors are generated), a PNG is stored for each frame in a // chunk. Large chunks could comsume too much space. - $this->getAnnotationQuery($file) + $this->getAnnotationQuery() ->chunkById(100, fn ($a) => $this->processAnnotationChunk($a, $path)); } @@ -125,16 +124,6 @@ protected function updateOrCreateFeatureVectors(Collection $annotations, \Genera } } - /** - * Get the FFMpeg video instance. - * - * @param string $path - */ - protected function getVideo($path) - { - return FFMpeg::create()->open($path); - } - /** * Get a video frame from a specific time as VipsImage object. * If thumbnail creation only used ffmpeg -ss ... (or a library call