diff --git a/examples/video_hybrid_search/.env.example b/examples/video_hybrid_search/.env.example new file mode 100644 index 000000000..114eb6fb9 --- /dev/null +++ b/examples/video_hybrid_search/.env.example @@ -0,0 +1,9 @@ +# Copy to .env (main.py loads it). Everything here is optional. + +# A Hugging Face token silences the "unauthenticated requests" warning and gives +# faster model downloads. Create one at https://huggingface.co/settings/tokens +HF_TOKEN= + +# Model overrides. Defaults: clip-vit-base-patch32 and whisper base. +# CLIP_MODEL=openai/clip-vit-large-patch14 +# WHISPER_MODEL=tiny diff --git a/examples/video_hybrid_search/.gitignore b/examples/video_hybrid_search/.gitignore new file mode 100644 index 000000000..7bdf15d98 --- /dev/null +++ b/examples/video_hybrid_search/.gitignore @@ -0,0 +1,11 @@ +.env +.venv/ +uv.lock +cocoindex.db +zvec_data/ +videos/*.mp4 +videos/*.mov +videos/*.mkv +videos/*.webm +*.egg-info/ +__pycache__/ diff --git a/examples/video_hybrid_search/ATTRIBUTION.md b/examples/video_hybrid_search/ATTRIBUTION.md new file mode 100644 index 000000000..fbde04814 --- /dev/null +++ b/examples/video_hybrid_search/ATTRIBUTION.md @@ -0,0 +1,24 @@ +# Sample video attribution + +`download_sample_videos.sh` fetches these clips from the Blender open movies and trims each to a short +segment. Credits and licenses below. The trimmed files are not checked into the repo. + +## tears_of_steel.mp4 + +- Title: Tears of Steel +- Author: Blender Foundation (mango.blender.org) +- License: CC BY 3.0 (https://creativecommons.org/licenses/by/3.0/) +- Source: https://mango.blender.org/ +- Direct file: https://download.blender.org/demo/movies/ToS/tears_of_steel_720p.mov +- Modifications: trimmed to 22 seconds starting at 0:25, scaled to 640px wide, transcoded to H.264/AAC mp4. +- Note: the license asks that images of the actors not be used in commercials. This example uses a short + non-commercial excerpt for local testing. + +## sintel.mp4 + +- Title: Sintel +- Author: Blender Foundation (durian.blender.org) +- License: CC BY 3.0 (https://creativecommons.org/licenses/by/3.0/) +- Source: https://durian.blender.org/ +- Direct file: https://download.blender.org/durian/movies/Sintel.2010.720p.mkv +- Modifications: trimmed to 22 seconds starting at 4:10, scaled to 640px wide, transcoded to H.264/AAC mp4. diff --git a/examples/video_hybrid_search/README.md b/examples/video_hybrid_search/README.md new file mode 100644 index 000000000..50359b1d8 --- /dev/null +++ b/examples/video_hybrid_search/README.md @@ -0,0 +1,169 @@ +# Video hybrid search (CocoIndex + zvec) + +Index a folder of videos into an embedded [zvec](https://zvec.org) collection, one document per scene, +then search it by keyframe vector, spoken words, and time window in a single query. CocoIndex keeps the +collection in sync as the folder changes: add a clip and only its scenes process, delete a clip and its +scenes drop, change the CLIP model and the embedding step reruns while transcripts stay cached. + +## The idea + +A video library grows every time you record or download something, and later you want to find a moment +by what is on screen, what someone said, and roughly when. Re-running the whole pipeline on every clip +is wasteful once transcription and embedding are in the mix. A plain folder watcher does not help +either: it will not remove a deleted clip's scenes, and it will not notice that swapping the embedder +should recompute the vectors. + +CocoIndex handles that. You declare the set of scene rows that should exist, and it reconciles the +collection to match, fingerprinting both the source content and the transform code. zvec holds the +keyframe vector, the transcript as a full-text field, and the scalar fields in one collection, so a +query can use all three at once. + +People have been building local video search tools lately (the +[Framedex](https://news.ycombinator.com/item?id=48222733) and +[edit-mind](https://news.ycombinator.com/item?id=48528029) discussions on Hacker News). Those projects +still have to manage index freshness themselves. That is the part CocoIndex owns here. + +## How it works + +One document per scene: + +```python +@dataclass +class Scene: + id: str # deterministic per (video_path, start) + video_path: str # scalar filter + start: float # scalar seconds + end: float # scalar seconds + transcript: Annotated[str, zvec.ZvecFtsType()] # full-text search over spoken words + embedding: NDArray[np.float32] # dense CLIP keyframe vector +``` + +The pipeline in [`main.py`](main.py) walks `./videos`, segments each clip into fixed `SCENE_SECONDS` +windows with `ffprobe` and `ffmpeg`, and for each scene pulls one keyframe and the audio slice, embeds +the keyframe with CLIP, transcribes the audio with faster-whisper, and declares a `Scene` row. + +Each heavy stage is its own memoized function, composed with `use_mount`: + +```python +@coco.fn(memo=True, deps=CLIP_MODEL_NAME) +async def embed_keyframe(frame_bytes: bytes) -> list[float]: ... + +@coco.fn(memo=True, deps=WHISPER_MODEL_NAME) +async def transcribe_audio(audio_bytes: bytes) -> str: ... + +embedding = await coco.use_mount(coco.component_subpath("embed", scene_id), embed_keyframe, frame_bytes) +transcript = await coco.use_mount(coco.component_subpath("transcribe", scene_id), transcribe_audio, audio_bytes) +``` + +Because embedding and transcription are separate memoized steps, changing the embedder recomputes only +`embed_keyframe` and leaves the transcripts cached. + +## Run it + +You need `ffmpeg` and `ffprobe` on your PATH (`brew install ffmpeg`, or `apt install ffmpeg`); the +download script uses them too. `main.py` checks for ffmpeg and errors early if it is missing. + +**1. Install, then use the example's venv.** + +```sh +cd examples/video_hybrid_search +uv sync +source .venv/bin/activate +``` + +Run everything through this venv. If you skip `source`, prefix the commands with `.venv/bin/` instead. +Plain `python` or `python3` uses your system interpreter, which does not have the project installed and +fails with `ImportError: cannot import name 'zvec' from 'cocoindex.connectors'`. + +The state db (`cocoindex.db`) and the zvec collection (`zvec_data`) are written next to `main.py`, so +run the commands from this folder. + +**2. Optional: a [Hugging Face token](https://huggingface.co/settings/tokens)** for faster, +warning-free model downloads. Copy the template and set `HF_TOKEN` in it: + +```sh +cp .env.example .env +``` + +**3. Get some clips into `./videos`.** Download a small, openly licensed sample set (two short clips with +real scenes and English dialogue, a few MB total): + +```sh +./download_sample_videos.sh +``` + +That gives you short segments from two Blender open movies, Tears of Steel and Sintel (both CC BY, +visually very different). ffmpeg pulls only the trimmed part over HTTP, so it is quick even though the +source films are large. Credits and licenses are in [ATTRIBUTION.md](ATTRIBUTION.md). + +No network, or want a deterministic offline set? `./make_sample_videos.sh` (macOS `say`) generates +solid-color clips with a spoken sentence. They drive the whole pipeline for a quick check, but the +visuals are blank, so CLIP has nothing to match and results come only from the transcript. + +**4. Index.** CocoIndex indexes the folder once and exits: + +```sh +cocoindex update main.py +``` + +The first run downloads the CLIP and whisper models. The default models are small; set +`CLIP_MODEL=openai/clip-vit-large-patch14` for higher-quality embeddings. + +> [!NOTE] +> Add `-L` to watch the folder live: `cocoindex update -L main.py`. It catches up, then reprocesses on +> every change, so adding or deleting a clip syncs its scenes within seconds. Live mode holds the +> terminal, so run queries and edit `./videos` from a second terminal in the same folder. + +**5. Query.** `query.py` reads the zvec collection directly, outside CocoIndex. It fuses a dense +sub-query and a full-text sub-query with reciprocal rank fusion, and filters on the scalar fields: + +```sh +python query.py "two people talking on a bridge" --fts "robotics" # tears_of_steel: visual + spoken +python query.py "a red-haired girl" --mode dense # sintel: visual only +python query.py "" --mode fts --fts "robot hand" # tears_of_steel: transcript only +python query.py "two people talking on a bridge" --filter "start < 15" # tears_of_steel: visual + time window +``` + +On the downloaded clips: the bridge query returns the Tears of Steel canal scene, the red-haired-girl +query returns Sintel, the robot-hand query returns the Tears of Steel line about a robot hand, and the +filter keeps only scenes before the 15-second mark. The first dense query loads the CLIP model, so it +pauses for a bit and prints `Loading the CLIP model...` while it works. `--mode fts` skips CLIP and +returns quickly. + +## Incremental behavior + +Re-run `cocoindex update main.py` after each change and watch what it does: + +- Add a clip to `./videos`, and only the new clip's scenes process. The rest are cache hits. +- Delete a clip, and its scenes disappear from the collection. +- Set `CLIP_MODEL` to a different model, and the embedding step reruns while the transcripts stay cached. + If the new model has a different vector size, zvec rebuilds the collection schema and rewrites the + vectors. To see only the embeddings change, switch between two same-size models, for example + `clip-vit-base-patch32` and `clip-vit-base-patch16`, both 512-dim. + +The state db and the zvec collection are a matched pair. To start over, delete both together: + +```sh +rm -rf cocoindex.db zvec_data +``` + +## Troubleshooting + +- **`ffmpeg not found` / `ffprobe not found`.** Install ffmpeg and make sure it is on your PATH. +- **`ImportError: cannot import name 'zvec' from 'cocoindex.connectors'`.** You ran with system Python. + Activate the venv (`source .venv/bin/activate`) or prefix commands with `.venv/bin/`. +- **A dense query seems to hang.** The first one loads the CLIP model, downloading it on the very first + run. Give it a minute. `--mode fts` skips CLIP. +- **`collection path .../zvec_data/scenes not exist`.** The state db and the collection are out of sync, + usually because one got deleted on its own. Reset both and re-index: `rm -rf cocoindex.db zvec_data`, + then `cocoindex update main.py`. +- **Empty or odd query results after changing models.** Query with the same `CLIP_MODEL` you indexed + under, so the query vector matches the stored size. Otherwise re-index. +- **Not on a Mac.** `make_sample_videos.sh` (the offline fallback) uses macOS `say`. Use + `./download_sample_videos.sh` instead, or add your own clips to `./videos`. + +## Notes + +- Scene IDs come from the video path plus start time, so re-runs line up with previous rows. Moving or + renaming a file changes its id and reprocesses it. +- Both sample scripts write into `./videos`, which is gitignored, so no video binaries live in the repo. diff --git a/examples/video_hybrid_search/download_sample_videos.sh b/examples/video_hybrid_search/download_sample_videos.sh new file mode 100755 index 000000000..69c36e498 --- /dev/null +++ b/examples/video_hybrid_search/download_sample_videos.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Download a small, openly licensed sample corpus into ./videos: two short clips +# from the Blender open movies (CC BY, real scenes with English dialogue), so both +# the visual and transcript sides of the demo have something to match. ffmpeg reads +# each source over HTTP and pulls only the trimmed segment, so the download stays +# small even though the source films are large. See ATTRIBUTION.md for licenses. +# +# Usage: ./download_sample_videos.sh [--force] (--force re-downloads existing clips) +# +# For a no-network fallback, use ./make_sample_videos.sh instead (generated clips). +set -euo pipefail + +cd "$(dirname "$0")" + +for tool in ffmpeg ffprobe; do + command -v "$tool" >/dev/null || { echo "error: '$tool' not found on PATH." >&2; exit 1; } +done + +FORCE=0 +case "${1:-}" in + "") ;; + --force) FORCE=1 ;; + *) echo "usage: $0 [--force]" >&2; exit 2 ;; +esac + +mkdir -p videos + +DUR=22 # seconds per clip + +# name | source URL | trim start (s) +CLIPS=( + "tears_of_steel|https://download.blender.org/demo/movies/ToS/tears_of_steel_720p.mov|25" + "sintel|https://download.blender.org/durian/movies/Sintel.2010.720p.mkv|250" +) + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +for clip in "${CLIPS[@]}"; do + IFS='|' read -r name url start <<<"$clip" + out="videos/$name.mp4" + if [ -f "$out" ] && [ "$FORCE" -eq 0 ]; then + echo "skip $out (exists; --force to redo)" + continue + fi + echo "fetching + trimming $name (${DUR}s from ${start}s)..." + # Trim into a temp file, validate it, and only then move it into videos/, so a + # failed run never leaves a broken clip that a later run would skip. -ss before -i + # seeks over HTTP so only the needed segment is downloaded; reconnect flags ride + # out transient network hiccups. + tmp_out="$TMP/$name.mp4" + ffmpeg -nostdin -v error -y \ + -reconnect 1 -reconnect_streamed 1 -reconnect_delay_max 5 \ + -ss "$start" -t "$DUR" -i "$url" \ + -vf "scale=640:-2" -c:v libx264 -pix_fmt yuv420p -c:a aac -movflags +faststart \ + "$tmp_out" + ffprobe -v error -show_entries format=duration -of csv=p=0 "$tmp_out" >/dev/null + mv "$tmp_out" "$out" + echo "wrote $out" +done + +echo "done. See ATTRIBUTION.md for licenses and credits." diff --git a/examples/video_hybrid_search/main.py b/examples/video_hybrid_search/main.py new file mode 100644 index 000000000..200ed5dd3 --- /dev/null +++ b/examples/video_hybrid_search/main.py @@ -0,0 +1,394 @@ +""" +Video Hybrid Search (v1) - CocoIndex pipeline definition. + +Walk local videos -> segment into fixed-interval scenes -> per scene, embed one +keyframe with CLIP and transcribe the audio slice with faster-whisper -> write one +document per scene into a zvec collection (dense keyframe vector + transcript FTS +field + scalar fields). + +CocoIndex owns the freshness: add a clip and only its scenes process, delete a clip +and its scenes drop, swap the CLIP model and the embedding step reruns while +transcripts stay cached. Querying runs straight against zvec, see `query.py`. + +This module is imported by `query.py` for the CLIP helpers. To index, run: + + cocoindex update main.py +""" + +from __future__ import annotations + +import asyncio +import functools +import io +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated, Any, Iterator + +import numpy as np +from dotenv import load_dotenv +from numpy.typing import NDArray + +import cocoindex as coco +from cocoindex.connectors import localfs, zvec +from cocoindex.resources.file import FileLike, PatternFilePathMatcher +from cocoindex.resources.schema import VectorSchema + +# torch, transformers, and PIL are heavy and only needed for CLIP embedding. Import +# them inside the embedding functions so an FTS-only query (which imports this module +# for its config) does not pay for them. +if TYPE_CHECKING: + import torch + from transformers import CLIPModel, CLIPProcessor + +# Anchor every path to this file's directory, so the state db and the zvec collection +# live next to the example and stay put no matter where you run from. +BASE_DIR = pathlib.Path(__file__).resolve().parent +# Load HF_TOKEN and any model overrides from a local .env if present. +load_dotenv(BASE_DIR / ".env") + +# --- Configuration --------------------------------------------------------- + +ZVEC_DB = coco.ContextKey[zvec.ManagedConnection]("video_hybrid_search_db") +DB_PATH = BASE_DIR / "cocoindex.db" # CocoIndex internal state store +ZVEC_BASE_PATH = BASE_DIR / "zvec_data" +VIDEOS_DIR = BASE_DIR / "videos" +COLLECTION_NAME = "scenes" + +# CLIP gives a shared text/image space. base-patch32 (512-dim) is the light default; +# set CLIP_MODEL=openai/clip-vit-large-patch14 (768-dim) for higher quality. +CLIP_MODEL_NAME = os.getenv("CLIP_MODEL", "openai/clip-vit-base-patch32") +# faster-whisper size: tiny / base / small / medium / large-v3. +WHISPER_MODEL_NAME = os.getenv("WHISPER_MODEL", "base") + +SCENE_SECONDS = 5.0 # fixed-interval segmentation (v1) +KEYFRAME_WIDTH = 384 # downscale keyframes before CLIP + + +@dataclass +class Scene: + id: str # deterministic per (video_path, start) -> zvec document id + video_path: str # scalar filter + start: float # scalar seconds + end: float # scalar seconds + transcript: Annotated[str, zvec.ZvecFtsType()] # FTS over spoken words + # Dense keyframe vector. The VectorSchema (size from the CLIP model) is supplied + # at mount time via column_overrides; metric defaults to cosine. + embedding: NDArray[np.float32] + + +# --- CLIP helpers (shared with query.py) ----------------------------------- + + +@functools.cache +def get_clip_model() -> tuple[CLIPModel, CLIPProcessor]: + from transformers import CLIPModel, CLIPProcessor + + model = CLIPModel.from_pretrained(CLIP_MODEL_NAME) + processor = CLIPProcessor.from_pretrained(CLIP_MODEL_NAME) + return model, processor + + +def _projected_features(out: Any) -> "torch.Tensor": + # transformers >=5 returns BaseModelOutputWithPooling with the projected features + # in pooler_output; transformers <5 returns the projected features tensor directly. + return out.pooler_output if hasattr(out, "pooler_output") else out + + +def embed_query(text: str) -> list[float]: + import torch + + model, processor = get_clip_model() + inputs = processor(text=[text], return_tensors="pt", padding=True) + with torch.no_grad(): + out = model.get_text_features(**inputs) + return _projected_features(out)[0].tolist() + + +def embed_image_bytes(img_bytes: bytes) -> list[float]: + import torch + from PIL import Image + + model, processor = get_clip_model() + image = Image.open(io.BytesIO(img_bytes)).convert("RGB") + inputs = processor(images=image, return_tensors="pt") + with torch.no_grad(): + out = model.get_image_features(**inputs) + return _projected_features(out)[0].tolist() + + +# --- faster-whisper helper ------------------------------------------------- + + +@functools.cache +def get_whisper_model() -> Any: + from faster_whisper import WhisperModel + + return WhisperModel(WHISPER_MODEL_NAME, device="cpu", compute_type="int8") + + +def _transcribe_wav_bytes(wav_bytes: bytes) -> str: + model = get_whisper_model() + segments, _info = model.transcribe(io.BytesIO(wav_bytes), vad_filter=True) + return " ".join(seg.text.strip() for seg in segments).strip() + + +# --- ffmpeg helpers -------------------------------------------------------- + + +def _require_ffmpeg() -> None: + missing = [tool for tool in ("ffmpeg", "ffprobe") if shutil.which(tool) is None] + if missing: + raise RuntimeError( + f"{' and '.join(missing)} not found on PATH. Install ffmpeg first " + "(brew install ffmpeg, or apt install ffmpeg)." + ) + + +def _ffprobe_duration(video_path: str) -> float: + out = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "csv=p=0", + video_path, + ], + capture_output=True, + text=True, + check=True, + ) + return float(out.stdout.strip()) + + +def _extract_keyframe(video_path: str, ts: float) -> bytes: + out = subprocess.run( + [ + "ffmpeg", + "-nostdin", + "-v", + "error", + "-ss", + f"{ts:.3f}", + "-i", + video_path, + "-frames:v", + "1", + "-vf", + f"scale={KEYFRAME_WIDTH}:-1", + "-f", + "image2", + "-c:v", + "mjpeg", + "pipe:1", + ], + capture_output=True, + check=True, + ) + return out.stdout + + +def _has_audio_stream(video_path: str) -> bool: + out = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "a", + "-show_entries", + "stream=index", + "-of", + "csv=p=0", + video_path, + ], + capture_output=True, + text=True, + check=True, + ) + return bool(out.stdout.strip()) + + +def _extract_audio(video_path: str, start: float, dur: float) -> bytes: + out = subprocess.run( + [ + "ffmpeg", + "-nostdin", + "-v", + "error", + "-ss", + f"{start:.3f}", + "-t", + f"{dur:.3f}", + "-i", + video_path, + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-f", + "wav", + "pipe:1", + ], + capture_output=True, + check=True, + ) + return out.stdout + + +def _scene_id(video_path: str, start: float) -> str: + # Deterministic and stable across runs so re-runs match previous rows. zvec + # doc ids allow [A-Za-z0-9._-] only, so sanitize path separators and colons. + return re.sub(r"[^A-Za-z0-9._-]", "_", f"{video_path}_{start:.2f}") + + +def _scene_bounds(duration: float) -> list[tuple[float, float]]: + bounds: list[tuple[float, float]] = [] + start = 0.0 + while start < duration: + end = min(start + SCENE_SECONDS, duration) + bounds.append((start, end)) + start = end + # Merge a sub-second trailing sliver into the previous scene. Otherwise a clip + # whose length is just over a multiple of SCENE_SECONDS makes a near-zero-length + # final scene, and seeking a keyframe at its midpoint runs off the end of the file. + if len(bounds) >= 2 and bounds[-1][1] - bounds[-1][0] < 1.0: + prev_start, _ = bounds[-2] + _, last_end = bounds.pop() + bounds[-1] = (prev_start, last_end) + return bounds + + +# --- Memoized per-scene steps ---------------------------------------------- +# Each expensive stage is its own memoized function, composed via use_mount, so a +# change to one reuses the others. deps= ties the memo to the model name, so +# swapping a model invalidates only that step. + + +@coco.fn(memo=True, deps=CLIP_MODEL_NAME) +async def embed_keyframe(frame_bytes: bytes) -> list[float]: + return await asyncio.to_thread(embed_image_bytes, frame_bytes) + + +@coco.fn(memo=True, deps=WHISPER_MODEL_NAME) +async def transcribe_audio(audio_bytes: bytes) -> str: + return await asyncio.to_thread(_transcribe_wav_bytes, audio_bytes) + + +# --- Per-video processor --------------------------------------------------- + + +@coco.fn(memo=True) +async def process_video( + file: FileLike, + target: zvec.CollectionTarget[Scene], +) -> None: + # Key on the path relative to the videos dir so ids stay short, stable, and + # portable across machines (sourcedir is absolute for cwd-independence). + raw_path = pathlib.Path(str(file.file_path.path)) + try: + video_path = raw_path.relative_to(VIDEOS_DIR).as_posix() + except ValueError: + video_path = raw_path.name + content = await file.read() + + with tempfile.NamedTemporaryFile(suffix=pathlib.Path(video_path).suffix) as tmp: + tmp.write(content) + tmp.flush() + duration = await asyncio.to_thread(_ffprobe_duration, tmp.name) + # Some videos have no audio track. Skip transcription for those so the + # visual side still indexes, instead of failing on the audio extraction. + has_audio = await asyncio.to_thread(_has_audio_stream, tmp.name) + + for start, end in _scene_bounds(duration): + scene_id = _scene_id(video_path, start) + mid = (start + end) / 2.0 + frame_bytes = await asyncio.to_thread(_extract_keyframe, tmp.name, mid) + + embedding = await coco.use_mount( + coco.component_subpath("embed", scene_id), + embed_keyframe, + frame_bytes, + ) + transcript = "" + if has_audio: + audio_bytes = await asyncio.to_thread( + _extract_audio, tmp.name, start, end - start + ) + transcript = await coco.use_mount( + coco.component_subpath("transcribe", scene_id), + transcribe_audio, + audio_bytes, + ) + + target.declare_row( + row=Scene( + id=scene_id, + video_path=video_path, + start=start, + end=end, + transcript=transcript, + embedding=np.asarray(embedding, dtype=np.float32), + ) + ) + + +# --- App wiring ------------------------------------------------------------ + + +@coco.lifespan +def coco_lifespan(builder: coco.EnvironmentBuilder) -> Iterator[None]: + # Pin the state db here so it does not depend on COCOINDEX_DB or a stray .env. + builder.settings.db_path = DB_PATH + with zvec.managed_connection(ZVEC_BASE_PATH) as conn: + builder.provide(ZVEC_DB, conn) + yield + + +@coco.fn +async def app_main(sourcedir: pathlib.Path) -> None: + _require_ffmpeg() + + model, _ = get_clip_model() + dim: int = model.config.projection_dim # type: ignore[assignment] + + target = await zvec.mount_collection_target( + ZVEC_DB, + COLLECTION_NAME, + await zvec.CollectionSchema.from_class( + Scene, + primary_key=["id"], + column_overrides={ + "embedding": VectorSchema(dtype=np.dtype(np.float32), size=dim) + }, + ), + ) + + files = localfs.walk_dir( + sourcedir, + recursive=True, + path_matcher=PatternFilePathMatcher( + included_patterns=["**/*.mp4", "**/*.mov", "**/*.mkv", "**/*.webm"] + ), + live=True, + ) + await coco.mount_each(process_video, files.items(), target) + + +app = coco.App( + coco.AppConfig(name="VideoHybridSearchV1"), + app_main, + sourcedir=VIDEOS_DIR, +) + + +if __name__ == "__main__": + app.update_blocking(report_to_stdout=True) diff --git a/examples/video_hybrid_search/make_sample_videos.sh b/examples/video_hybrid_search/make_sample_videos.sh new file mode 100755 index 000000000..ce708ce53 --- /dev/null +++ b/examples/video_hybrid_search/make_sample_videos.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Offline fallback corpus: short clips with real speech (so the transcript FTS +# field is exercised) over solid-color backgrounds. Uses macOS `say` for +# text-to-speech, so it only runs on a Mac. The visuals are blank, so this is a +# deterministic pipeline check, not a visual-search demo. +# +# For real footage, use ./download_sample_videos.sh instead. The pipeline just +# walks whatever videos land in ./videos. +set -euo pipefail + +cd "$(dirname "$0")" +mkdir -p videos +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +gen() { + local name="$1" bg="$2" text="$3" + say -o "$TMP/$name.aiff" "$text" + ffmpeg -nostdin -v error -y \ + -f lavfi -i "$bg" \ + -i "$TMP/$name.aiff" \ + -shortest -c:v libx264 -pix_fmt yuv420p -c:a aac \ + "videos/$name.mp4" + echo "wrote videos/$name.mp4" +} + +gen dog_park "color=c=green:s=640x360:r=15" \ + "A dog runs across the green park chasing a bright red ball in the sunshine. \ + Two children laugh and clap as the puppy leaps into the air. \ + Later the dog rests under a tall oak tree beside the pond." +gen kitchen "testsrc2=s=640x360:r=15" \ + "The chef slices fresh tomatoes and onions on a wooden board in the kitchen. \ + She heats olive oil in a pan and stirs in garlic and basil. \ + The sauce simmers slowly while bread bakes in the oven." +gen revenue "color=c=navy:s=640x360:r=15" \ + "The analyst explains quarterly revenue growth and profit margins on a chart. \ + Sales rose fifteen percent while operating costs stayed flat. \ + The team reviews the forecast for the next fiscal year." diff --git a/examples/video_hybrid_search/pyproject.toml b/examples/video_hybrid_search/pyproject.toml new file mode 100644 index 000000000..52e250798 --- /dev/null +++ b/examples/video_hybrid_search/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "video-hybrid-search-v1" +version = "0.1.0" +description = "CocoIndex v1 example: incremental video indexing into a zvec hybrid store (keyframe vector + transcript FTS + scalar), queried directly." +requires-python = ">=3.11" +dependencies = [ + # 1.0.14 is the first release with the zvec FTS connector (ZvecFtsType). + "cocoindex[zvec]>=1.0.14", + "torch>=2.0.0", + "transformers>=4.29.0", + "pillow>=10.0.0", + "numpy>=1.24.0", + "faster-whisper>=1.0.0", + "python-dotenv>=1.0.1", +] + +[tool.setuptools] +packages = [] diff --git a/examples/video_hybrid_search/query.py b/examples/video_hybrid_search/query.py new file mode 100644 index 000000000..79bd092f8 --- /dev/null +++ b/examples/video_hybrid_search/query.py @@ -0,0 +1,122 @@ +""" +Hybrid query over the zvec scene collection. + +Runs straight against zvec (querying stays outside CocoIndex, which only writes). +Combines three signals in one call: + + - dense: CLIP text embedding vs the keyframe vector + - fts: full-text match over the transcript + - scalar: a boolean filter over start / end / video_path + +A reranker (RRF by default) fuses the dense and fts sub-queries. + +Examples (against the downloaded sample corpus): + + python query.py "two people talking on a bridge" --fts "robotics" + python query.py "a red-haired girl" --mode dense + python query.py "" --mode fts --fts "robot hand" + python query.py "two people talking on a bridge" --filter "start < 15" +""" + +from __future__ import annotations + +import argparse + +import zvec # the zvec package: Query / Fts / RrfReRanker +from cocoindex.connectors import zvec as coco_zvec # the connector: managed_connection + +import main # CLIP helpers + collection config + + +def _run( + text: str, + fts: str | None, + scalar_filter: str | None, + topk: int, + mode: str, +) -> None: + queries: list[zvec.Query] = [] + if mode in ("hybrid", "dense") and text: + # Loads the CLIP model on the first dense query, which can take a while + # (model download on the very first run, then load from cache). + print("Loading the CLIP model to embed the query...", flush=True) + queries.append( + zvec.Query(field_name="embedding", vector=main.embed_query(text)) + ) + if mode in ("hybrid", "fts"): + match = fts if fts is not None else text + if match: + queries.append( + zvec.Query(field_name="transcript", fts=zvec.Fts(match_string=match)) + ) + + if not queries: + raise SystemExit("Nothing to query: provide query text and/or --fts.") + + # A reranker only applies when fusing more than one sub-query. + reranker = zvec.RrfReRanker() if len(queries) > 1 else None + + if not (main.ZVEC_BASE_PATH / main.COLLECTION_NAME).exists(): + raise SystemExit( + f"No collection at {main.ZVEC_BASE_PATH / main.COLLECTION_NAME}. " + "Index the videos first with: cocoindex update main.py" + ) + + print("Opening the zvec collection...", flush=True) + with coco_zvec.managed_connection(main.ZVEC_BASE_PATH) as conn: + col = conn.open_existing(main.COLLECTION_NAME) + results = col.query( + queries=queries, + topk=topk, + filter=scalar_filter, + reranker=reranker, + output_fields=["video_path", "start", "end", "transcript"], + ) + + if not results: + print("No matches.") + return + + for i, doc in enumerate(results, 1): + f = doc.fields or {} + start = f.get("start") + end = f.get("end") + span = f"{start:.1f}-{end:.1f}s" if start is not None else "?" + transcript = (f.get("transcript") or "").strip() + snippet = transcript[:100] + ("..." if len(transcript) > 100 else "") + print(f"{i}. [{doc.score:.4f}] {f.get('video_path')} @ {span}") + if snippet: + print(f" {snippet}") + + +def main_cli() -> None: + p = argparse.ArgumentParser( + description="Hybrid search over the zvec scene collection." + ) + p.add_argument( + "query", help="Natural-language query (dense side). Use '' for fts-only." + ) + p.add_argument( + "--fts", default=None, help="FTS match string. Defaults to the query text." + ) + p.add_argument( + "--filter", + dest="scalar_filter", + default=None, + help='zvec scalar filter, e.g. "start >= 10 and end < 60".', + ) + p.add_argument( + "--topk", type=int, default=10, help="Number of results (default 10)." + ) + p.add_argument( + "--mode", + choices=["hybrid", "dense", "fts"], + default="hybrid", + help="Which signals to use (default hybrid).", + ) + args = p.parse_args() + _run(args.query, args.fts, args.scalar_filter, args.topk, args.mode) + + +if __name__ == "__main__": + main_cli() diff --git a/examples/video_hybrid_search/videos/.gitkeep b/examples/video_hybrid_search/videos/.gitkeep new file mode 100644 index 000000000..0a922c1f8 --- /dev/null +++ b/examples/video_hybrid_search/videos/.gitkeep @@ -0,0 +1 @@ +# Drop your video clips here, or run ./make_sample_videos.sh to generate a sample set.