From b5c7f903e8d84ad99d9be9944c0d34f92c44895d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 13:18:21 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(tools):=20add=20vdl=20=E2=80=94=20hard?= =?UTF-8?q?ened=20CLI=20video/audio=20downloader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wraps yt-dlp with the operational layer it does not provide: classified errors driving retry policy, bounded concurrency, partial-failure isolation, and structured run telemetry. - config.py frozen DownloadConfig, validated once at the boundary - formats.py format selectors that degrade progressively so single-stream sites still resolve instead of erroring on a hard constraint - errors.py taxonomy + retry classification + stable exit codes - batch.py thread-pool execution; one failed URL never stops the run - telemetry.py HEALTH / PRESSURE / EFFICIENCY, JSON to stderr - ffmpeg.py system ffmpeg, falling back to the imageio-ffmpeg binary so merging and audio extraction work with no system packages Retry policy is the load-bearing detail: yt-dlp reports a permanent 404 and a timed-out socket with the same "Unable to download webpage" prefix, so the classifier parses HTTP status out of the message and retries only what can succeed. Measured on a dead URL: 10s of pointless backoff down to 1.6s. Verified end to end against CC-licensed Blender demo media — download, mp3 extraction, container remux, batch with an intentional failure, and telemetry output. 79 tests, no network required (yt_dlp.YoutubeDL is stubbed). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01US2uXf9esJvHouvwvvUxCB --- tools/video-downloader/.gitignore | 6 + tools/video-downloader/README.md | 169 ++++++++++++++ tools/video-downloader/pyproject.toml | 30 +++ tools/video-downloader/requirements.txt | 2 + tools/video-downloader/tests/test_cli.py | 68 ++++++ tools/video-downloader/tests/test_config.py | 60 +++++ .../video-downloader/tests/test_downloader.py | 157 +++++++++++++ tools/video-downloader/tests/test_errors.py | 70 ++++++ tools/video-downloader/tests/test_formats.py | 51 +++++ tools/video-downloader/tests/test_options.py | 91 ++++++++ tools/video-downloader/vdl/__init__.py | 12 + tools/video-downloader/vdl/__main__.py | 6 + tools/video-downloader/vdl/batch.py | 76 +++++++ tools/video-downloader/vdl/cli.py | 215 ++++++++++++++++++ tools/video-downloader/vdl/config.py | 83 +++++++ tools/video-downloader/vdl/downloader.py | 95 ++++++++ tools/video-downloader/vdl/errors.py | 117 ++++++++++ tools/video-downloader/vdl/ffmpeg.py | 39 ++++ tools/video-downloader/vdl/formats.py | 63 +++++ tools/video-downloader/vdl/options.py | 132 +++++++++++ tools/video-downloader/vdl/progress.py | 107 +++++++++ tools/video-downloader/vdl/telemetry.py | 108 +++++++++ 22 files changed, 1757 insertions(+) create mode 100644 tools/video-downloader/.gitignore create mode 100644 tools/video-downloader/README.md create mode 100644 tools/video-downloader/pyproject.toml create mode 100644 tools/video-downloader/requirements.txt create mode 100644 tools/video-downloader/tests/test_cli.py create mode 100644 tools/video-downloader/tests/test_config.py create mode 100644 tools/video-downloader/tests/test_downloader.py create mode 100644 tools/video-downloader/tests/test_errors.py create mode 100644 tools/video-downloader/tests/test_formats.py create mode 100644 tools/video-downloader/tests/test_options.py create mode 100644 tools/video-downloader/vdl/__init__.py create mode 100644 tools/video-downloader/vdl/__main__.py create mode 100644 tools/video-downloader/vdl/batch.py create mode 100644 tools/video-downloader/vdl/cli.py create mode 100644 tools/video-downloader/vdl/config.py create mode 100644 tools/video-downloader/vdl/downloader.py create mode 100644 tools/video-downloader/vdl/errors.py create mode 100644 tools/video-downloader/vdl/ffmpeg.py create mode 100644 tools/video-downloader/vdl/formats.py create mode 100644 tools/video-downloader/vdl/options.py create mode 100644 tools/video-downloader/vdl/progress.py create mode 100644 tools/video-downloader/vdl/telemetry.py diff --git a/tools/video-downloader/.gitignore b/tools/video-downloader/.gitignore new file mode 100644 index 0000000..c070ffa --- /dev/null +++ b/tools/video-downloader/.gitignore @@ -0,0 +1,6 @@ +downloads/ +*.part +*.ytdl +*.egg-info/ +__pycache__/ +.pytest_cache/ diff --git a/tools/video-downloader/README.md b/tools/video-downloader/README.md new file mode 100644 index 0000000..2c7ca33 --- /dev/null +++ b/tools/video-downloader/README.md @@ -0,0 +1,169 @@ +# vdl — video downloader + +A hardened command-line downloader for video and audio, built on +[yt-dlp](https://github.com/yt-dlp/yt-dlp) (1800+ supported sites). + +yt-dlp does the extraction. `vdl` adds the parts you actually need on top of it: +classified errors with correct retry policy, bounded concurrency for batches, +partial-failure isolation, structured telemetry, and a CLI you can remember. + +--- + +## Install + +```bash +cd tools/video-downloader +pip install -r requirements.txt # yt-dlp + a bundled ffmpeg binary +python -m vdl --help +``` + +Optionally install it as a real command: + +```bash +pip install -e . +vdl --help +``` + +`ffmpeg` is needed for merging split streams, converting containers, extracting +audio, and tagging. `vdl` uses a system `ffmpeg` if one is on `PATH`; otherwise +it falls back to the binary shipped by `imageio-ffmpeg`, so the default install +works with no system packages. If neither is present it warns up front instead +of failing halfway through. + +--- + +## Usage + +```bash +vdl URL # best available quality +vdl URL -q 1080 -c mp4 # cap at 1080p, force mp4 +vdl URL --audio-only # extract audio as mp3 +vdl URL --audio-only --audio-codec flac # lossless audio +vdl URL --subs en --embed-subs # fetch + mux English subtitles +vdl URL --thumbnail # embed cover art +vdl URL --section 00:01:30-00:04:00 # clip a time range +vdl URL --info # list formats, download nothing +``` + +### Batches and playlists + +```bash +vdl -f urls.txt -j 5 # 5 at a time from a URL list +vdl URL --playlist # whole playlist, not just the video +vdl URL --playlist --items 1-10 # first ten entries +vdl -f urls.txt --archive done.txt # skip anything already downloaded +``` + +`urls.txt` is one URL per line; blank lines and `#` comments are ignored. + +### Network + +```bash +vdl URL -r 4M # throttle to 4 MB/s +vdl URL --retries 5 # more attempts on transport failure +vdl URL --cookies-from-browser firefox # use your browser session +vdl URL --cookies cookies.txt # or a Netscape cookie file +``` + +--- + +## Options + +| Flag | Meaning | +|---|---| +| `-o, --output DIR` | output directory (default `./downloads`) | +| `-q, --quality` | `best`, `2160`, `1440`, `1080`, `720`, `480`, `360`, `worst` | +| `-c, --container` | `auto`, `mp4`, `mkv`, `webm` — `auto` avoids re-encoding | +| `-a, --audio-only` | extract audio only | +| `--audio-codec` | `mp3`, `m4a`, `opus`, `wav`, `flac` | +| `--subs LANG...` | download subtitle languages | +| `--embed-subs` | mux subtitles into the file | +| `--thumbnail` | embed cover art | +| `--no-metadata` | skip metadata tagging | +| `--section START-END` | clip a time range | +| `-p, --playlist` | expand playlists | +| `--items SPEC` | playlist entries, e.g. `1-10` or `2,4,7` | +| `--archive FILE` | remember completed IDs and skip them | +| `-j, --concurrency N` | parallel downloads (default 3) | +| `--fragments N` | parallel fragments per download (default 4) | +| `-r, --rate-limit` | throttle, e.g. `4M`, `800K` | +| `--retries N` | retries per URL on transport failure (default 3) | +| `--cookies FILE` / `--cookies-from-browser` | authenticated sessions | +| `--name TEMPLATE` | filename template (yt-dlp syntax) | +| `--overwrite` / `--no-resume` | re-download policy | +| `--info` | list available formats and exit | +| `--simulate` | full run without writing files | +| `--json-logs` | structured telemetry to stderr | +| `--quiet` | suppress progress | + +--- + +## Design notes + +**Errors are classified, and classification drives retries.** yt-dlp reports a +dead 404 and a timed-out socket with the same `Unable to download webpage` +prefix. Retrying the first is pure waste. `vdl` parses the HTTP status out of +the message and only retries what can actually succeed — transport failures, +429s, and 5xx — with exponential backoff. DRM walls, private videos, and 404s +fail immediately with a message naming the real cause. + +| Exit code | Meaning | +|---|---| +| `0` | all downloads succeeded | +| `1` | at least one item failed | +| `2` | invalid configuration | +| `3` | unsupported URL | +| `4` | protected content (DRM, private, login required) | +| `5` | network failure after all retries | +| `6` | ffmpeg merge/postprocessing failure | +| `7` | content not found (permanent 4xx) | + +**One failure never kills a batch.** Each URL is independent; the run reports +partial success and exits `1`. Interrupting with Ctrl-C keeps partial files, +which the next run resumes. + +**Every run explains itself.** `--json-logs` emits one JSON object per event to +stderr — separate from the progress UI on stdout, so `vdl ... 2> run.jsonl` +gives a clean machine-readable log covering health (items ok/failed/skipped), +pressure (elapsed, throughput, items/min), and efficiency (bytes, retry +overhead, average item time). + +**Format selection degrades progressively.** The `-f` expression falls back from +capped split streams → capped progressive → anything available, so a site +offering a single muxed stream still succeeds instead of erroring on an +unsatisfiable constraint. + +--- + +## Layout + +``` +tools/video-downloader/ +├── vdl/ +│ ├── cli.py argument parsing, output formatting, exit codes +│ ├── config.py DownloadConfig — validated once, frozen +│ ├── formats.py yt-dlp format selector construction +│ ├── options.py DownloadConfig → yt-dlp options dict +│ ├── downloader.py single URL, retry policy, error classification +│ ├── batch.py concurrent execution, failure isolation +│ ├── progress.py terminal progress rendering +│ ├── telemetry.py HEALTH / PRESSURE / EFFICIENCY signals +│ ├── ffmpeg.py ffmpeg discovery with bundled fallback +│ └── errors.py error taxonomy, retry classification, exit codes +└── tests/ 79 tests, no network access required +``` + +Run the suite with `python -m pytest` from `tools/video-downloader`. The tests +replace `yt_dlp.YoutubeDL` with a scripted stub, so retry policy, classification +and concurrency are verified offline. + +--- + +## Scope + +`vdl` downloads what a site serves to an ordinary client. It does not break DRM, +defeat paywalls, or bypass authentication — for login-gated content it reuses +cookies *you* already hold via `--cookies-from-browser`. + +Download only what you own, what is offered for download, or what its licence +permits. Site terms of service and copyright still apply. diff --git a/tools/video-downloader/pyproject.toml b/tools/video-downloader/pyproject.toml new file mode 100644 index 0000000..c91ed0e --- /dev/null +++ b/tools/video-downloader/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "vdl" +version = "1.0.0" +description = "Hardened CLI video/audio downloader built on yt-dlp" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +dependencies = [ + "yt-dlp>=2024.1.1", +] + +[project.optional-dependencies] +# Ships a static ffmpeg binary so merging and audio extraction work without a +# system-wide install. +ffmpeg = ["imageio-ffmpeg>=0.4.9"] +dev = ["pytest>=7.4"] + +[project.scripts] +vdl = "vdl.cli:main" + +[tool.setuptools] +packages = ["vdl"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] diff --git a/tools/video-downloader/requirements.txt b/tools/video-downloader/requirements.txt new file mode 100644 index 0000000..f9fdf23 --- /dev/null +++ b/tools/video-downloader/requirements.txt @@ -0,0 +1,2 @@ +yt-dlp>=2024.1.1 +imageio-ffmpeg>=0.4.9 diff --git a/tools/video-downloader/tests/test_cli.py b/tools/video-downloader/tests/test_cli.py new file mode 100644 index 0000000..e0fd5c7 --- /dev/null +++ b/tools/video-downloader/tests/test_cli.py @@ -0,0 +1,68 @@ +from pathlib import Path + +import pytest + +from vdl import cli +from vdl.errors import ConfigError + + +def parse(argv): + return cli.build_parser().parse_args(argv) + + +def test_minimal_invocation(): + config = cli.to_config(parse(["https://e.com/a"])) + assert config.urls == ("https://e.com/a",) + assert config.output_dir == Path("downloads") + + +def test_flags_map_onto_config(): + config = cli.to_config( + parse(["https://e.com/a", "-q", "720", "-c", "mp4", "-o", "/tmp/out", + "-j", "5", "-r", "2M", "--subs", "en", "--embed-subs", "--thumbnail"]) + ) + assert (config.quality, config.container) == ("720", "mp4") + assert config.output_dir == Path("/tmp/out") + assert config.concurrency == 5 + assert config.rate_limit == "2M" + assert config.subtitles == ("en",) + assert config.embed_subtitles and config.embed_thumbnail + + +def test_duplicate_urls_collapse_preserving_order(): + config = cli.to_config(parse(["https://e.com/b", "https://e.com/a", "https://e.com/b"])) + assert config.urls == ("https://e.com/b", "https://e.com/a") + + +def test_url_file_entries_are_appended(tmp_path): + listing = tmp_path / "urls.txt" + listing.write_text("https://e.com/x\n# note\nhttps://e.com/y\n") + config = cli.to_config(parse(["https://e.com/a", "-f", str(listing)])) + assert config.urls == ("https://e.com/a", "https://e.com/x", "https://e.com/y") + + +def test_missing_url_file_is_a_config_error(tmp_path): + with pytest.raises(ConfigError, match="not found"): + cli.to_config(parse(["-f", str(tmp_path / "nope.txt")])) + + +def test_items_implies_playlist_mode(): + config = cli.to_config(parse(["https://e.com/list", "--items", "1-10"])) + assert config.playlist is True + assert config.playlist_items == "1-10" + + +def test_no_urls_exits_with_config_code(capsys): + assert cli.main([]) == ConfigError.exit_code + assert "no URLs" in capsys.readouterr().err + + +def test_conflicting_flags_exit_with_config_code(capsys): + assert cli.main(["https://e.com/a", "--audio-only", "-c", "mp4"]) == ConfigError.exit_code + assert "audio-only" in capsys.readouterr().err + + +def test_help_lists_every_group(): + text = cli.build_parser().format_help() + for group in ("quality", "extras", "playlists", "network", "behaviour"): + assert group in text diff --git a/tools/video-downloader/tests/test_config.py b/tools/video-downloader/tests/test_config.py new file mode 100644 index 0000000..1a41314 --- /dev/null +++ b/tools/video-downloader/tests/test_config.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import pytest + +from vdl.config import DownloadConfig +from vdl.errors import ConfigError + + +def make(**kwargs) -> DownloadConfig: + return DownloadConfig(urls=("https://example.com/v",), **kwargs) + + +def test_defaults_are_valid(): + config = make() + assert config.quality == "best" + assert config.container == "auto" + assert config.resume is True + assert config.output_dir == Path("downloads") + + +def test_rejects_empty_url_list(): + with pytest.raises(ConfigError, match="no URLs"): + DownloadConfig(urls=()) + + +@pytest.mark.parametrize( + "kwargs, message", + [ + ({"quality": "8k"}, "quality"), + ({"container": "avi"}, "container"), + ({"audio_codec": "aac"}, "audio codec"), + ({"concurrency": 0}, "concurrency"), + ({"fragment_concurrency": 0}, "fragment concurrency"), + ({"retries": -1}, "retries"), + ], +) +def test_rejects_invalid_values(kwargs, message): + with pytest.raises(ConfigError, match=message): + make(**kwargs) + + +def test_audio_only_conflicts_with_container(): + with pytest.raises(ConfigError, match="audio-only"): + make(audio_only=True, container="mp4") + + +def test_embed_subs_requires_languages(): + with pytest.raises(ConfigError, match="requires --subs"): + make(embed_subtitles=True) + + +def test_cookie_sources_are_mutually_exclusive(): + with pytest.raises(ConfigError, match="not both"): + make(cookie_file=Path("c.txt"), cookies_from_browser="firefox") + + +def test_config_is_immutable(): + config = make() + with pytest.raises(Exception): + config.quality = "720" # type: ignore[misc] diff --git a/tools/video-downloader/tests/test_downloader.py b/tools/video-downloader/tests/test_downloader.py new file mode 100644 index 0000000..a694e22 --- /dev/null +++ b/tools/video-downloader/tests/test_downloader.py @@ -0,0 +1,157 @@ +"""Downloader and batch behaviour, exercised against a fake yt-dlp. + +No network access: `yt_dlp.YoutubeDL` is replaced with a stub whose scripted +outcomes let us assert retry, classification, and concurrency policy. +""" + +from __future__ import annotations + +import sys +import types + +import pytest + +from vdl import batch, downloader, errors +from vdl.config import DownloadConfig + + +class FakeYoutubeDL: + """Stands in for yt_dlp.YoutubeDL. Outcomes are popped from a script.""" + + script: list = [] + calls: list = [] + + def __init__(self, opts): + self.opts = opts + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def extract_info(self, url, download=True): + FakeYoutubeDL.calls.append((url, download)) + outcome = FakeYoutubeDL.script.pop(0) + if isinstance(outcome, Exception): + raise outcome + # Drive the progress hook so byte accounting is exercised too. + for hook in self.opts.get("progress_hooks", []): + hook({"status": "downloading", "downloaded_bytes": 512, + "total_bytes": 1024, "info_dict": outcome}) + hook({"status": "finished", "downloaded_bytes": 1024, "info_dict": outcome}) + return outcome + + +@pytest.fixture(autouse=True) +def fake_ytdlp(monkeypatch): + module = types.ModuleType("yt_dlp") + module.YoutubeDL = FakeYoutubeDL + utils = types.ModuleType("yt_dlp.utils") + utils.parse_duration = lambda text: float(text) if text else None + module.utils = utils + monkeypatch.setitem(sys.modules, "yt_dlp", module) + monkeypatch.setitem(sys.modules, "yt_dlp.utils", utils) + monkeypatch.setattr("time.sleep", lambda _: None) # collapse backoff + FakeYoutubeDL.script = [] + FakeYoutubeDL.calls = [] + yield + + +def info(title="Clip", path="/tmp/clip.mp4"): + return {"title": title, "id": "abc123", + "requested_downloads": [{"filepath": path}]} + + +def make(tmp_path, **kwargs) -> DownloadConfig: + kwargs.setdefault("urls", ("https://example.com/v1",)) + kwargs.setdefault("output_dir", tmp_path) + kwargs.setdefault("quiet", True) + return DownloadConfig(**kwargs) + + +def test_successful_download_reports_path_and_bytes(tmp_path): + FakeYoutubeDL.script = [info()] + result = downloader.download("https://example.com/v1", make(tmp_path)) + assert result.status == "ok" + assert result.title == "Clip" + assert result.filepath == "/tmp/clip.mp4" + assert result.bytes_downloaded == 1024 + assert result.attempts == 1 + + +def test_network_failure_retries_then_succeeds(tmp_path): + FakeYoutubeDL.script = [Exception("connection reset by peer"), info()] + result = downloader.download("https://example.com/v1", make(tmp_path, retries=3)) + assert result.status == "ok" + assert result.attempts == 2 + + +def test_retries_are_bounded(tmp_path): + FakeYoutubeDL.script = [Exception("connection reset") for _ in range(10)] + result = downloader.download("https://example.com/v1", make(tmp_path, retries=2)) + assert result.status == "failed" + assert result.attempts == 3 # initial attempt + 2 retries + assert result.error_class == "NetworkError" + + +def test_protected_content_fails_without_retrying(tmp_path): + FakeYoutubeDL.script = [Exception("This video is DRM protected"), info()] + result = downloader.download("https://example.com/v1", make(tmp_path, retries=5)) + assert result.status == "failed" + assert result.error_class == "ProtectedContentError" + assert result.attempts == 1 + assert len(FakeYoutubeDL.calls) == 1 + + +def test_none_result_is_treated_as_failure(tmp_path): + FakeYoutubeDL.script = [None] + result = downloader.download("https://example.com/v1", make(tmp_path)) + assert result.status == "failed" + + +def test_probe_does_not_download(tmp_path): + FakeYoutubeDL.script = [info()] + downloader.probe("https://example.com/v1", make(tmp_path)) + assert FakeYoutubeDL.calls == [("https://example.com/v1", False)] + + +def test_probe_raises_classified_error(tmp_path): + FakeYoutubeDL.script = [Exception("Unsupported URL: https://example.com/v1")] + with pytest.raises(errors.UnsupportedUrlError): + downloader.probe("https://example.com/v1", make(tmp_path)) + + +def test_batch_continues_past_a_failure(tmp_path): + FakeYoutubeDL.script = [info("A"), Exception("This video is private"), info("C")] + config = make( + tmp_path, + urls=("https://e.com/a", "https://e.com/b", "https://e.com/c"), + concurrency=1, + ) + metrics = batch.run(config) + assert metrics.succeeded == 2 + assert metrics.failed == 1 + assert len(metrics.items) == 3 + + +def test_batch_creates_output_directory(tmp_path): + FakeYoutubeDL.script = [info()] + target = tmp_path / "nested" / "clips" + batch.run(make(tmp_path, output_dir=target)) + assert target.is_dir() + + +def test_metrics_snapshot_exposes_required_signals(tmp_path): + FakeYoutubeDL.script = [info()] + snapshot = batch.run(make(tmp_path)).snapshot() + assert set(snapshot) == {"health", "pressure", "efficiency", "items"} + assert snapshot["health"]["items_ok"] == 1 + assert snapshot["efficiency"]["bytes_downloaded"] == 1024 + assert snapshot["pressure"]["throughput_bps"] >= 0 + + +def test_url_file_ignores_comments_and_blanks(tmp_path): + listing = tmp_path / "urls.txt" + listing.write_text("# batch\n\nhttps://e.com/a\n https://e.com/b \n\n") + assert batch.read_url_file(listing) == ["https://e.com/a", "https://e.com/b"] diff --git a/tools/video-downloader/tests/test_errors.py b/tools/video-downloader/tests/test_errors.py new file mode 100644 index 0000000..eed239d --- /dev/null +++ b/tools/video-downloader/tests/test_errors.py @@ -0,0 +1,70 @@ +import pytest + +from vdl import errors + + +@pytest.mark.parametrize( + "message, expected", + [ + ("ERROR: This video is DRM protected", errors.ProtectedContentError), + ("This video is private", errors.ProtectedContentError), + ("Join this channel to get access to members-only content", + errors.ProtectedContentError), + ("Sign in to confirm you're not a bot", errors.ProtectedContentError), + ("Unsupported URL: https://example.com/x", errors.UnsupportedUrlError), + ("Unable to download webpage: The read operation timed out", errors.NetworkError), + ("[generic] gone: Unable to download webpage: HTTP Error 404: Not Found", + errors.NotFoundError), + ("Unable to download webpage: HTTP Error 410: Gone", errors.NotFoundError), + ("Unable to download webpage: HTTP Error 451: Unavailable For Legal Reasons", + errors.NotFoundError), + ("Unable to download webpage: HTTP Error 403: Forbidden", + errors.ProtectedContentError), + ("Unable to download webpage: HTTP Error 401: Unauthorized", + errors.ProtectedContentError), + ("Unable to download webpage: HTTP Error 429: Too Many Requests", + errors.NetworkError), + ("Unable to download webpage: HTTP Error 503: Service Unavailable", + errors.NetworkError), + ("Connection reset by peer", errors.NetworkError), + ("ffmpeg exited with code 1", errors.MergeError), + ("something entirely new", errors.VdlError), + ], +) +def test_classification(message, expected): + assert errors.classify(message) is expected + + +def test_classification_is_case_insensitive(): + assert errors.classify("VIDEO UNAVAILABLE") is errors.ProtectedContentError + + +def test_protected_content_beats_network_marker(): + # A DRM wall reported alongside a transport hiccup must not be retried. + message = "This video is DRM protected; connection reset" + assert errors.classify(message) is errors.ProtectedContentError + + +def test_only_network_errors_retry(): + assert errors.is_retryable(errors.NetworkError("x")) is True + assert errors.is_retryable(errors.ProtectedContentError("x")) is False + assert errors.is_retryable(errors.MergeError("x")) is False + assert errors.is_retryable(errors.NotFoundError("x")) is False + assert errors.is_retryable(errors.VdlError("x")) is False + + +def test_permanent_status_wins_over_generic_network_marker(): + # yt-dlp wraps a 404 in "Unable to download webpage", which is also a + # transient marker. The permanent status must win or we retry a dead URL. + message = "Unable to download webpage: HTTP Error 404: Not Found" + assert errors.classify(message) is errors.NotFoundError + assert errors.is_retryable(errors.classify(message)("x")) is False + + +def test_exit_codes_are_distinct(): + classes = [ + errors.VdlError, errors.ConfigError, errors.UnsupportedUrlError, + errors.ProtectedContentError, errors.NetworkError, errors.MergeError, + errors.NotFoundError, + ] + assert len({c.exit_code for c in classes}) == len(classes) diff --git a/tools/video-downloader/tests/test_formats.py b/tools/video-downloader/tests/test_formats.py new file mode 100644 index 0000000..edc3945 --- /dev/null +++ b/tools/video-downloader/tests/test_formats.py @@ -0,0 +1,51 @@ +from vdl import formats +from vdl.config import DownloadConfig + + +def make(**kwargs) -> DownloadConfig: + return DownloadConfig(urls=("https://example.com/v",), **kwargs) + + +def test_best_quality_has_no_height_cap(): + selector = formats.video_selector(make()) + assert "height" not in selector + assert selector.startswith("bestvideo+bestaudio") + + +def test_quality_cap_applies_to_every_video_candidate(): + selector = formats.video_selector(make(quality="720")) + candidates = selector.split("/") + assert all("[height<=720]" in c for c in candidates if c != "best") + + +def test_container_preference_is_expressed_before_fallback(): + selector = formats.video_selector(make(quality="1080", container="mp4")) + assert selector.startswith("bestvideo[height<=1080][ext=mp4]+bestaudio[ext=mp4]") + # A bare fallback must remain so sites without mp4 still resolve. + assert selector.endswith("best") + + +def test_selector_candidates_are_deduplicated(): + selector = formats.video_selector(make()) + candidates = selector.split("/") + assert len(candidates) == len(set(candidates)) + + +def test_worst_quality_is_its_own_path(): + assert formats.video_selector(make(quality="worst")) == "worstvideo+worstaudio/worst" + + +def test_audio_only_ignores_video_selector(): + assert formats.selector(make(audio_only=True)) == "bestaudio/best" + + +def test_merge_container_none_when_auto(): + assert formats.merge_container(make()) is None + + +def test_merge_container_set_when_explicit(): + assert formats.merge_container(make(container="mkv")) == "mkv" + + +def test_merge_container_none_for_audio_only(): + assert formats.merge_container(make(audio_only=True, audio_codec="opus")) is None diff --git a/tools/video-downloader/tests/test_options.py b/tools/video-downloader/tests/test_options.py new file mode 100644 index 0000000..8371edb --- /dev/null +++ b/tools/video-downloader/tests/test_options.py @@ -0,0 +1,91 @@ +from pathlib import Path + +import pytest + +from vdl import options +from vdl.config import DownloadConfig + + +def make(**kwargs) -> DownloadConfig: + return DownloadConfig(urls=("https://example.com/v",), **kwargs) + + +def keys(opts, key): + return [p["key"] for p in opts["postprocessors"]] if key == "pp" else opts[key] + + +def test_output_template_lands_under_output_dir(): + opts = options.build(make(output_dir=Path("/tmp/clips"))) + assert opts["outtmpl"]["default"].startswith("/tmp/clips/") + assert opts["paths"]["home"] == "/tmp/clips" + + +def test_single_video_does_not_expand_playlist_by_default(): + assert options.build(make())["noplaylist"] is True + assert options.build(make(playlist=True))["noplaylist"] is False + + +def test_audio_only_adds_extract_audio_postprocessor(): + opts = options.build(make(audio_only=True, audio_codec="opus")) + extract = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegExtractAudio") + assert extract["preferredcodec"] == "opus" + + +def test_metadata_postprocessor_is_on_by_default_and_removable(): + assert "FFmpegMetadata" in keys(options.build(make()), "pp") + assert "FFmpegMetadata" not in keys(options.build(make(embed_metadata=False)), "pp") + + +def test_subtitles_request_both_manual_and_auto_tracks(): + opts = options.build(make(subtitles=("en", "es"))) + assert opts["writesubtitles"] is True + assert opts["writeautomaticsub"] is True + assert opts["subtitleslangs"] == ["en", "es"] + + +def test_merge_format_omitted_when_container_auto(): + assert "merge_output_format" not in options.build(make()) + assert options.build(make(container="mkv"))["merge_output_format"] == "mkv" + + +@pytest.mark.parametrize( + "text, expected", + [("1024", 1024), ("800K", 819200), ("4M", 4194304), ("1.5M", 1572864), ("2MB", 2097152)], +) +def test_rate_limit_parsing(text, expected): + assert options._parse_rate(text) == expected + + +def test_errors_are_not_swallowed_by_ytdlp(): + # The batch layer owns retry and failure policy; yt-dlp must surface errors. + assert options.build(make())["ignoreerrors"] is False + + +def test_cookie_file_and_browser_map_to_distinct_options(): + assert options.build(make(cookie_file=Path("c.txt")))["cookiefile"] == "c.txt" + assert options.build(make(cookies_from_browser="firefox"))["cookiesfrombrowser"] == ("firefox",) + + +def test_archive_enables_skip_on_rerun(): + opts = options.build(make(archive_file=Path("done.txt"))) + assert opts["download_archive"] == "done.txt" + + +def test_section_produces_keyframe_accurate_range(): + opts = options.build(make(sections="00:01:30-00:02:00")) + assert opts["force_keyframes_at_cuts"] is True + ranges = opts["download_ranges"]({"duration": 600}, None) + assert ranges == [{"start_time": 90, "end_time": 120}] + + +def test_open_ended_section_runs_to_end_of_video(): + opts = options.build(make(sections="30-")) + assert opts["download_ranges"]({"duration": 600}, None) == [ + {"start_time": 30, "end_time": 600} + ] + + +def test_extra_args_override_generated_options(): + opts = options.build(make(extra_ytdlp_args={"format": "worst", "custom": 1})) + assert opts["format"] == "worst" + assert opts["custom"] == 1 diff --git a/tools/video-downloader/vdl/__init__.py b/tools/video-downloader/vdl/__init__.py new file mode 100644 index 0000000..fd4da1d --- /dev/null +++ b/tools/video-downloader/vdl/__init__.py @@ -0,0 +1,12 @@ +"""vdl — a thin, hardened CLI over yt-dlp. + +Public surface is deliberately small: build a DownloadConfig, hand it to +`batch.run`, read the RunMetrics. +""" + +from .config import DownloadConfig +from .errors import VdlError +from .telemetry import ItemResult, RunMetrics + +__version__ = "1.0.0" +__all__ = ["DownloadConfig", "ItemResult", "RunMetrics", "VdlError"] diff --git a/tools/video-downloader/vdl/__main__.py b/tools/video-downloader/vdl/__main__.py new file mode 100644 index 0000000..e83dc81 --- /dev/null +++ b/tools/video-downloader/vdl/__main__.py @@ -0,0 +1,6 @@ +"""Entry point for `python -m vdl`.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/video-downloader/vdl/batch.py b/tools/video-downloader/vdl/batch.py new file mode 100644 index 0000000..53fbd7a --- /dev/null +++ b/tools/video-downloader/vdl/batch.py @@ -0,0 +1,76 @@ +"""Concurrent multi-URL execution. + +Downloads are network-bound, so a thread pool is the right shape: yt-dlp +releases the GIL during socket and disk work. One failed URL never stops the +others — the run reports partial success and exits non-zero. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from . import downloader +from .config import DownloadConfig +from .telemetry import ItemResult, RunMetrics, emit + + +def read_url_file(path: Path) -> list[str]: + """One URL per line; `#` comments and blanks ignored.""" + lines = path.read_text(encoding="utf-8").splitlines() + return [ + stripped + for line in lines + if (stripped := line.strip()) and not stripped.startswith("#") + ] + + +def run(config: DownloadConfig) -> RunMetrics: + """Execute every URL in the config, honouring the concurrency limit.""" + metrics = RunMetrics() + config.output_dir.mkdir(parents=True, exist_ok=True) + + emit( + "run.start", + {"urls": len(config.urls), "concurrency": config.concurrency, + "output_dir": str(config.output_dir)}, + enabled=config.json_logs, + ) + + # A single URL needs no pool — keeps stack traces and Ctrl-C behaviour simple. + if len(config.urls) == 1 or config.concurrency == 1: + for url in config.urls: + _record(metrics, downloader.download(url, config), config) + return _finish(metrics, config) + + workers = min(config.concurrency, len(config.urls)) + with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="vdl") as pool: + futures = {pool.submit(downloader.download, url, config): url for url in config.urls} + for future in as_completed(futures): + url = futures[future] + try: + _record(metrics, future.result(), config) + except BaseException as exc: # worker crashed outside the error contract + _record( + metrics, + ItemResult(url=url, status="failed", error=str(exc), + error_class=type(exc).__name__), + config, + ) + return _finish(metrics, config) + + +def _record(metrics: RunMetrics, result: ItemResult, config: DownloadConfig) -> None: + metrics.record(result) + emit( + f"item.{result.status}", + {"url": result.url, "title": result.title, "path": result.filepath, + "bytes": result.bytes_downloaded, "duration_s": result.duration_s, + "attempts": result.attempts, "error": result.error}, + enabled=config.json_logs, + ) + + +def _finish(metrics: RunMetrics, config: DownloadConfig) -> RunMetrics: + emit("run.finish", metrics.snapshot(), enabled=config.json_logs) + return metrics diff --git a/tools/video-downloader/vdl/cli.py b/tools/video-downloader/vdl/cli.py new file mode 100644 index 0000000..11a2f9b --- /dev/null +++ b/tools/video-downloader/vdl/cli.py @@ -0,0 +1,215 @@ +"""Command-line surface. Parses arguments into a DownloadConfig and runs it.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from . import batch, downloader, ffmpeg +from .config import AUDIO_CODECS, QUALITIES, VIDEO_CONTAINERS, DownloadConfig +from .errors import ConfigError, VdlError +from .progress import human_bytes, human_seconds + +_EPILOG = """\ +examples: + vdl URL download at best quality + vdl URL -q 1080 -c mp4 cap at 1080p, mp4 container + vdl URL --audio-only extract audio as mp3 + vdl URL --subs en --embed-subs fetch and burn in English subtitles + vdl -f urls.txt -j 5 batch a file of URLs, 5 at a time + vdl URL --playlist --items 1-10 first ten entries of a playlist + vdl URL --section 00:01:30-00:04:00 clip a time range + vdl URL --info print available formats, download nothing + +Downloads what the site serves to a normal client. It does not break DRM or +paywalls. Only download material you own or are licensed to keep. +""" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="vdl", + description="Download video or audio from any site yt-dlp supports.", + epilog=_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("urls", nargs="*", help="one or more video URLs") + parser.add_argument("-f", "--url-file", type=Path, help="file with one URL per line") + parser.add_argument("-o", "--output", type=Path, default=Path("downloads"), + help="output directory (default: ./downloads)") + + quality = parser.add_argument_group("quality") + quality.add_argument("-q", "--quality", choices=QUALITIES, default="best", + help="max vertical resolution (default: best)") + quality.add_argument("-c", "--container", choices=VIDEO_CONTAINERS, default="auto", + help="output container; auto avoids re-encoding (default: auto)") + quality.add_argument("-a", "--audio-only", action="store_true", + help="extract audio only") + quality.add_argument("--audio-codec", choices=AUDIO_CODECS, default="mp3", + help="codec for --audio-only (default: mp3)") + + extras = parser.add_argument_group("extras") + extras.add_argument("--subs", nargs="+", metavar="LANG", default=[], + help="subtitle languages, e.g. en es") + extras.add_argument("--embed-subs", action="store_true", help="mux subtitles into the file") + extras.add_argument("--thumbnail", action="store_true", help="embed cover art") + extras.add_argument("--no-metadata", action="store_true", help="skip metadata tagging") + extras.add_argument("--section", metavar="START-END", + help="clip a time range, e.g. 00:01:30-00:04:00") + + playlist = parser.add_argument_group("playlists") + playlist.add_argument("-p", "--playlist", action="store_true", + help="download the whole playlist, not just the linked video") + playlist.add_argument("--items", metavar="SPEC", + help="playlist entries to take, e.g. 1-10 or 2,4,7") + playlist.add_argument("--archive", type=Path, metavar="FILE", + help="record downloaded IDs here and skip them next run") + + network = parser.add_argument_group("network") + network.add_argument("-j", "--concurrency", type=int, default=3, + help="parallel downloads (default: 3)") + network.add_argument("--fragments", type=int, default=4, + help="parallel fragments per download (default: 4)") + network.add_argument("-r", "--rate-limit", metavar="RATE", + help="throttle, e.g. 4M or 800K") + network.add_argument("--retries", type=int, default=3, + help="retries per URL on network failure (default: 3)") + network.add_argument("--cookies", type=Path, metavar="FILE", + help="Netscape cookie file, for sites you are logged into") + network.add_argument("--cookies-from-browser", metavar="BROWSER", + help="pull cookies from an installed browser, e.g. firefox") + + behaviour = parser.add_argument_group("behaviour") + behaviour.add_argument("--name", dest="template", + default="%(title).200B [%(id)s].%(ext)s", + help="output filename template") + behaviour.add_argument("--overwrite", action="store_true", help="re-download existing files") + behaviour.add_argument("--no-resume", action="store_true", help="do not resume partial files") + behaviour.add_argument("--info", action="store_true", + help="list available formats and exit") + behaviour.add_argument("--simulate", action="store_true", + help="run every step except writing the file") + behaviour.add_argument("--json-logs", action="store_true", + help="emit structured telemetry to stderr") + behaviour.add_argument("--quiet", action="store_true", help="suppress progress output") + behaviour.add_argument("--ffmpeg", metavar="PATH", help="explicit ffmpeg location") + return parser + + +def to_config(args: argparse.Namespace) -> DownloadConfig: + urls = list(args.urls) + if args.url_file: + if not args.url_file.exists(): + raise ConfigError(f"URL file not found: {args.url_file}") + urls.extend(batch.read_url_file(args.url_file)) + + # Deduplicate while preserving the order the user gave. + seen: set[str] = set() + urls = [u for u in urls if not (u in seen or seen.add(u))] + + return DownloadConfig( + urls=tuple(urls), + output_dir=args.output, + quality=args.quality, + container=args.container, + audio_only=args.audio_only, + audio_codec=args.audio_codec, + subtitles=tuple(args.subs), + embed_subtitles=args.embed_subs, + embed_thumbnail=args.thumbnail, + embed_metadata=not args.no_metadata, + playlist=args.playlist or bool(args.items), + playlist_items=args.items, + concurrency=args.concurrency, + fragment_concurrency=args.fragments, + rate_limit=args.rate_limit, + retries=args.retries, + cookies_from_browser=args.cookies_from_browser, + cookie_file=args.cookies, + archive_file=args.archive, + filename_template=args.template, + sections=args.section, + resume=not args.no_resume, + overwrite=args.overwrite, + simulate=args.simulate, + quiet=args.quiet, + json_logs=args.json_logs, + ffmpeg_location=args.ffmpeg, + ) + + +def print_info(config: DownloadConfig) -> int: + """Show what the site offers for the first URL, then stop.""" + info = downloader.probe(config.urls[0], config) + print(f"\n{info.get('title', 'untitled')}") + print(f" uploader : {info.get('uploader', '-')}") + print(f" duration : {human_seconds(info.get('duration'))}") + print(f" site : {info.get('extractor_key', '-')}\n") + print(f" {'ID':<16}{'EXT':<6}{'RESOLUTION':<13}{'CODEC':<18}{'SIZE':>10}") + for fmt in info.get("formats", []): + size = fmt.get("filesize") or fmt.get("filesize_approx") + codec = f"{fmt.get('vcodec', '-')}/{fmt.get('acodec', '-')}" + print( + f" {str(fmt.get('format_id')):<16}{str(fmt.get('ext')):<6}" + f"{str(fmt.get('resolution', '-')):<13}{codec[:17]:<18}{human_bytes(size):>10}" + ) + return 0 + + +def summarize(metrics, config: DownloadConfig) -> None: + snap = metrics.snapshot() + health, pressure = snap["health"], snap["pressure"] + print( + f"\n{health['items_ok']} ok · {health['items_failed']} failed · " + f"{human_bytes(snap['efficiency']['bytes_downloaded'])} in " + f"{human_seconds(pressure['elapsed_s'])} " + f"({human_bytes(pressure['throughput_bps'])}/s)" + ) + for item in metrics.items: + if item.status == "ok": + print(f" → {item.filepath or item.title}") + else: + print(f" ✗ {item.url}\n {item.error_class}: {item.error}", file=sys.stderr) + if health["items_ok"]: + print(f"\nSaved to {config.output_dir.resolve()}") + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + + try: + config = to_config(args) + except ConfigError as exc: + print(f"error: {exc}", file=sys.stderr) + return exc.exit_code + + if not config.quiet and ffmpeg.requires_ffmpeg( + audio_only=config.audio_only, + container=config.container, + embedding=config.embed_subtitles or config.embed_thumbnail or config.embed_metadata, + ) and not (config.ffmpeg_location or ffmpeg.locate()): + print( + "warning: ffmpeg not found — merging, conversion and tagging will be skipped.\n" + " install it, or run: pip install imageio-ffmpeg", + file=sys.stderr, + ) + + try: + if args.info: + return print_info(config) + metrics = batch.run(config) + except VdlError as exc: + print(f"error: {exc}", file=sys.stderr) + return exc.exit_code + except KeyboardInterrupt: + print("\ninterrupted — partial files are kept and will resume next run", file=sys.stderr) + return 130 + + if not config.quiet: + summarize(metrics, config) + return 0 if metrics.failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/video-downloader/vdl/config.py b/tools/video-downloader/vdl/config.py new file mode 100644 index 0000000..3c00927 --- /dev/null +++ b/tools/video-downloader/vdl/config.py @@ -0,0 +1,83 @@ +"""Immutable download configuration — the single contract between CLI and engine. + +Validation happens once, here. Everything downstream may assume a valid config. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from .errors import ConfigError + +#: Container the caller wants on disk. `auto` keeps whatever the site serves, +#: which avoids a re-encode and is therefore the default. +VIDEO_CONTAINERS = ("auto", "mp4", "mkv", "webm") +AUDIO_CODECS = ("mp3", "m4a", "opus", "wav", "flac") + +#: Vertical resolution cap. `best` means no cap. +QUALITIES = ("best", "2160", "1440", "1080", "720", "480", "360", "worst") + + +@dataclass(frozen=True) +class DownloadConfig: + """Everything one download run needs. Frozen so no stage can mutate it.""" + + urls: tuple[str, ...] + output_dir: Path = Path("downloads") + quality: str = "best" + container: str = "auto" + audio_only: bool = False + audio_codec: str = "mp3" + subtitles: tuple[str, ...] = () + embed_subtitles: bool = False + embed_thumbnail: bool = False + embed_metadata: bool = True + playlist: bool = False + playlist_items: str | None = None + concurrency: int = 3 + fragment_concurrency: int = 4 + rate_limit: str | None = None + retries: int = 3 + cookies_from_browser: str | None = None + cookie_file: Path | None = None + archive_file: Path | None = None + filename_template: str = "%(title).200B [%(id)s].%(ext)s" + sections: str | None = None + resume: bool = True + overwrite: bool = False + simulate: bool = False + quiet: bool = False + json_logs: bool = False + ffmpeg_location: str | None = None + extra_ytdlp_args: dict = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.urls: + raise ConfigError("no URLs supplied") + if self.quality not in QUALITIES: + raise ConfigError( + f"quality {self.quality!r} invalid; choose from {', '.join(QUALITIES)}" + ) + if self.container not in VIDEO_CONTAINERS: + raise ConfigError( + f"container {self.container!r} invalid; choose from {', '.join(VIDEO_CONTAINERS)}" + ) + if self.audio_codec not in AUDIO_CODECS: + raise ConfigError( + f"audio codec {self.audio_codec!r} invalid; choose from {', '.join(AUDIO_CODECS)}" + ) + if self.concurrency < 1: + raise ConfigError("concurrency must be >= 1") + if self.fragment_concurrency < 1: + raise ConfigError("fragment concurrency must be >= 1") + if self.retries < 0: + raise ConfigError("retries must be >= 0") + if self.audio_only and self.container != "auto": + raise ConfigError("--container conflicts with --audio-only; use --audio-codec") + if self.embed_subtitles and not self.subtitles: + raise ConfigError("--embed-subs requires --subs LANG") + if self.overwrite and self.resume is False: + return # explicit clean re-download; nothing to reconcile + if self.cookie_file and self.cookies_from_browser: + raise ConfigError("use either --cookies FILE or --cookies-from-browser, not both") diff --git a/tools/video-downloader/vdl/downloader.py b/tools/video-downloader/vdl/downloader.py new file mode 100644 index 0000000..6de0c31 --- /dev/null +++ b/tools/video-downloader/vdl/downloader.py @@ -0,0 +1,95 @@ +"""Single-URL download with classified errors and bounded retries.""" + +from __future__ import annotations + +import time +from typing import Any + +from . import errors, options +from .config import DownloadConfig +from .progress import ProgressReporter, SilentLogger +from .telemetry import ItemResult + +#: Exponential backoff base, in seconds. Only transport errors reach it. +_BACKOFF_BASE = 2.0 + + +def probe(url: str, config: DownloadConfig) -> dict[str, Any]: + """Fetch metadata without downloading. Raises a classified VdlError.""" + import yt_dlp + + opts = options.build(config, logger=SilentLogger()) + opts["simulate"] = True + try: + with yt_dlp.YoutubeDL(opts) as ydl: + return ydl.extract_info(url, download=False) or {} + except Exception as exc: # yt-dlp raises many shapes; classify uniformly + raise errors.classify(str(exc))(str(exc)) from exc + + +def download(url: str, config: DownloadConfig) -> ItemResult: + """Download one URL, retrying only transport failures. + + Returns an ItemResult in every case — the batch layer decides whether a + failure should stop the run, so this function does not raise on a failed + download, only records it. + """ + started = time.monotonic() + attempts = 0 + last_error: BaseException | None = None + + while attempts <= config.retries: + attempts += 1 + reporter = ProgressReporter(quiet=config.quiet) + try: + info = _attempt(url, config, reporter) + return ItemResult( + url=url, + status="ok", + title=info.get("title"), + filepath=_resolved_path(info), + bytes_downloaded=reporter.bytes_downloaded, + duration_s=round(time.monotonic() - started, 2), + attempts=attempts, + ) + except errors.VdlError as exc: + last_error = exc + if not errors.is_retryable(exc) or attempts > config.retries: + break + time.sleep(_BACKOFF_BASE ** (attempts - 1)) + + return ItemResult( + url=url, + status="failed", + duration_s=round(time.monotonic() - started, 2), + error=str(last_error), + error_class=type(last_error).__name__, + attempts=attempts, + ) + + +def _attempt(url: str, config: DownloadConfig, reporter: ProgressReporter) -> dict[str, Any]: + import yt_dlp + + logger = SilentLogger() + opts = options.build(config, progress_hooks=[reporter.hook], logger=logger) + + try: + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=not config.simulate) + except Exception as exc: + message = str(exc) or "; ".join(logger.errors) or "unknown download failure" + raise errors.classify(message)(message) from exc + + if info is None: + message = "; ".join(logger.errors) or "extractor returned no result" + raise errors.classify(message)(message) + return info + + +def _resolved_path(info: dict[str, Any]) -> str | None: + """Final on-disk path, after any postprocessing renamed the file.""" + if requested := info.get("requested_downloads"): + entry = requested[0] + return entry.get("filepath") or entry.get("_filename") + return info.get("filepath") or info.get("_filename") diff --git a/tools/video-downloader/vdl/errors.py b/tools/video-downloader/vdl/errors.py new file mode 100644 index 0000000..b686b99 --- /dev/null +++ b/tools/video-downloader/vdl/errors.py @@ -0,0 +1,117 @@ +"""Error taxonomy. Every failure the CLI can surface maps to exactly one class. + +Exit codes are stable — scripts wrapping `vdl` branch on them. +""" + +from __future__ import annotations + +import re + + +class VdlError(Exception): + """Base for every error this tool raises deliberately.""" + + exit_code = 1 + + +class ConfigError(VdlError): + """Caller supplied contradictory or invalid options.""" + + exit_code = 2 + + +class UnsupportedUrlError(VdlError): + """No extractor claims this URL.""" + + exit_code = 3 + + +class ProtectedContentError(VdlError): + """Content is DRM-encrypted or behind an account the tool cannot enter. + + Not recoverable by retry. yt-dlp does not decrypt DRM and neither do we — + this exists so the CLI reports the real reason instead of a generic failure. + """ + + exit_code = 4 + + +class NetworkError(VdlError): + """Transient transport failure. Retryable.""" + + exit_code = 5 + + +class MergeError(VdlError): + """Streams downloaded but ffmpeg could not combine them.""" + + exit_code = 6 + + +class NotFoundError(VdlError): + """The URL resolved but the content is gone. Retrying cannot help.""" + + exit_code = 7 + + +#: Substrings yt-dlp puts in DownloadError messages for unrecoverable access +#: problems. Matched case-insensitively to classify before retrying. +_PROTECTED_MARKERS = ( + "drm", + "this video is private", + "members-only", + "sign in to confirm", + "requires authentication", + "purchase", + "subscribe to this channel", + "video unavailable", + "geo restricted", + "not available in your country", +) + +_NETWORK_MARKERS = ( + "timed out", + "connection reset", + "temporary failure", + "unable to download webpage", + "read operation timed out", + "connection aborted", + "remote end closed", +) + +#: HTTP statuses that will never succeed on retry. Kept ahead of the generic +#: network markers because yt-dlp reports a 404 as "Unable to download webpage", +#: which would otherwise burn the full retry budget on content that is gone. +_AUTH_HTTP = re.compile(r"http error (401|403)\b", re.IGNORECASE) +_GONE_HTTP = re.compile(r"http error (400|404|405|410|451)\b", re.IGNORECASE) +#: Rate limits and server faults are worth another attempt. +_TRANSIENT_HTTP = re.compile(r"http error (408|429|5\d\d)\b", re.IGNORECASE) + + +def classify(message: str) -> type[VdlError]: + """Map a raw yt-dlp error message to the narrowest error class. + + Order matters — it decides retry behaviour. Unrecoverable cases are matched + first so a DRM wall or a dead URL never consumes the retry budget. + """ + lowered = message.lower() + if any(marker in lowered for marker in _PROTECTED_MARKERS): + return ProtectedContentError + if _AUTH_HTTP.search(lowered): + return ProtectedContentError + if _GONE_HTTP.search(lowered): + return NotFoundError + if "unsupported url" in lowered or "no suitable extractor" in lowered: + return UnsupportedUrlError + if _TRANSIENT_HTTP.search(lowered): + return NetworkError + if any(marker in lowered for marker in _NETWORK_MARKERS): + return NetworkError + if "ffmpeg" in lowered or "postprocessing" in lowered: + return MergeError + return VdlError + + +def is_retryable(error: BaseException) -> bool: + """Only transport failures earn another attempt.""" + return isinstance(error, NetworkError) diff --git a/tools/video-downloader/vdl/ffmpeg.py b/tools/video-downloader/vdl/ffmpeg.py new file mode 100644 index 0000000..6c39b83 --- /dev/null +++ b/tools/video-downloader/vdl/ffmpeg.py @@ -0,0 +1,39 @@ +"""ffmpeg discovery. + +Merging split video+audio streams, remuxing, and audio extraction all require +ffmpeg. Rather than fail late with a postprocessing error, resolve it up front +and degrade with a clear message. +""" + +from __future__ import annotations + +import shutil +from functools import lru_cache + + +@lru_cache(maxsize=1) +def locate() -> str | None: + """Return a usable ffmpeg path, or None. + + Order: PATH first (a system ffmpeg is what the user expects to be used), + then the pip-installed `imageio-ffmpeg` binary as a zero-config fallback so + the tool works on machines where installing ffmpeg system-wide is awkward. + """ + system = shutil.which("ffmpeg") + if system: + return system + + try: + import imageio_ffmpeg + except ImportError: + return None + + try: + return imageio_ffmpeg.get_ffmpeg_exe() + except Exception: # binary missing for this platform + return None + + +def requires_ffmpeg(*, audio_only: bool, container: str, embedding: bool) -> bool: + """Whether the requested operation will invoke a postprocessor.""" + return audio_only or container != "auto" or embedding diff --git a/tools/video-downloader/vdl/formats.py b/tools/video-downloader/vdl/formats.py new file mode 100644 index 0000000..d1781bc --- /dev/null +++ b/tools/video-downloader/vdl/formats.py @@ -0,0 +1,63 @@ +"""Format selector construction. + +yt-dlp format strings are the single highest-leverage knob in the whole tool — +they decide resolution, codec, and whether a merge is needed. They are built +here and nowhere else, so the rules stay auditable in one place. +""" + +from __future__ import annotations + +from .config import DownloadConfig + +#: Containers that cannot hold arbitrary codecs; picking the matching stream up +#: front avoids a full re-encode during merge. +_CONTAINER_PREFERENCE = { + "mp4": "[ext=mp4]", + "webm": "[ext=webm]", +} + + +def video_selector(config: DownloadConfig) -> str: + """Build the `-f` expression for a video download. + + Falls back progressively: capped split streams → capped progressive → + anything. A site that offers only one muxed stream still succeeds. + """ + height = "" if config.quality in ("best", "worst") else f"[height<={config.quality}]" + ext = _CONTAINER_PREFERENCE.get(config.container, "") + + if config.quality == "worst": + return "worstvideo+worstaudio/worst" + + candidates = [ + f"bestvideo{height}{ext}+bestaudio{ext}", + f"bestvideo{height}+bestaudio", + f"best{height}{ext}", + f"best{height}", + "best", + ] + # Dedupe while preserving order: with container=auto the ext-qualified and + # bare candidates collapse into each other. + seen: set[str] = set() + ordered = [c for c in candidates if not (c in seen or seen.add(c))] + return "/".join(ordered) + + +def audio_selector() -> str: + """Audio-only downloads always take the best audio stream available.""" + return "bestaudio/best" + + +def selector(config: DownloadConfig) -> str: + return audio_selector() if config.audio_only else video_selector(config) + + +def merge_container(config: DownloadConfig) -> str | None: + """Container to merge split streams into, or None to let yt-dlp decide. + + `auto` deliberately returns None: forcing a container is what triggers + expensive re-encodes when the source codecs do not fit. + """ + if config.audio_only or config.container == "auto": + return None + return config.container diff --git a/tools/video-downloader/vdl/options.py b/tools/video-downloader/vdl/options.py new file mode 100644 index 0000000..bcc08bb --- /dev/null +++ b/tools/video-downloader/vdl/options.py @@ -0,0 +1,132 @@ +"""Translate a DownloadConfig into a yt-dlp options dict. + +This is the only place that knows yt-dlp's option vocabulary. Keeping the +mapping in one pure function makes it testable without touching the network. +""" + +from __future__ import annotations + +from typing import Any, Callable + +from . import ffmpeg, formats +from .config import DownloadConfig + + +def _postprocessors(config: DownloadConfig) -> list[dict[str, Any]]: + chain: list[dict[str, Any]] = [] + + if config.audio_only: + chain.append( + { + "key": "FFmpegExtractAudio", + "preferredcodec": config.audio_codec, + "preferredquality": "0", # 0 = best VBR for lossy codecs + } + ) + + if config.embed_subtitles: + chain.append({"key": "FFmpegEmbedSubtitle", "already_have_subtitle": False}) + + if config.embed_metadata: + chain.append({"key": "FFmpegMetadata", "add_metadata": True}) + + if config.embed_thumbnail: + chain.append({"key": "EmbedThumbnail", "already_have_thumbnail": False}) + + return chain + + +def build( + config: DownloadConfig, + *, + progress_hooks: list[Callable[[dict], None]] | None = None, + logger: Any | None = None, +) -> dict[str, Any]: + """Produce the options dict handed to `yt_dlp.YoutubeDL`.""" + opts: dict[str, Any] = { + "format": formats.selector(config), + "outtmpl": {"default": str(config.output_dir / config.filename_template)}, + "paths": {"home": str(config.output_dir)}, + "noplaylist": not config.playlist, + "continuedl": config.resume, + "overwrites": config.overwrite, + "retries": config.retries, + "fragment_retries": config.retries, + "concurrent_fragment_downloads": config.fragment_concurrency, + "ignoreerrors": False, # batch layer owns error policy, not yt-dlp + "quiet": True, # our own reporter renders progress + "no_warnings": config.quiet, + "noprogress": True, + "simulate": config.simulate, + "writethumbnail": config.embed_thumbnail, + "addmetadata": config.embed_metadata, + "postprocessors": _postprocessors(config), + "progress_hooks": progress_hooks or [], + "trim_file_name": 200, # keep paths under filesystem limits + } + + merge_to = formats.merge_container(config) + if merge_to: + opts["merge_output_format"] = merge_to + + if config.subtitles: + opts["writesubtitles"] = True + opts["writeautomaticsub"] = True + opts["subtitleslangs"] = list(config.subtitles) + + if config.playlist_items: + opts["playlist_items"] = config.playlist_items + + if config.rate_limit: + opts["ratelimit"] = _parse_rate(config.rate_limit) + + if config.sections: + # download_ranges wants a callable; yt-dlp ships the parser for it. + from yt_dlp.utils import parse_duration + + opts["download_ranges"] = _section_ranges(config.sections, parse_duration) + opts["force_keyframes_at_cuts"] = True + + if config.cookie_file: + opts["cookiefile"] = str(config.cookie_file) + if config.cookies_from_browser: + opts["cookiesfrombrowser"] = (config.cookies_from_browser,) + + if config.archive_file: + opts["download_archive"] = str(config.archive_file) + + location = config.ffmpeg_location or ffmpeg.locate() + if location: + opts["ffmpeg_location"] = location + + if logger is not None: + opts["logger"] = logger + + opts.update(config.extra_ytdlp_args) + return opts + + +def _parse_rate(value: str) -> int: + """Convert `4.5M` / `800K` / `1200` into bytes per second.""" + units = {"k": 1024, "m": 1024**2, "g": 1024**3} + text = value.strip().lower().rstrip("b") + if text and text[-1] in units: + return int(float(text[:-1]) * units[text[-1]]) + return int(float(text)) + + +def _section_ranges(spec: str, parse_duration) -> Callable[[Any, Any], list[dict]]: + """Build a download_ranges callable from `START-END` timestamps. + + Accepts `00:01:30-00:02:45` or bare seconds `90-165`. An open end (`90-`) + runs to the end of the video. + """ + start_text, _, end_text = spec.partition("-") + start = parse_duration(start_text) or 0 + end = parse_duration(end_text) if end_text else None + + def ranges(info_dict, ydl): # noqa: ARG001 - yt-dlp calls with both + stop = end if end is not None else info_dict.get("duration") or 0 + return [{"start_time": start, "end_time": stop}] + + return ranges diff --git a/tools/video-downloader/vdl/progress.py b/tools/video-downloader/vdl/progress.py new file mode 100644 index 0000000..c40be80 --- /dev/null +++ b/tools/video-downloader/vdl/progress.py @@ -0,0 +1,107 @@ +"""Terminal progress reporting. + +yt-dlp calls the hook from the downloader thread, so all state here is +per-hook-instance and writes are serialised through one shared lock. +""" + +from __future__ import annotations + +import sys +import threading +import time + +_WRITE_LOCK = threading.Lock() + + +def human_bytes(value: float | None) -> str: + if not value: + return "?" + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024: + return f"{value:.1f}{unit}" + value /= 1024 + return f"{value:.1f}PB" + + +def human_seconds(value: float | None) -> str: + if value is None: + return "--:--" + minutes, seconds = divmod(int(value), 60) + hours, minutes = divmod(minutes, 60) + return f"{hours}:{minutes:02d}:{seconds:02d}" if hours else f"{minutes:02d}:{seconds:02d}" + + +class ProgressReporter: + """Renders one line per active download, throttled to 10 updates/second.""" + + def __init__(self, *, quiet: bool = False, label: str = "") -> None: + self.quiet = quiet + self.label = label + self.bytes_downloaded = 0 + self._last_render = 0.0 + + def hook(self, status: dict) -> None: + state = status.get("status") + if state == "downloading": + self._on_progress(status) + elif state == "finished": + self._on_finished(status) + + def _on_progress(self, status: dict) -> None: + self.bytes_downloaded = status.get("downloaded_bytes") or self.bytes_downloaded + if self.quiet: + return + now = time.monotonic() + if now - self._last_render < 0.1: + return + self._last_render = now + + total = status.get("total_bytes") or status.get("total_bytes_estimate") + done = status.get("downloaded_bytes") or 0 + pct = f"{done / total * 100:5.1f}%" if total else " ??.?%" + speed = status.get("speed") + line = ( + f" {self._name(status):<44.44} {pct} " + f"{human_bytes(done):>9}/{human_bytes(total):<9} " + f"{human_bytes(speed):>9}/s ETA {human_seconds(status.get('eta'))}" + ) + self._write(f"\r{line}", newline=False) + + def _on_finished(self, status: dict) -> None: + self.bytes_downloaded = status.get("downloaded_bytes") or self.bytes_downloaded + if self.quiet: + return + self._write(f"\r {self._name(status):<44.44} {'done':>7} {' ' * 42}", newline=True) + + def _name(self, status: dict) -> str: + info = status.get("info_dict") or {} + return self.label or info.get("title") or status.get("filename", "stream") + + @staticmethod + def _write(text: str, *, newline: bool) -> None: + with _WRITE_LOCK: + sys.stdout.write(text + ("\n" if newline else "")) + sys.stdout.flush() + + +class SilentLogger: + """Absorbs yt-dlp's own logging so it never collides with our progress line. + + Errors are captured rather than printed — the downloader re-raises them + with classification, which is the message the user actually needs. + """ + + def __init__(self) -> None: + self.errors: list[str] = [] + + def debug(self, msg: str) -> None: # noqa: D102 + pass + + def info(self, msg: str) -> None: # noqa: D102 + pass + + def warning(self, msg: str) -> None: # noqa: D102 + pass + + def error(self, msg: str) -> None: # noqa: D102 + self.errors.append(msg) diff --git a/tools/video-downloader/vdl/telemetry.py b/tools/video-downloader/vdl/telemetry.py new file mode 100644 index 0000000..b09555c --- /dev/null +++ b/tools/video-downloader/vdl/telemetry.py @@ -0,0 +1,108 @@ +"""Structured run telemetry: HEALTH, PRESSURE, EFFICIENCY. + +A download run that cannot explain what it did is not finished. Every stage +emits one structured record; `--json-logs` streams them to stderr for machine +consumption while the human reporter writes to stdout. +""" + +from __future__ import annotations + +import json +import sys +import threading +import time +from dataclasses import asdict, dataclass, field + + +@dataclass +class ItemResult: + """Outcome of one URL.""" + + url: str + status: str # ok | failed | skipped + title: str | None = None + filepath: str | None = None + bytes_downloaded: int = 0 + duration_s: float = 0.0 + error: str | None = None + error_class: str | None = None + attempts: int = 1 + + +@dataclass +class RunMetrics: + """Aggregate view of a whole run. Safe to mutate from worker threads.""" + + started_at: float = field(default_factory=time.time) + items: list[ItemResult] = field(default_factory=list) + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + def record(self, result: ItemResult) -> None: + with self._lock: + self.items.append(result) + + @property + def elapsed_s(self) -> float: + return time.time() - self.started_at + + @property + def succeeded(self) -> int: + return sum(1 for i in self.items if i.status == "ok") + + @property + def failed(self) -> int: + return sum(1 for i in self.items if i.status == "failed") + + @property + def skipped(self) -> int: + return sum(1 for i in self.items if i.status == "skipped") + + @property + def total_bytes(self) -> int: + return sum(i.bytes_downloaded for i in self.items) + + def snapshot(self) -> dict: + """The three required signals, plus the per-item audit trail.""" + elapsed = max(self.elapsed_s, 1e-6) + return { + "health": { + "alive": True, + "items_total": len(self.items), + "items_ok": self.succeeded, + "items_failed": self.failed, + "items_skipped": self.skipped, + }, + "pressure": { + "elapsed_s": round(elapsed, 2), + "throughput_bps": int(self.total_bytes / elapsed), + "items_per_min": round(len(self.items) / elapsed * 60, 2), + }, + "efficiency": { + "bytes_downloaded": self.total_bytes, + "retry_overhead": sum(max(i.attempts - 1, 0) for i in self.items), + "avg_item_s": round( + sum(i.duration_s for i in self.items) / max(len(self.items), 1), 2 + ), + }, + "items": [ + {k: v for k, v in asdict(i).items() if not k.startswith("_")} + for i in self.items + ], + } + + +def emit(event: str, payload: dict, *, enabled: bool) -> None: + """Write one structured JSON line to stderr. + + stderr keeps machine output separate from the progress UI on stdout, so + `vdl ... 2> run.jsonl` yields a clean, parseable log. + """ + if not enabled: + return + record = { + "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "service": "vdl", + "event": event, + "data": payload, + } + print(json.dumps(record, default=str), file=sys.stderr, flush=True) From d77ca5974e32e8201f13fca4e935e4e469c70e4f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 20:09:29 +0000 Subject: [PATCH 2/2] feat(siphon): add Vercel web front end for the downloader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Siphon — the vdl engine behind a static UI plus one serverless function. Resolution, not proxying: Vercel functions cap at 60s with no persistent disk, so streaming full videos through one fails on anything sizable. The function probes the URL and returns direct stream URLs; the browser fetches from the origin CDN. The server never touches video bytes. - api/resolve.py URL validation + stream resolution, shared error taxonomy - public/index.html obsidian/gold UI, Cinzel + DM Mono per house style - vercel.json 60s/1024MB function budget, nosniff/DENY/no-referrer validate() is the security boundary. The function feeds caller-supplied URLs to yt-dlp, which fetches whatever it is handed — an SSRF gadget pointed at Vercel's internal network unless fenced. Scheme, length, and every resolved address are checked before extraction; private, loopback, link-local, reserved and multicast space are refused, including 169.254.169.254. If DNS returns several addresses and any one is internal, the request is rejected. Two bugs found while verifying against real media: - _kind() read vcodec=None as "no video track", but yt-dlp uses the literal string "none" for absent and None for unprobed. A plain .avi came back labelled audio-only. Unknown now reads as present. - _label() hardcoded "audio only" whenever height was missing, mislabelling the same generic-extractor case. It now derives from the stream kind. requirements.txt narrowed to yt-dlp so the lambda does not ship a 25MB ffmpeg binary it never invokes; ffmpeg stays an optional CLI extra. 114 tests (35 new), all offline. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01US2uXf9esJvHouvwvvUxCB --- tools/video-downloader/README.md | 45 ++++- tools/video-downloader/api/resolve.py | 191 +++++++++++++++++++ tools/video-downloader/public/index.html | 228 +++++++++++++++++++++++ tools/video-downloader/requirements.txt | 1 - tools/video-downloader/tests/test_api.py | 160 ++++++++++++++++ tools/video-downloader/vercel.json | 16 ++ 6 files changed, 636 insertions(+), 5 deletions(-) create mode 100644 tools/video-downloader/api/resolve.py create mode 100644 tools/video-downloader/public/index.html create mode 100644 tools/video-downloader/tests/test_api.py create mode 100644 tools/video-downloader/vercel.json diff --git a/tools/video-downloader/README.md b/tools/video-downloader/README.md index 2c7ca33..6109e1b 100644 --- a/tools/video-downloader/README.md +++ b/tools/video-downloader/README.md @@ -13,14 +13,15 @@ partial-failure isolation, structured telemetry, and a CLI you can remember. ```bash cd tools/video-downloader -pip install -r requirements.txt # yt-dlp + a bundled ffmpeg binary +pip install -r requirements.txt # core: yt-dlp +pip install imageio-ffmpeg # optional: bundled ffmpeg binary python -m vdl --help ``` -Optionally install it as a real command: +Optionally install it as a real command, with ffmpeg included: ```bash -pip install -e . +pip install -e ".[ffmpeg]" vdl --help ``` @@ -98,6 +99,40 @@ vdl URL --cookies cookies.txt # or a Netscape cookie file --- +## Siphon — the web front end + +The same engine, deployed to Vercel as **Siphon**. Paste a URL, get every +downloadable stream as a direct link. + +``` +public/index.html static UI (obsidian/gold, Cinzel + DM Mono) +api/resolve.py serverless function — metadata resolution only +vercel.json function limits + security headers +``` + +**Why it resolves instead of proxying.** Vercel functions cap at 60s and have +no persistent disk, so streaming a full video through one fails on anything +sizable. Siphon's function probes the URL and returns the direct stream URLs; +the browser then fetches from the origin CDN. No middleman, no egress bill, no +timeout, and the server never touches video bytes. + +**The security boundary is `validate()`.** The function hands +attacker-controlled URLs to yt-dlp, which will fetch whatever it is given — +that is an SSRF gadget aimed at Vercel's internal network unless it is fenced. +Every request is checked before extraction: `http`/`https` only, length capped, +and every resolved address rejected if it lands in private, loopback, +link-local, reserved, or multicast space (including the `169.254.169.254` +metadata endpoint). If DNS returns several addresses and *any* is internal, the +request is refused. + +Deploy it yourself with the project root set to `tools/video-downloader`: + +```bash +vercel --cwd tools/video-downloader +``` + +--- + ## Design notes **Errors are classified, and classification drives retries.** yt-dlp reports a @@ -139,6 +174,8 @@ unsatisfiable constraint. ``` tools/video-downloader/ +├── api/resolve.py Vercel function — URL validation + stream resolution +├── public/index.html Siphon web UI ├── vdl/ │ ├── cli.py argument parsing, output formatting, exit codes │ ├── config.py DownloadConfig — validated once, frozen @@ -150,7 +187,7 @@ tools/video-downloader/ │ ├── telemetry.py HEALTH / PRESSURE / EFFICIENCY signals │ ├── ffmpeg.py ffmpeg discovery with bundled fallback │ └── errors.py error taxonomy, retry classification, exit codes -└── tests/ 79 tests, no network access required +└── tests/ 114 tests, no network access required ``` Run the suite with `python -m pytest` from `tools/video-downloader`. The tests diff --git a/tools/video-downloader/api/resolve.py b/tools/video-downloader/api/resolve.py new file mode 100644 index 0000000..d0d58ca --- /dev/null +++ b/tools/video-downloader/api/resolve.py @@ -0,0 +1,191 @@ +"""Vercel serverless function: resolve a URL to its downloadable streams. + +Deliberately metadata-only. Vercel functions have a 60s ceiling and no +persistent disk, so proxying video bytes through here would fail on anything +sizable. Instead we return the direct stream URLs and let the browser fetch +from the origin CDN — no middleman, no egress bill, no timeout. +""" + +from __future__ import annotations + +import ipaddress +import json +import os +import socket +import sys +from http.server import BaseHTTPRequestHandler +from urllib.parse import urlparse + +# The shared error taxonomy lives in the CLI package one level up. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from vdl.errors import classify # noqa: E402 + +MAX_BODY_BYTES = 8 * 1024 +MAX_FORMATS = 40 + + +class UrlRejected(ValueError): + """Input failed validation before any network call was made.""" + + +def validate(raw: str) -> str: + """Reject anything that is not a public http(s) URL. + + yt-dlp will happily fetch whatever it is handed, so an unvalidated URL + turns this function into an SSRF gadget pointed at Vercel's internal + network. Scheme and resolved-IP checks both happen before extraction. + """ + if not raw or not isinstance(raw, str): + raise UrlRejected("no URL supplied") + raw = raw.strip() + if len(raw) > 2048: + raise UrlRejected("URL too long") + + parsed = urlparse(raw) + if parsed.scheme not in ("http", "https"): + raise UrlRejected("only http and https URLs are accepted") + if not parsed.hostname: + raise UrlRejected("URL has no host") + + for family, _, _, _, address in _resolve(parsed.hostname): + ip = ipaddress.ip_address(address[0]) + if (ip.is_private or ip.is_loopback or ip.is_link_local + or ip.is_reserved or ip.is_multicast): + raise UrlRejected("host resolves to a non-public address") + return raw + + +def _resolve(hostname: str): + try: + return socket.getaddrinfo(hostname, None) + except socket.gaierror as exc: + raise UrlRejected(f"cannot resolve host: {hostname}") from exc + + +def _kind(fmt: dict) -> str: + """Classify a stream as progressive / video-only / audio-only. + + yt-dlp signals an absent track with the literal string "none". A missing + key or None means the extractor never probed it — common for direct file + URLs via the generic extractor — so absence of information is treated as + "present", not "absent". Reading None as absent mislabels a plain .avi + download as audio-only. + """ + has_video = fmt.get("vcodec") != "none" + has_audio = fmt.get("acodec") != "none" + if has_video and has_audio: + return "progressive" # ready to play as-is + return "video" if has_video else "audio" + + +def _label(fmt: dict) -> str: + """Human-facing quality label, derived from whatever the extractor knew.""" + if fmt.get("resolution"): + return fmt["resolution"] + if fmt.get("height"): + return f"{fmt.get('width') or '?'}x{fmt['height']}" + return "audio only" if _kind(fmt) == "audio" else "source file" + + +def _rank(fmt: dict) -> tuple: + """Best first: progressive streams, then by height, then by bitrate.""" + order = {"progressive": 0, "video": 1, "audio": 2} + return ( + order.get(_kind(fmt), 3), + -(fmt.get("height") or 0), + -(fmt.get("tbr") or 0), + ) + + +def extract(url: str) -> dict: + """Probe the URL and shape the response for the client.""" + import yt_dlp + + opts = { + "quiet": True, + "no_warnings": True, + "skip_download": True, + "noplaylist": True, + "socket_timeout": 20, + } + with yt_dlp.YoutubeDL(opts) as ydl: + info = ydl.extract_info(url, download=False) or {} + + # A playlist URL yields entries instead of formats; surface the first item. + if info.get("_type") == "playlist" and info.get("entries"): + info = info["entries"][0] or {} + + usable = [f for f in info.get("formats", []) if f.get("url")] + usable.sort(key=_rank) + + formats = [ + { + "format_id": f.get("format_id"), + "ext": f.get("ext"), + "resolution": _label(f), + "fps": f.get("fps"), + "vcodec": f.get("vcodec"), + "acodec": f.get("acodec"), + "filesize": f.get("filesize") or f.get("filesize_approx"), + "tbr": f.get("tbr"), + "kind": _kind(f), + "note": f.get("format_note"), + "url": f.get("url"), + } + for f in usable[:MAX_FORMATS] + ] + + return { + "title": info.get("title") or "untitled", + "uploader": info.get("uploader") or info.get("channel"), + "duration": info.get("duration"), + "thumbnail": info.get("thumbnail"), + "extractor": info.get("extractor_key"), + "webpage_url": info.get("webpage_url") or url, + "formats": formats, + } + + +class handler(BaseHTTPRequestHandler): + def do_OPTIONS(self) -> None: # noqa: N802 - name fixed by BaseHTTPRequestHandler + self._respond(204, {}) + + def do_POST(self) -> None: # noqa: N802 + try: + length = int(self.headers.get("content-length") or 0) + except ValueError: + return self._respond(400, {"error": "invalid content-length"}) + if length > MAX_BODY_BYTES: + return self._respond(413, {"error": "request body too large"}) + + try: + payload = json.loads(self.rfile.read(length) or b"{}") + url = validate(payload.get("url", "")) + except UrlRejected as exc: + return self._respond(400, {"error": str(exc)}) + except (ValueError, AttributeError): + return self._respond(400, {"error": "malformed JSON body"}) + + try: + return self._respond(200, extract(url)) + except Exception as exc: # classify so the UI can explain the real cause + message = str(exc) + kind = classify(message) + status = 404 if kind.__name__ == "NotFoundError" else 422 + return self._respond(status, {"error": message, "kind": kind.__name__}) + + def _respond(self, status: int, body: dict) -> None: + encoded = json.dumps(body).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.send_header("access-control-allow-origin", "*") + self.send_header("access-control-allow-headers", "content-type") + self.send_header("access-control-allow-methods", "POST, OPTIONS") + self.end_headers() + if status != 204: + self.wfile.write(encoded) + + def log_message(self, *args) -> None: + """Silence per-request stderr noise; Vercel logs requests already.""" diff --git a/tools/video-downloader/public/index.html b/tools/video-downloader/public/index.html new file mode 100644 index 0000000..2c0c162 --- /dev/null +++ b/tools/video-downloader/public/index.html @@ -0,0 +1,228 @@ + + + + + +Siphon — Pull any stream + + + + + + +
+
+
+ +
Pull any stream
+
+ +
+ + +
+

Resolves the URL to its direct streams. Transfer runs browser → source, never through this server.

+ +
+ +
+
+ Siphon reads what a site already serves to a normal browser.
+ It does not break DRM, defeat paywalls, or bypass logins.
+ Download only what you own or are licensed to keep. +
+
+ + + + diff --git a/tools/video-downloader/requirements.txt b/tools/video-downloader/requirements.txt index f9fdf23..6921055 100644 --- a/tools/video-downloader/requirements.txt +++ b/tools/video-downloader/requirements.txt @@ -1,2 +1 @@ yt-dlp>=2024.1.1 -imageio-ffmpeg>=0.4.9 diff --git a/tools/video-downloader/tests/test_api.py b/tools/video-downloader/tests/test_api.py new file mode 100644 index 0000000..20a036e --- /dev/null +++ b/tools/video-downloader/tests/test_api.py @@ -0,0 +1,160 @@ +"""Tests for the Vercel resolve function — input validation above all. + +The handler hands attacker-controlled URLs to yt-dlp, so `validate` is the +security boundary of the whole deployment. It gets the most coverage here. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from api.resolve import UrlRejected, _kind, _label, _rank, validate # noqa: E402 + + +@pytest.fixture +def public_dns(monkeypatch): + """Resolve every hostname to a public address unless a test says otherwise.""" + monkeypatch.setattr( + "api.resolve._resolve", + lambda host: [(2, 1, 6, "", ("93.184.216.34", 0))], + ) + + +def test_accepts_public_https_url(public_dns): + assert validate("https://example.com/watch?v=1") == "https://example.com/watch?v=1" + + +def test_strips_surrounding_whitespace(public_dns): + assert validate(" https://example.com/v ") == "https://example.com/v" + + +@pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + "ftp://example.com/v.mp4", + "gopher://example.com/", + "javascript:alert(1)", + "data:text/html,