From f3639a20dbad866315f87a78efc5d53bda035804 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 08:13:13 -0700 Subject: [PATCH 01/14] docs: design spec for the layout control surface --- ...026-08-29-layout-control-surface-design.md | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-29-layout-control-surface-design.md diff --git a/docs/superpowers/specs/2026-08-29-layout-control-surface-design.md b/docs/superpowers/specs/2026-08-29-layout-control-surface-design.md new file mode 100644 index 0000000..24a8c0f --- /dev/null +++ b/docs/superpowers/specs/2026-08-29-layout-control-surface-design.md @@ -0,0 +1,179 @@ +# Layout control surface — design + +## What this is + +`/layout` is what the NDI broadcaster captures for the physical wall — its +header, cursor and context menu must never appear in that feed. But an +operator still needs every interaction the plain grid page offers: hover to +unmute, scroll to zoom, drag to pan, double-click to lock audio, and the +right-click menu. This adds a second, ordinary browser tab — `/layout-control` +— showing the header controls plus one rectangle per video cell. Interacting +with a rectangle relays the equivalent action to the real cell on `/layout`, +which alone owns the real YouTube players. `/layout`'s own header is hidden +by CSS only; nothing about its behavior changes when no control window is +open. + +## Why + +The broadcaster captures whatever `/layout` renders, full-frame. Any visible +chrome — buttons, a cursor, a context menu — goes out over NDI onto the real +wall. The fix is not to remove the interactions, it is to move where the +*human* interacts from, while the actual player state stays exactly where it +already lives. + +## Non-goals + +- Not built for `/` (the plain grid). `wall-engine.js`'s new mechanism is + generic enough to extend there later, but no second page is built for it + now, and `/`'s behavior must not change at all. +- Not a network protocol. `BroadcastChannel` only connects tabs in the same + browser, same profile, same machine — which is already true of how the NDI + broadcaster works (it captures a local Chrome window). No server involvement, + no new backend route for this feature. +- Not a video preview. `/layout-control`'s rectangles are plain divs with a + title label — no second set of YouTube embeds, no synchronization problem + between two live players for the same video. + +## Where this lives + +Same branch family: `feat/layout-control-surface`, branched from +`feat/cloudflare-deploy` (which now carries the merged `/layout` work). + +New files: + +| File | Responsibility | +|---|---| +| `static/layout-control.html` | The control window's markup: the same header controls as `layout.html`/`player.html`, plus a `#grid` of plain rectangles (no iframes). | +| `static/layout-control.js` | Connects to the `BroadcastChannel`, renders rectangles and header state purely from received snapshots (no independent fetch of config or screen geometry — see "Geometry" below), turns mouse/keyboard input into intents, and executes read-only context-menu actions locally. | +| `tests/test_layout_control_smoke.py` | Browser test opening both `/layout` and `/layout-control` in one Playwright browser context (required for `BroadcastChannel` to connect them) and asserting an interaction on one lands on the other. | + +Modified: + +| File | Change | +|---|---| +| `static/wall-engine.js` | Every local event handler is refactored to build a small "intent" object and hand it to one `applyIntent()` function, instead of mutating state inline. `startWall()` gains an opt-in option that opens a `BroadcastChannel`: incoming messages feed `applyIntent()`, and a state snapshot is published on every meaningful change plus a slow heartbeat. When the option is absent (`/`'s `player.js`), none of this activates. | +| `static/layout-page.js` | Passes the new option to `startWall()`. | +| `static/layout.html` | CSS-only: the `
` is visually hidden (`display: none` or off-screen), not removed from the DOM — `wall-engine.js`'s `getElementById` lookups must keep succeeding. | +| `ytmatrix/main.py` | One more explicit route, `GET /layout-control`, identical in shape to the existing `/layout`/`/config` routes. | +| `scripts/build-dist.sh` | One more line shipping `layout-control.html`; `layout-control.js` is already picked up by the existing `static/*.js` glob. | + +## The intent/snapshot protocol + +Two message types travel over one `BroadcastChannel` (name: +`"yt-matrix-layout-control"`). + +### Intents (control window → broadcast window) + +Every one of today's local DOM-event effects becomes an intent object. +Coordinates are normalized (0..1, relative to the *sending* rectangle's own +box) so the two windows never need matching pixel sizes — the broadcast +window converts to its own real cell's pixel space using its own +`getBoundingClientRect()` before calling the existing `zoomAt`/`panBy` math +unchanged. Wheel `deltaY`'s *sign* is what `nextZoom` actually uses, so it +travels as-is with no scaling. + +``` +{ type: "play" } +{ type: "pause" } +{ type: "muteToggle" } +{ type: "rewind" } +{ type: "shuffle" } +{ type: "resetView" } +{ type: "newQuery", prompt: string | null } +{ type: "hoverUnmuteToggle", checked: boolean } +{ type: "followToggle", checked: boolean } +{ type: "cellHoverEnter", index: number } +{ type: "cellHoverLeave" } +{ type: "cellWheel", index: number, deltaY: number, x: number, y: number } +{ type: "cellDragStart", index: number } +{ type: "cellDragMove", index: number, dx: number, dy: number } // fraction of the sending rect's own width/height +{ type: "cellDragEnd", index: number } +{ type: "cellDblclick", index: number } +{ type: "cellMenuAction", index: number, action: "togglePlay" | "toggleLock" | "restart" | "resetZoom" | "replaceReserve" } +``` + +`applyIntent()` in `wall-engine.js` is the single place that used to be ten +different event-listener bodies; each listener now just builds the matching +intent object and calls it locally, and the exact same function runs when the +same object arrives from the channel. + +### Snapshots (broadcast window → control window) + +Published on every change that would have been visible to a local user +(rebuild, mute/lock change, status change, pre-roll completion, etc.) plus a +1-second heartbeat so a late-joining or reconnecting control window is never +more than a second stale. + +``` +{ + type: "snapshot", + global: { + status: string, + statusState: "" | "busy" | "error", + audioIndicatorText: string, + audioLocked: boolean, + muted: boolean, + prerolled: boolean, + wantPlaying: boolean, + playDisabled: boolean, + hoverUnmuteChecked: boolean, + followChecked: boolean, + newQueryVisible: boolean, + newQueryDisabled: boolean, + reservesLeft: number, + }, + cells: [ + { + index: number, + videoId: string | null, + title: string | null, + empty: boolean, + zoom: number, + locked: boolean, + audible: boolean, + playing: boolean, + currentTime: number, + rect: { left: number, top: number, width: number, height: number }, // percentages, same numbers layout-page.js's own cellRect(index) already computes + }, + ... + ], +} +``` + +`cells[i].rect` is the load-bearing reason `layout-control.js` never fetches +`static/layout/screens.json` or `/api/config` itself: the broadcast window +has already resolved exact geometry for its own DOM via `computeLayout`, and +handing those same percentages over the channel means both windows are +provably showing the same layout, by construction, rather than by two +independent computations agreeing. Before the first snapshot arrives (or if +none has arrived in over 2 heartbeats), `/layout-control` shows a plain +"waiting for /layout to connect…" status instead of any rectangles. + +### Context menu + +Right-clicking a rectangle in `/layout-control` opens the same ten items +`wall-engine.js`'s `menuItems()` already offers, built in +`layout-control.js` from the cached snapshot data for that cell: + +- **Executed locally, no intent sent** (everything the label only needs + `videoId`/`title`/`currentTime` for): Copy video URL at time, Copy video + URL, Copy video ID, Copy title, Open on YouTube at time. +- **Sent as a `cellMenuAction` intent** (everything that mutates real player + state, which only the broadcast window owns): Play/Pause this cell + (`togglePlay`), Lock/unlock audio to this cell (`toggleLock`), Restart this + cell (`restart`), Reset zoom (`resetZoom`), Replace with next reserve + (`replaceReserve`). + +## Testing + +- The full existing browser suites (`tests/test_player_smoke.py` for `/`, + `tests/test_layout_smoke.py` for `/layout` with no control window open) + must keep passing **unmodified** — proof the `applyIntent` refactor changed + nothing observable when no `BroadcastChannel` message ever arrives. +- `tests/test_layout_control_smoke.py` opens both pages via + `context.new_page()` twice in one Playwright `BrowserContext` (same + browser, same profile — required for the two tabs' `BroadcastChannel`s to + be the same channel), drives at least one intent from the control page + (e.g. a wheel event on a rectangle) and asserts the corresponding cell on + `/layout` actually zoomed, plus one menu-relayed action (e.g. toggling + play/pause) and confirms it landed on the real player. From 6ab3f364e248b209b9bf7f1cc87bc7a334ecb55d Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 08:20:05 -0700 Subject: [PATCH 02/14] docs: implementation plan for the layout control surface --- .../2026-08-29-layout-control-surface.md | 1693 +++++++++++++++++ 1 file changed, 1693 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-29-layout-control-surface.md diff --git a/docs/superpowers/plans/2026-08-29-layout-control-surface.md b/docs/superpowers/plans/2026-08-29-layout-control-surface.md new file mode 100644 index 0000000..c1f220d --- /dev/null +++ b/docs/superpowers/plans/2026-08-29-layout-control-surface.md @@ -0,0 +1,1693 @@ +# Layout Control Surface Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a second, ordinary browser tab (`/layout-control`) that mirrors `/layout`'s controls and per-cell interactions over `BroadcastChannel`, so an operator can drive the broadcast-clean `/layout` page (header hidden by CSS) without touching it directly. + +**Architecture:** Every local event handler in `wall-engine.js` is refactored to construct a small "intent" object and hand it to one `applyIntent()` dispatcher, instead of mutating state inline — the same function then runs whether the intent came from a real DOM event or arrived over a `BroadcastChannel`. `wall-engine.js` also gains an opt-in snapshot publisher (per-cell + global state) so a control page can render without any state of its own. `/layout-control` is a pure consumer: no independent fetch of config or geometry, everything comes from snapshots. + +**Tech Stack:** Vanilla JS ES modules (as the rest of `static/`), `BroadcastChannel` (native browser API, no library), Playwright for the two-page browser test. + +**Spec:** `docs/superpowers/specs/2026-08-29-layout-control-surface-design.md` + +## Global Constraints + +- **Zero behavior change on `/` and on `/layout` with no control tab open.** `tests/test_player_smoke.py` and `tests/test_layout_smoke.py` (both marked `browser`) must pass, unmodified, after every task in this plan. +- **`BroadcastChannel` only, no server involvement.** It connects tabs in the same browser/profile/machine, which is already required for the NDI broadcaster to work. No new backend route carries control traffic. +- Coordinates in intents that describe a pointer position are normalized (0..1, relative to the *sending* window's own cell rect) — the receiving side always converts using its own `getBoundingClientRect()`. Wheel `deltaY` travels as-is (only its sign matters to `nextZoom`). +- Every mutating context-menu action becomes a `cellMenuAction` intent; every read-only one (copy/open) executes locally wherever it was clicked, using cached snapshot data. +- `uv run ruff check . && uv run ruff format .` is not applicable to the JS-only tasks in this plan (no Python touched until none) but `node --check ` must be run on every JS file created or modified, and the two existing browser suites plus the new one must all pass before every commit that touches `wall-engine.js`. + +--- + +### Task 1: Extract named global-action functions + +**Files:** +- Modify: `static/wall-engine.js` + +**Interfaces:** +- Consumes: nothing new. +- Produces: `rewindAll()`, `toggleMute()`, `shuffleWall()`, `resetAllViews()` — each a pure extraction of an existing inline listener body, called by that same listener and by nothing else yet (Task 3 wires them into `applyIntent`). + +- [ ] **Step 1: Extract `rewindAll()`** + +Replace: +```js +rewindButton.addEventListener("click", () => { + for (const player of livePlayers()) { + try { + player.seekTo(config.playback.start_offset, true); + // seekTo resumes a player that is not already paused, so a paused wall + // would quietly start playing. Put it back. + if (!wantPlaying) player.pauseVideo(); + } catch { + // A player mid-teardown; skip it. + } + } +}); +``` +with: +```js +function rewindAll() { + for (const player of livePlayers()) { + try { + player.seekTo(config.playback.start_offset, true); + // seekTo resumes a player that is not already paused, so a paused wall + // would quietly start playing. Put it back. + if (!wantPlaying) player.pauseVideo(); + } catch { + // A player mid-teardown; skip it. + } + } +} + +rewindButton.addEventListener("click", rewindAll); +``` + +- [ ] **Step 2: Extract `toggleMute()`** + +Replace: +```js +muteButton.addEventListener("click", () => { + muted = !muted; + refreshMuteButton(); + // Unmuting eight players at once is only permitted off a user gesture -- + // this click is it. Doing it any other way leaves some players silent. + applyMuteStateToAll(); +}); +``` +with: +```js +function toggleMute() { + muted = !muted; + refreshMuteButton(); + // Unmuting eight players at once is only permitted off a user gesture -- + // this click is it. Doing it any other way leaves some players silent. + applyMuteStateToAll(); +} + +muteButton.addEventListener("click", toggleMute); +``` + +- [ ] **Step 3: Extract `shuffleWall()`** + +Replace: +```js +shuffleButton.addEventListener("click", () => { + const wasPlaying = wantPlaying; + slotState = shuffleSlots( + [...slotState.slots, ...slotState.reserves], + computeLayout(config).totalCells, + ); + rebuild(); + // Shuffling is not a new query, so it should not silently stop the wall. + // rebuild() clears wantPlaying via pre-roll; put it back if it was running. + wantPlaying = wasPlaying; +}); +``` +with: +```js +function shuffleWall() { + const wasPlaying = wantPlaying; + slotState = shuffleSlots( + [...slotState.slots, ...slotState.reserves], + computeLayout(config).totalCells, + ); + rebuild(); + // Shuffling is not a new query, so it should not silently stop the wall. + // rebuild() clears wantPlaying via pre-roll; put it back if it was running. + wantPlaying = wasPlaying; +} + +shuffleButton.addEventListener("click", shuffleWall); +``` + +- [ ] **Step 4: Extract `resetAllViews()`** + +Replace: +```js +resetViewButton.addEventListener("click", () => { + views.clear(); + for (const cell of gridEl.children) { + delete cell.dataset.zoomed; + applyCoverFit(cell); + } +}); +``` +with: +```js +function resetAllViews() { + views.clear(); + for (const cell of gridEl.children) { + delete cell.dataset.zoomed; + applyCoverFit(cell); + } +} + +resetViewButton.addEventListener("click", resetAllViews); +``` + +- [ ] **Step 5: Verify no behavior change** + +```bash +node --check static/wall-engine.js +node --test 'static/*.test.mjs' +uv run pytest tests/test_player_smoke.py -m browser -v +uv run pytest tests/test_layout_smoke.py -m browser -v +``` +Expected: identical pass counts to before this task (142 node tests unaffected since none touch `wall-engine.js` directly, 40 browser tests for `/`, 3 for `/layout`). + +- [ ] **Step 6: Commit** + +```bash +git add static/wall-engine.js +git commit -m "refactor: name the global header-control actions" +``` + +--- + +### Task 2: Extract named per-cell action functions + +**Files:** +- Modify: `static/wall-engine.js` + +**Interfaces:** +- Consumes: nothing new. +- Produces: `togglePlayForCell(index)`, `toggleLockedIndex(index)`, `restartCell(index)`, `resetCellZoom(index)` — each an extraction of logic currently duplicated or inlined between the `dblclick` listener and the context menu's `run` callbacks. + +- [ ] **Step 1: Add the four functions** + +Insert immediately before `function playerForCell(cell) {`: + +```js +function togglePlayForCell(index) { + const player = players[index]; + let state = -1; + try { + state = player?.getPlayerState?.() ?? -1; + } catch { + // A player mid-teardown; nothing useful to do. + } + if (state === 1) player?.pauseVideo?.(); + else player?.playVideo?.(); +} + +function toggleLockedIndex(index) { + lockedIndex = lockedIndex === index ? null : index; + for (const other of gridEl.children) delete other.dataset.locked; + const cell = gridEl.children[index]; + if (lockedIndex !== null && cell) cell.dataset.locked = "true"; + applyMuteStateToAll(); +} + +function restartCell(index) { + players[index]?.seekTo?.(config.playback.start_offset, true); +} + +function resetCellZoom(index) { + views.delete(index); + const cell = gridEl.children[index]; + if (!cell) return; + cell.dataset.zoomed = "false"; + applyCoverFit(cell); +} +``` + +- [ ] **Step 2: Use them in the context menu** + +In `menuItems(cell)`, replace the four `run` callbacks: + +```js +// Old: + { + label: playing ? "Pause this cell" : "Play this cell", + run: () => (playing ? player?.pauseVideo?.() : player?.playVideo?.()), + }, +``` +```js +// New: + { + label: playing ? "Pause this cell" : "Play this cell", + run: () => togglePlayForCell(index), + }, +``` + +```js +// Old: + { + label: lockedIndex === index ? "Unlock audio" : "Lock audio to this cell", + hint: "double-click", + run: () => { + lockedIndex = lockedIndex === index ? null : index; + for (const other of gridEl.children) delete other.dataset.locked; + if (lockedIndex !== null) cell.dataset.locked = "true"; + applyMuteStateToAll(); + }, + }, +``` +```js +// New: + { + label: lockedIndex === index ? "Unlock audio" : "Lock audio to this cell", + hint: "double-click", + run: () => toggleLockedIndex(index), + }, +``` + +```js +// Old: + { + label: "Restart this cell", + run: () => player?.seekTo?.(config.playback.start_offset, true), + }, +``` +```js +// New: + { + label: "Restart this cell", + run: () => restartCell(index), + }, +``` + +```js +// Old: + { + label: "Reset zoom", + hint: `${(viewFor(cell).zoom ?? 1).toFixed(2)}×`, + run: () => { + views.delete(index); + cell.dataset.zoomed = "false"; + applyCoverFit(cell); + }, + }, +``` +```js +// New: + { + label: "Reset zoom", + hint: `${(viewFor(cell).zoom ?? 1).toFixed(2)}×`, + run: () => resetCellZoom(index), + }, +``` + +- [ ] **Step 3: Use `toggleLockedIndex` in the dblclick listener** + +Replace: +```js +gridEl.addEventListener("dblclick", (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + const index = [...gridEl.children].indexOf(cell); + lockedIndex = lockedIndex === index ? null : index; + + for (const other of gridEl.children) delete other.dataset.locked; + if (lockedIndex !== null) cell.dataset.locked = "true"; + applyMuteStateToAll(); +}); +``` +with: +```js +gridEl.addEventListener("dblclick", (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + toggleLockedIndex([...gridEl.children].indexOf(cell)); +}); +``` + +- [ ] **Step 4: Verify no behavior change** + +```bash +node --check static/wall-engine.js +node --test 'static/*.test.mjs' +uv run pytest tests/test_player_smoke.py -m browser -v +uv run pytest tests/test_layout_smoke.py -m browser -v +``` +Expected: same pass counts as Task 1's verification — in particular `test_double_click_locks_audio_to_one_cell`, `test_double_clicking_another_cell_moves_the_lock`, `test_double_clicking_the_same_cell_turns_the_lock_off`, `test_reset_zoom_is_offered_and_works` and `test_right_click_offers_copy_url_at_time` in `tests/test_player_smoke.py` all still pass — they exercise exactly the code this task moved. + +- [ ] **Step 5: Commit** + +```bash +git add static/wall-engine.js +git commit -m "refactor: name the per-cell actions shared by dblclick and the context menu" +``` + +--- + +### Task 3: `applyIntent()` — the single dispatch point + +This is the highest-risk task in the plan: every local listener changes shape. The binding requirement is that the *effect* of every listener is unchanged; only how it's invoked changes (directly, or through one more function call). + +**Files:** +- Modify: `static/wall-engine.js` + +**Interfaces:** +- Consumes: `rewindAll`, `toggleMute`, `shuffleWall`, `resetAllViews` (Task 1); `togglePlayForCell`, `toggleLockedIndex`, `restartCell`, `resetCellZoom` (Task 2). +- Produces: `applyIntent(intent)` (async), `queueZoom(cell, index, width, height, deltaY, x, y)`, `applyCellWheel(index, deltaY, xFraction, yFraction)`, `applyCellPan(index, dxFraction, dyFraction)`, `applyCellMenuAction(index, action)`. The full intent vocabulary is exactly the one in the design spec's "Intents" section. + +- [ ] **Step 1: Extract `queueZoom` from the wheel listener, and add the three new dispatch helpers** + +Insert immediately before `gridEl.addEventListener("wheel", ...)` (i.e., right after `let pendingZoom = null;`): + +```js +function queueZoom(cell, index, width, height, deltaY, x, y) { + if (pendingZoom && pendingZoom.index === index) { + // Same cell, same frame: sum the deltas so no scrolling is lost, and + // track the cursor to wherever it ended up. + pendingZoom.deltaY += deltaY; + pendingZoom.x = x; + pendingZoom.y = y; + return; + } + // A different cell mid-frame: land the queued one first rather than + // dropping it. + if (pendingZoom) flushZoom(); + pendingZoom = { cell, index, width, height, deltaY, x, y }; + requestAnimationFrame(flushZoom); +} + +// x, y arrive normalized (0..1) -- from a real local event on THIS page's own +// cell, or relayed from /layout-control's differently-sized rectangle for +// the same cell. Either way this converts to this page's own real pixel +// space before reusing the exact zoom math a local wheel event already used. +function applyCellWheel(index, deltaY, xFraction, yFraction) { + const cell = gridEl.children[index]; + if (!cell || cell.dataset.empty === "true") return; + const bounds = cell.getBoundingClientRect(); + queueZoom(cell, index, bounds.width, bounds.height, deltaY, xFraction * bounds.width, yFraction * bounds.height); +} + +// dx, dy arrive as a fraction of the SENDER's own cell size (already an +// incremental delta, not an absolute position) -- rescaling by this page's +// own bounds is what makes a drag feel proportionally the same regardless of +// how big /layout-control's rectangle happens to be. +function applyCellPan(index, dxFraction, dyFraction) { + const cell = gridEl.children[index]; + if (!cell) return; + const bounds = cell.getBoundingClientRect(); + views.set( + index, + panBy(views.get(index) ?? IDENTITY_VIEW, dxFraction * bounds.width, dyFraction * bounds.height), + ); + applyCoverFit(cell); +} + +function applyCellMenuAction(index, action) { + switch (action) { + case "togglePlay": + togglePlayForCell(index); + break; + case "toggleLock": + toggleLockedIndex(index); + break; + case "restart": + restartCell(index); + break; + case "resetZoom": + resetCellZoom(index); + break; + case "replaceReserve": + handlePlayerError(index); + break; + default: + wlog(`applyCellMenuAction: unknown action ${action}`); + } +} + +// Every user action reachable from the header or a cell -- a real DOM event +// on THIS page, or one relayed from /layout-control over BroadcastChannel -- +// funnels through here. That symmetry is the whole point: a control-window +// message and a local click must produce identical effects, so this is the +// only place either kind is handled. publishSnapshot() (added when +// BroadcastChannel wiring lands) is a no-op until then. +async function applyIntent(intent) { + switch (intent.type) { + case "play": + startAll(); + break; + case "pause": + pauseAll(); + break; + case "muteToggle": + toggleMute(); + break; + case "rewind": + rewindAll(); + break; + case "shuffle": + shuffleWall(); + break; + case "resetView": + resetAllViews(); + break; + case "newQuery": + await requestNewQuery(intent.prompt ?? null); + break; + case "hoverUnmuteToggle": + hoverUnmuteCheckbox.checked = intent.checked; + setAudibleCell(null); + applyMuteStateToAll(); + break; + case "followToggle": + followCheckbox.checked = intent.checked; + break; + case "cellHoverEnter": + if (hoverUnmuteCheckbox.checked) setAudibleCell(intent.index); + break; + case "cellHoverLeave": + if (hoverUnmuteCheckbox.checked) setAudibleCell(null); + break; + case "cellWheel": + applyCellWheel(intent.index, intent.deltaY, intent.x, intent.y); + break; + case "cellDragStart": + // Purely a cursor affordance; kept for parity with a locally-driven drag. + gridEl.children[intent.index]?.setAttribute("data-dragging", "true"); + break; + case "cellDragMove": + applyCellPan(intent.index, intent.dx, intent.dy); + break; + case "cellDragEnd": + gridEl.children[intent.index]?.removeAttribute("data-dragging"); + break; + case "cellDblclick": + toggleLockedIndex(intent.index); + break; + case "cellMenuAction": + applyCellMenuAction(intent.index, intent.action); + break; + default: + wlog(`applyIntent: unknown intent type ${intent.type}`); + return; + } +} +``` + +- [ ] **Step 2: Rewire the wheel listener** + +Replace: +```js +gridEl.addEventListener( + "wheel", + (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + event.preventDefault(); + + const index = [...gridEl.children].indexOf(cell); + const bounds = cell.getBoundingClientRect(); + const x = event.clientX - bounds.left; + const y = event.clientY - bounds.top; + + if (pendingZoom && pendingZoom.index === index) { + // Same cell, same frame: sum the deltas so no scrolling is lost, and + // track the cursor to wherever it ended up. + pendingZoom.deltaY += event.deltaY; + pendingZoom.x = x; + pendingZoom.y = y; + return; + } + + // A different cell mid-frame: land the queued one first rather than + // dropping it. + if (pendingZoom) flushZoom(); + + pendingZoom = { + cell, + index, + width: bounds.width, + height: bounds.height, + deltaY: event.deltaY, + x, + y, + }; + requestAnimationFrame(flushZoom); + }, + { passive: false }, +); +``` +with: +```js +gridEl.addEventListener( + "wheel", + (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + event.preventDefault(); + const index = [...gridEl.children].indexOf(cell); + const bounds = cell.getBoundingClientRect(); + applyIntent({ + type: "cellWheel", + index, + deltaY: event.deltaY, + x: bounds.width ? (event.clientX - bounds.left) / bounds.width : 0, + y: bounds.height ? (event.clientY - bounds.top) / bounds.height : 0, + }); + }, + { passive: false }, +); +``` + +- [ ] **Step 3: Rewire drag (pointerdown / pointermove / endDrag)** + +Replace: +```js +gridEl.addEventListener("pointerdown", (event) => { + if (event.button !== 0) return; // left button only; right opens the menu + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + drag = { + cell, + index: [...gridEl.children].indexOf(cell), + x: event.clientX, + y: event.clientY, + }; + cell.dataset.dragging = "true"; + try { + cell.setPointerCapture(event.pointerId); + } catch { + // Capture is a convenience; the document-level pointerup still ends it. + } +}); + +gridEl.addEventListener("pointermove", (event) => { + if (!drag) return; + const dx = event.clientX - drag.x; + const dy = event.clientY - drag.y; + drag.x = event.clientX; + drag.y = event.clientY; + views.set(drag.index, panBy(views.get(drag.index) ?? IDENTITY_VIEW, dx, dy)); + applyCoverFit(drag.cell); +}); + +function endDrag(event) { + if (!drag) return; + try { + drag.cell.releasePointerCapture(event.pointerId); + } catch { + // Already released, or never captured. + } + delete drag.cell.dataset.dragging; + drag = null; +} +``` +with: +```js +gridEl.addEventListener("pointerdown", (event) => { + if (event.button !== 0) return; // left button only; right opens the menu + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + const index = [...gridEl.children].indexOf(cell); + drag = { cell, index, x: event.clientX, y: event.clientY }; + try { + cell.setPointerCapture(event.pointerId); + } catch { + // Capture is a convenience; the document-level pointerup still ends it. + } + applyIntent({ type: "cellDragStart", index }); +}); + +gridEl.addEventListener("pointermove", (event) => { + if (!drag) return; + const dx = event.clientX - drag.x; + const dy = event.clientY - drag.y; + const bounds = drag.cell.getBoundingClientRect(); + drag.x = event.clientX; + drag.y = event.clientY; + applyIntent({ + type: "cellDragMove", + index: drag.index, + dx: bounds.width ? dx / bounds.width : 0, + dy: bounds.height ? dy / bounds.height : 0, + }); +}); + +function endDrag(event) { + if (!drag) return; + try { + drag.cell.releasePointerCapture(event.pointerId); + } catch { + // Already released, or never captured. + } + applyIntent({ type: "cellDragEnd", index: drag.index }); + drag = null; +} +``` + +- [ ] **Step 4: Rewire dblclick and hover** + +Replace: +```js +gridEl.addEventListener("dblclick", (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + toggleLockedIndex([...gridEl.children].indexOf(cell)); +}); + +gridEl.addEventListener("pointerover", (event) => { + if (!hoverUnmuteCheckbox.checked) return; + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + setAudibleCell([...gridEl.children].indexOf(cell)); +}); + +// pointerleave on the grid, not per cell: moving between adjacent cells would +// otherwise blip the audio off and on again between them. +gridEl.addEventListener("pointerleave", () => { + if (!hoverUnmuteCheckbox.checked) return; + setAudibleCell(null); +}); +``` +with: +```js +gridEl.addEventListener("dblclick", (event) => { + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + applyIntent({ type: "cellDblclick", index: [...gridEl.children].indexOf(cell) }); +}); + +gridEl.addEventListener("pointerover", (event) => { + if (!hoverUnmuteCheckbox.checked) return; + const cell = event.target.closest(".cell"); + if (!cell || cell.dataset.empty === "true") return; + applyIntent({ type: "cellHoverEnter", index: [...gridEl.children].indexOf(cell) }); +}); + +// pointerleave on the grid, not per cell: moving between adjacent cells would +// otherwise blip the audio off and on again between them. +gridEl.addEventListener("pointerleave", () => { + if (!hoverUnmuteCheckbox.checked) return; + applyIntent({ type: "cellHoverLeave" }); +}); +``` + +- [ ] **Step 5: Rewire the header buttons and checkboxes** + +Replace: +```js +playButton.addEventListener("click", startAll); +pauseButton.addEventListener("click", pauseAll); + +rewindButton.addEventListener("click", rewindAll); + +muteButton.addEventListener("click", toggleMute); +``` +with: +```js +playButton.addEventListener("click", () => applyIntent({ type: "play" })); +pauseButton.addEventListener("click", () => applyIntent({ type: "pause" })); +rewindButton.addEventListener("click", () => applyIntent({ type: "rewind" })); +muteButton.addEventListener("click", () => applyIntent({ type: "muteToggle" })); +``` + +Replace: +```js +hoverUnmuteCheckbox.addEventListener("change", () => { + // Leaving the mode must not strand a cell audible or the whole wall silent. + setAudibleCell(null); + applyMuteStateToAll(); +}); +``` +with: +```js +hoverUnmuteCheckbox.addEventListener("change", () => + applyIntent({ type: "hoverUnmuteToggle", checked: hoverUnmuteCheckbox.checked }), +); +``` + +Replace: +```js +shuffleButton.addEventListener("click", shuffleWall); +``` +with: +```js +shuffleButton.addEventListener("click", () => applyIntent({ type: "shuffle" })); +``` + +Replace: +```js +resetViewButton.addEventListener("click", resetAllViews); +``` +with: +```js +resetViewButton.addEventListener("click", () => applyIntent({ type: "resetView" })); +``` + +Replace: +```js +newQueryButton.addEventListener("click", async () => { + const prompt = promptInput.value.trim(); + promptInput.disabled = true; + try { + await requestNewQuery(prompt || null); + } finally { + promptInput.disabled = false; + } +}); +``` +with: +```js +newQueryButton.addEventListener("click", async () => { + const prompt = promptInput.value.trim(); + promptInput.disabled = true; + try { + await applyIntent({ type: "newQuery", prompt: prompt || null }); + } finally { + promptInput.disabled = false; + } +}); +``` + +`followCheckbox` has no existing listener to replace — this task does not add one (nothing currently reacts live to it; `/layout-control`'s own checkbox will drive it via a `followToggle` intent once Task 6 exists, and `applyIntent`'s `followToggle` case already handles that from Step 1). + +- [ ] **Step 6: Verify no behavior change** + +```bash +node --check static/wall-engine.js +node --test 'static/*.test.mjs' +uv run pytest tests/ -v +uv run pytest tests/test_player_smoke.py -m browser -v +uv run pytest tests/test_layout_smoke.py -m browser -v +``` +Expected: every suite passes at its pre-existing count — this is the proof that routing every listener through one more function call changed nothing observable. Pay particular attention to `test_scroll_wheel_zooms_toward_the_pointer`, `test_zooming_out_pulls_back_to_the_whole_frame`, `test_drag_pans_a_zoomed_cell`, and `test_dragging_cannot_open_a_gap` — these exercise exactly the normalize-then-rescale round trip Step 2/3 introduced, and on `/` that round trip must be a no-op (dividing and re-multiplying by the same cell's own bounds). + +- [ ] **Step 7: Commit** + +```bash +git add static/wall-engine.js +git commit -m "refactor: route every local interaction through one applyIntent dispatcher" +``` + +--- + +### Task 4: Snapshot publishing + `BroadcastChannel` wiring + +**Files:** +- Modify: `static/wall-engine.js` + +**Interfaces:** +- Consumes: `applyIntent` (Task 3), `computeLayout` (existing). +- Produces: `startWall({ computeLayout, controlChannel })` — `controlChannel` is `null` by default (nothing new activates) or a channel-name string. `buildSnapshot()`, `publishSnapshot()`. + +- [ ] **Step 1: Add the option and the snapshot machinery** + +Change the `startWall` signature: +```js +// Old: +export function startWall({ computeLayout = defaultComputeLayout } = {}) { +``` +```js +// New: +export function startWall({ computeLayout = defaultComputeLayout, controlChannel = null } = {}) { +``` + +Insert, immediately before `function refreshControls() {`: + +```js +// null until startWall({ controlChannel }) opens one -- publishSnapshot() +// below is a no-op until then, so every call site can call it unconditionally +// without checking whether a control page exists. +let broadcastChannel = null; + +function buildSnapshot() { + const layout = computeLayout(config); + const cells = slotState.slots.map((videoId, index) => { + const player = players[index]; + let currentTime = 0; + let playing = false; + try { + currentTime = player?.getCurrentTime?.() ?? 0; + playing = (player?.getPlayerState?.() ?? -1) === 1; + } catch { + // A player mid-teardown; snapshot with the defaults above. + } + return { + index, + videoId: videoId ?? null, + title: videoId ? (titles.get(videoId) ?? videoId) : null, + empty: !videoId, + zoom: views.get(index)?.zoom ?? 1, + locked: lockedIndex === index, + audible: isAudible(index, currentAudioTarget()), + playing, + currentTime, + // Percentages, same object shape buildCells() already applies to this + // page's own cell -- /layout-control positions its rectangle from this + // directly, so the two pages can never disagree about geometry. + rect: layout.cellRect ? layout.cellRect(index) : null, + }; + }); + + return { + type: "snapshot", + global: { + status: statusEl.textContent, + statusState: statusEl.dataset.state ?? "", + audioIndicatorText: audioEl.textContent, + audioLocked: audioEl.dataset.locked === "true", + muted, + prerolled, + wantPlaying, + playDisabled: playButton.disabled, + hoverUnmuteChecked: hoverUnmuteCheckbox.checked, + followChecked: followCheckbox.checked, + newQueryVisible: !newQueryButton.hidden, + newQueryDisabled: newQueryButton.disabled, + reservesLeft: slotState.reserves.length, + }, + cells, + }; +} + +function publishSnapshot() { + if (!broadcastChannel) return; + try { + broadcastChannel.postMessage(buildSnapshot()); + } catch { + // A channel can throw if it has already been closed; nothing useful to do. + } +} +``` + +- [ ] **Step 2: Publish after the moments that matter, plus a heartbeat** + +At the end of `applyIntent`'s switch (immediately before its closing `}`), i.e. change: +```js +// Old: + default: + wlog(`applyIntent: unknown intent type ${intent.type}`); + return; + } +} +``` +```js +// New: + default: + wlog(`applyIntent: unknown intent type ${intent.type}`); + return; + } + publishSnapshot(); +} +``` + +At the end of `finishPreroll`, change: +```js +// Old: + // A new set starts paused unless the user asked it to follow the play state. + if (wantPlaying && followCheckbox.checked) { + startAll(); + } else { + wantPlaying = false; + refreshControls(); + setStatus(`${statusPrefix} · ready — press Play`); + } +} +``` +```js +// New: + // A new set starts paused unless the user asked it to follow the play state. + if (wantPlaying && followCheckbox.checked) { + startAll(); + } else { + wantPlaying = false; + refreshControls(); + setStatus(`${statusPrefix} · ready — press Play`); + } + publishSnapshot(); +} +``` + +- [ ] **Step 3: Open the channel and subscribe, only when asked** + +Immediately before the final `connectSocket({` call at the bottom of `startWall`, insert: + +```js +if (controlChannel) { + broadcastChannel = new BroadcastChannel(controlChannel); + broadcastChannel.addEventListener("message", (event) => { + applyIntent(event.data); + }); + // A heartbeat, not the only source of truth: applyIntent and finishPreroll + // already publish on every discrete change. This just guarantees a + // late-joining or reconnecting control tab is never more than a second + // stale, without instrumenting every low-level mutation site. + setInterval(publishSnapshot, 1000); +} + +``` + +- [ ] **Step 4: Verify no behavior change on `/` or default `/layout`** + +Neither `player.js` nor `layout-page.js` passes `controlChannel` yet (that's Task 5), so `broadcastChannel` stays `null` everywhere in the app today and `publishSnapshot()` is a no-op. + +```bash +node --check static/wall-engine.js +node --test 'static/*.test.mjs' +uv run pytest tests/ -v +uv run pytest tests/test_player_smoke.py -m browser -v +uv run pytest tests/test_layout_smoke.py -m browser -v +``` +Expected: identical pass counts to Task 3's verification. + +- [ ] **Step 5: Commit** + +```bash +git add static/wall-engine.js +git commit -m "feat: opt-in BroadcastChannel snapshot publishing and intent intake" +``` + +--- + +### Task 5: `/layout` hides its chrome and opts in + +**Files:** +- Modify: `static/layout.html` +- Modify: `static/layout-page.js` + +**Interfaces:** +- Consumes: `startWall`'s `controlChannel` option (Task 4). +- Produces: nothing consumed elsewhere in this plan except the literal channel name string, which Task 6 must match exactly: `"yt-matrix-layout-control"`. + +- [ ] **Step 1: Hide the header, keep it in the DOM** + +In `static/layout.html`'s ` + + +
+ + + + + + + + + + + + loading… + ← broadcast +
+
+ + + + diff --git a/static/layout-control.js b/static/layout-control.js new file mode 100644 index 0000000..07d8451 --- /dev/null +++ b/static/layout-control.js @@ -0,0 +1,285 @@ +import { videoUrl, formatTimecode } from "./grid-logic.js"; + +const CHANNEL_NAME = "yt-matrix-layout-control"; +const STALE_AFTER_MS = 2500; + +const channel = new BroadcastChannel(CHANNEL_NAME); + +const gridEl = document.getElementById("grid"); +const statusEl = document.getElementById("status"); +const playButton = document.getElementById("play"); +const pauseButton = document.getElementById("pause"); +const muteButton = document.getElementById("mute"); +const newQueryButton = document.getElementById("new-query"); +const followCheckbox = document.getElementById("follow"); +const promptInput = document.getElementById("prompt"); +const hoverUnmuteCheckbox = document.getElementById("hover-unmute"); +const menuEl = document.getElementById("menu"); +const resetViewButton = document.getElementById("reset-view"); +const shuffleButton = document.getElementById("shuffle"); +const audioEl = document.getElementById("audio"); +const rewindButton = document.getElementById("rewind"); + +function send(intent) { + channel.postMessage(intent); +} + +function setStatus(text, state = "") { + statusEl.textContent = text; + statusEl.dataset.state = state; +} + +setStatus("waiting for /layout to connect…", "busy"); + +// The last snapshot's cells, kept only so the context menu can be built +// without a round trip -- every mutating action still goes back over the +// channel as an intent. +let latestCells = []; +let staleTimer = null; + +function renderFromSnapshot(snapshot) { + latestCells = snapshot.cells; + const g = snapshot.global; + setStatus(g.status, g.statusState); + audioEl.textContent = g.audioIndicatorText; + audioEl.dataset.locked = String(g.audioLocked); + muteButton.textContent = g.muted ? "Unmute" : "Mute"; + muteButton.dataset.muted = String(g.muted); + playButton.disabled = g.playDisabled; + hoverUnmuteCheckbox.checked = g.hoverUnmuteChecked; + followCheckbox.checked = g.followChecked; + newQueryButton.hidden = !g.newQueryVisible; + newQueryButton.disabled = g.newQueryDisabled; + + if (gridEl.children.length !== snapshot.cells.length) { + gridEl.replaceChildren(); + for (let i = 0; i < snapshot.cells.length; i += 1) { + const cell = document.createElement("div"); + cell.className = "cell"; + const label = document.createElement("span"); + label.className = "label"; + cell.appendChild(label); + gridEl.appendChild(cell); + } + } + snapshot.cells.forEach((cellData, index) => { + const cell = gridEl.children[index]; + if (!cell) return; + if (cellData.rect) Object.assign(cell.style, cellData.rect); + cell.dataset.empty = cellData.empty ? "true" : "false"; + cell.dataset.audible = String(cellData.audible); + cell.dataset.locked = String(cellData.locked); + if (cellData.videoId) cell.dataset.videoId = cellData.videoId; + else delete cell.dataset.videoId; + cell.querySelector(".label").textContent = cellData.title ?? ""; + }); + + clearTimeout(staleTimer); + staleTimer = setTimeout(() => { + setStatus("no update from /layout in a while — is it still open?", "error"); + }, STALE_AFTER_MS); +} + +channel.addEventListener("message", (event) => { + if (event.data?.type === "snapshot") renderFromSnapshot(event.data); +}); + +playButton.addEventListener("click", () => send({ type: "play" })); +pauseButton.addEventListener("click", () => send({ type: "pause" })); +muteButton.addEventListener("click", () => send({ type: "muteToggle" })); +rewindButton.addEventListener("click", () => send({ type: "rewind" })); +shuffleButton.addEventListener("click", () => send({ type: "shuffle" })); +resetViewButton.addEventListener("click", () => send({ type: "resetView" })); +hoverUnmuteCheckbox.addEventListener("change", () => + send({ type: "hoverUnmuteToggle", checked: hoverUnmuteCheckbox.checked }), +); +followCheckbox.addEventListener("change", () => + send({ type: "followToggle", checked: followCheckbox.checked }), +); +promptInput.addEventListener("keydown", (event) => { + if (event.key !== "Enter" || newQueryButton.disabled || newQueryButton.hidden) return; + event.preventDefault(); + newQueryButton.click(); +}); +newQueryButton.addEventListener("click", () => { + const prompt = promptInput.value.trim(); + send({ type: "newQuery", prompt: prompt || null }); +}); + +function cellIndexOf(target) { + const cell = target.closest(".cell"); + return cell ? [...gridEl.children].indexOf(cell) : -1; +} + +gridEl.addEventListener("pointerover", (event) => { + const index = cellIndexOf(event.target); + if (index >= 0) send({ type: "cellHoverEnter", index }); +}); +gridEl.addEventListener("pointerleave", () => send({ type: "cellHoverLeave" })); + +gridEl.addEventListener( + "wheel", + (event) => { + const cell = event.target.closest(".cell"); + if (!cell) return; + event.preventDefault(); + const bounds = cell.getBoundingClientRect(); + send({ + type: "cellWheel", + index: [...gridEl.children].indexOf(cell), + deltaY: event.deltaY, + x: bounds.width ? (event.clientX - bounds.left) / bounds.width : 0, + y: bounds.height ? (event.clientY - bounds.top) / bounds.height : 0, + }); + }, + { passive: false }, +); + +let drag = null; +gridEl.addEventListener("pointerdown", (event) => { + if (event.button !== 0) return; + const cell = event.target.closest(".cell"); + if (!cell) return; + const index = [...gridEl.children].indexOf(cell); + drag = { cell, index, x: event.clientX, y: event.clientY }; + try { + cell.setPointerCapture(event.pointerId); + } catch { + // Convenience only. + } + send({ type: "cellDragStart", index }); +}); +gridEl.addEventListener("pointermove", (event) => { + if (!drag) return; + const dx = event.clientX - drag.x; + const dy = event.clientY - drag.y; + const bounds = drag.cell.getBoundingClientRect(); + drag.x = event.clientX; + drag.y = event.clientY; + send({ + type: "cellDragMove", + index: drag.index, + dx: bounds.width ? dx / bounds.width : 0, + dy: bounds.height ? dy / bounds.height : 0, + }); +}); +function endDrag(event) { + if (!drag) return; + try { + drag.cell.releasePointerCapture(event.pointerId); + } catch { + // Already released. + } + send({ type: "cellDragEnd", index: drag.index }); + drag = null; +} +document.addEventListener("pointerup", endDrag); +document.addEventListener("pointercancel", endDrag); + +gridEl.addEventListener("dblclick", (event) => { + const index = cellIndexOf(event.target); + if (index >= 0) send({ type: "cellDblclick", index }); +}); + +// --- context menu, built from the last snapshot ---------------------------- +// +// Copy/open actions execute right here -- the snapshot already carries +// everything they need. Everything that mutates a real player only the +// broadcast page owns goes back over the channel as a cellMenuAction intent. + +async function copyText(text, label) { + try { + await navigator.clipboard.writeText(text); + setStatus(`copied ${label}`, "busy"); + } catch { + setStatus("clipboard blocked by the browser", "error"); + } +} + +function menuItemsFor(cellData) { + if (!cellData || cellData.empty) return []; + const { index, videoId, title, currentTime, locked, playing, zoom } = cellData; + const name = title ?? videoId; + const relay = (action) => () => send({ type: "cellMenuAction", index, action }); + return [ + { + label: "Copy video URL at time", + hint: formatTimecode(currentTime), + run: () => copyText(videoUrl(videoId, currentTime), `URL at ${formatTimecode(currentTime)}`), + }, + { label: "Copy video URL", run: () => copyText(videoUrl(videoId), "URL") }, + { label: "Copy video ID", hint: videoId, run: () => copyText(videoId, "video ID") }, + { label: "Copy title", run: () => copyText(name, "title") }, + { + label: "Open on YouTube at time", + run: () => window.open(videoUrl(videoId, currentTime), "_blank", "noopener"), + }, + { label: playing ? "Pause this cell" : "Play this cell", run: relay("togglePlay") }, + { + label: locked ? "Unlock audio" : "Lock audio to this cell", + hint: "double-click", + run: relay("toggleLock"), + }, + { label: "Restart this cell", run: relay("restart") }, + { label: "Reset zoom", hint: `${(zoom ?? 1).toFixed(2)}×`, run: relay("resetZoom") }, + { label: "Replace with next reserve", run: relay("replaceReserve") }, + ]; +} + +function closeMenu() { + menuEl.hidden = true; +} + +function openMenu(index, x, y) { + const cellData = latestCells[index]; + const items = menuItemsFor(cellData); + if (items.length === 0) return; + + menuEl.replaceChildren(); + const head = document.createElement("div"); + head.className = "head"; + head.textContent = cellData.title ?? cellData.videoId; + menuEl.appendChild(head); + + for (const item of items) { + const button = document.createElement("button"); + const label = document.createElement("span"); + label.textContent = item.label; + button.appendChild(label); + if (item.hint) { + const hint = document.createElement("span"); + hint.className = "hint"; + hint.textContent = item.hint; + button.appendChild(hint); + } + button.addEventListener("click", () => { + item.run(); + closeMenu(); + }); + menuEl.appendChild(button); + } + + menuEl.hidden = false; + menuEl.style.left = `${x}px`; + menuEl.style.top = `${y}px`; + const rect = menuEl.getBoundingClientRect(); + if (rect.right > window.innerWidth) { + menuEl.style.left = `${Math.max(0, window.innerWidth - rect.width - 4)}px`; + } + if (rect.bottom > window.innerHeight) { + menuEl.style.top = `${Math.max(0, window.innerHeight - rect.height - 4)}px`; + } +} + +gridEl.addEventListener("contextmenu", (event) => { + const index = cellIndexOf(event.target); + if (index < 0) return; + event.preventDefault(); + openMenu(index, event.clientX, event.clientY); +}); +document.addEventListener("click", (event) => { + if (!menuEl.hidden && !menuEl.contains(event.target)) closeMenu(); +}); +document.addEventListener("keydown", (event) => { + if (event.key === "Escape") closeMenu(); +}); From af248c348f3483bd95bf986871be106109df306a Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 09:15:39 -0700 Subject: [PATCH 10/14] feat: serve /layout-control locally and ship it in dist/ Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FEUEMdhXo2SQ4LjtE2twRt --- scripts/build-dist.sh | 1 + ytmatrix/main.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh index ee983e6..1b62888 100755 --- a/scripts/build-dist.sh +++ b/scripts/build-dist.sh @@ -17,6 +17,7 @@ mkdir -p "$dist/static" cp "$root/static/player.html" "$dist/index.html" cp "$root/static/config.html" "$dist/config.html" cp "$root/static/layout.html" "$dist/layout.html" +cp "$root/static/layout-control.html" "$dist/layout-control.html" mkdir -p "$dist/static/layout" cp "$root/static/layout/screens.json" "$dist/static/layout/screens.json" # *.js only: grid-logic.test.mjs is a node test and must not ship. diff --git a/ytmatrix/main.py b/ytmatrix/main.py index 6cf0257..77fa07c 100644 --- a/ytmatrix/main.py +++ b/ytmatrix/main.py @@ -78,6 +78,10 @@ async def _config_page() -> FileResponse: async def _layout_page() -> FileResponse: return FileResponse(dist / "layout.html") + @app.get("/layout-control", include_in_schema=False) + async def _layout_control_page() -> FileResponse: + return FileResponse(dist / "layout-control.html") + app.mount("/", StaticFiles(directory=dist, html=True), name="dist") uvicorn.run( From a64173be56d9438d59a1bfe50c21bd58e3e02641 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 09:26:13 -0700 Subject: [PATCH 11/14] test: two-page browser coverage for the layout control surface Opens /layout and /layout-control from one Playwright BrowserContext and proves a wheel zoom, a menu action, and a mute toggle on the control page all relay over BroadcastChannel and land on the real broadcast page. --- tests/test_layout_control_smoke.py | 218 +++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 tests/test_layout_control_smoke.py diff --git a/tests/test_layout_control_smoke.py b/tests/test_layout_control_smoke.py new file mode 100644 index 0000000..e18580e --- /dev/null +++ b/tests/test_layout_control_smoke.py @@ -0,0 +1,218 @@ +"""Browser smoke test for /layout-control: an interaction on the control +page must land on the real broadcast page, over BroadcastChannel. + +Marked `browser`, excluded from the default suite. Both pages come from one +Playwright BrowserContext -- BroadcastChannel only connects tabs in the same +browser profile, which is exactly the constraint this feature is built +around (the NDI broadcaster already only ever captures a local window). +""" + +from __future__ import annotations + +import asyncio +import os +import socket +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import pytest +import yaml +from playwright.sync_api import sync_playwright + +from ytmatrix import cache, youtube +from ytmatrix.store import FileStore + +REPO_ROOT = Path(__file__).resolve().parent.parent + +pytestmark = pytest.mark.browser + +CONFIG = { + "query": "golden cover", + "grid": {"cols": 4, "rows": 2}, + "search": { + "order": "relevance", + "video_duration": "any", + "safe_search": "moderate", + "relevance_language": "en", + }, + "playback": {"muted": True, "autoplay_on_change": True, "start_offset": 0, "loop": True}, + "cache": {"ttl_hours": 24}, + "query_generation": {"enabled": True}, +} + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture(scope="session") +def _fresh_dist(): + subprocess.run( + ["bash", str(REPO_ROOT / "scripts" / "build-dist.sh")], + cwd=str(REPO_ROOT), + check=True, + capture_output=True, + ) + + +@pytest.fixture +def running_server(tmp_path, _fresh_dist): + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(CONFIG)) + + cache_dir = tmp_path / "cache" + params = youtube.build_params("golden cover", "relevance", "any", "moderate", "en") + base = [ + "43DOm50YWaI", + "1qwq1UCG9c4", + "AuaABDWFs_8", + "0-xSuAexKTw", + "huqUnIVAjHg", + "GnDfJC1vPlQ", + "R7EH2TKJHYQ", + "uSAPVDS2LUo", + ] + ids = [base[i % len(base)] for i in range(50)] + asyncio.run( + cache.write( + FileStore(cache_dir), + params, + [{"video_id": v, "title": v, "channel": "c"} for v in ids], + ) + ) + + port = _find_free_port() + env = { + **os.environ, + "YOUTUBE_API_KEY": "SMOKE_TEST_KEY_UNUSED", + "YTMATRIX_HOST": "127.0.0.1", + "YTMATRIX_PORT": str(port), + "YTMATRIX_CONFIG_PATH": str(config_path), + "YTMATRIX_CACHE_DIR": str(cache_dir), + "YTMATRIX_RUNTIME_DIR": str(tmp_path / "runtime"), + } + process = subprocess.Popen( + [sys.executable, "-m", "ytmatrix.main"], + cwd=str(REPO_ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + try: + if ( + httpx.get( + f"https://localhost:{port}/healthz", verify=False, timeout=1.0 + ).status_code + == 200 + ): + break + except httpx.HTTPError: + pass + time.sleep(0.3) + else: + raise RuntimeError("server did not become healthy in time") + yield f"https://localhost:{port}" + finally: + process.terminate() + process.wait(timeout=5) + + +def test_a_wheel_on_the_control_page_zooms_the_real_cell(running_server): + with sync_playwright() as p: + browser = p.chromium.launch() + context = browser.new_context(ignore_https_errors=True) + broadcast = context.new_page() + control = context.new_page() + + broadcast.goto(f"{running_server}/layout", wait_until="load") + broadcast.wait_for_function( + "document.querySelectorAll('.cell iframe').length === 8", timeout=20_000 + ) + control.goto(f"{running_server}/layout-control", wait_until="load") + control.wait_for_selector(".cell", timeout=20_000) + + def broadcast_zoom(nth=0): + return broadcast.evaluate( + f"""() => {{ + const cells = document.querySelectorAll('.cell'); + const f = cells[{nth}].querySelector('iframe').getBoundingClientRect(); + const c = cells[{nth}].getBoundingClientRect(); + return f.width / c.width; + }}""" + ) + + before = broadcast_zoom() + + box = control.locator(".cell").first.bounding_box() + control.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2) + for _ in range(8): + control.mouse.wheel(0, -120) + + broadcast.wait_for_function( + f"""() => {{ + const cells = document.querySelectorAll('.cell'); + const f = cells[0].querySelector('iframe').getBoundingClientRect(); + const c = cells[0].getBoundingClientRect(); + return f.width / c.width > {before + 0.2}; + }}""", + timeout=10_000, + ) + browser.close() + + +def test_a_menu_action_on_the_control_page_pauses_the_real_player(running_server): + with sync_playwright() as p: + browser = p.chromium.launch(args=["--autoplay-policy=no-user-gesture-required"]) + context = browser.new_context(ignore_https_errors=True) + broadcast = context.new_page() + control = context.new_page() + + broadcast.goto(f"{running_server}/layout", wait_until="load") + broadcast.wait_for_function("window.__prerolled === true", timeout=40_000) + broadcast.evaluate("window.__players.forEach(p => p && p.playVideo())") + broadcast.wait_for_function( + "window.__players.some(p => p && p.getPlayerState() === 1)", timeout=10_000 + ) + + control.goto(f"{running_server}/layout-control", wait_until="load") + control.wait_for_selector('.cell[data-empty="false"]', timeout=20_000) + + control.locator(".cell").first.click(button="right") + control.wait_for_selector("#menu:not([hidden])", timeout=5_000) + control.locator("#menu button", has_text="Pause this cell").first.click() + + broadcast.wait_for_function( + "window.__players[0] && window.__players[0].getPlayerState() !== 1", timeout=10_000 + ) + browser.close() + + +def test_the_control_page_reflects_mute_state_from_the_broadcast_page(running_server): + with sync_playwright() as p: + browser = p.chromium.launch() + context = browser.new_context(ignore_https_errors=True) + broadcast = context.new_page() + control = context.new_page() + + broadcast.goto(f"{running_server}/layout", wait_until="load") + broadcast.wait_for_function("window.__prerolled === true", timeout=40_000) + + control.goto(f"{running_server}/layout-control", wait_until="load") + control.wait_for_function( + "document.getElementById('mute').textContent.trim() === 'Unmute'", timeout=10_000 + ) + + control.click("#mute") + broadcast.wait_for_function("window.__players.every(p => !p.isMuted())", timeout=10_000) + control.wait_for_function( + "document.getElementById('mute').textContent.trim() === 'Mute'", timeout=10_000 + ) + browser.close() From 0ef7868ed78089b0751cfa987f04b82ea9f0cfc6 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 09:56:10 -0700 Subject: [PATCH 12/14] fix: address final review findings -- newQuery re-entrancy guard, autoplay investigation, CLAUDE.md, and CSS cleanup applyIntent's newQuery case now checks `generating` before dispatching. The flag existed already, but the guard a local click gets is the button's own disabled state -- and a relayed click never sees that, because /layout-control's button only learns it is disabled when the next snapshot arrives up to a heartbeat later. Two fast clicks there were two generations at 100 units each. The control page also disables its button optimistically now, so the feedback is immediate either way. The open question about whether unmuting survives the gesture moving to a different document is answered empirically rather than argued: probed with Playwright Chromium launched WITHOUT --autoplay-policy=no-user-gesture-required, headless and headed, a control-page Play took all 8 players to state 1, a control-page Unmute took all 8 to isMuted()===false while all 8 stayed at state 1, and hover-to-unmute made exactly cell 0 audible with everything still playing. The policy gates starting new audible playback, not clearing the mute flag on media already running -- and preroll starts everything muted, so by the time an unmute is relayed the gate is already behind us. No kiosk launch flag is required. The mute test now pins that permanently: it plays and unmutes entirely from the control page and asserts the players are unmuted AND still at state 1, since a browser refusing the unmute could pause rather than re-mute. CLAUDE.md gains the two new files, the amended wall-engine.js and build-dist.sh rows, a third browser suite in gotcha 14, a third quota trigger in gotcha 2 (including that BroadcastChannel is a real broadcast, so two open /layout tabs both spend), and gotchas 40 and 41. layout-control.html loses the CSS it inherited from player.html and never needed: the iframe pointer-events rule, the data-preroll opacity gate, the "deliberately not styled" comment that this page contradicts, the shadowed "no playable result" empty-cell rule, and the duplicate #grid/.cell base block. user-select:none is kept, moved onto the surviving rule -- it was cascading from the deleted one and dragging would otherwise select the cell label. Also: /layout hides #menu alongside header (preventDefault only stops the browser's menu, not the app's own, which would paint into the NDI feed); publishSnapshot logs what it catches instead of swallowing it, and skips the post when buildSnapshot returns null because config has not landed yet; applyCellPan and toggleLockedIndex get the empty-cell guard applyCellWheel already had; the control page's reserve menu item shows how many are left; and a status message the control page wrote itself is held for 1.5s so the heartbeat cannot paint over a copy confirmation before it is read. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FEUEMdhXo2SQ4LjtE2twRt --- CLAUDE.md | 98 +++++++++++++++++++++++++++--- static/layout-control.html | 44 +++----------- static/layout-control.js | 35 +++++++++-- static/layout.html | 9 ++- static/wall-engine.js | 39 ++++++++++-- tests/test_layout_control_smoke.py | 32 +++++++++- 6 files changed, 200 insertions(+), 57 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7bbd835..82919f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ cp .env.example .env # YOUTUBE_API_KEY ./run.sh # https://localhost:8444/ uv run pytest tests/ -v # default suite, never hits the network node --test 'static/*.test.mjs' # pure frontend logic -uv run pytest tests/test_player_smoke.py -m browser -v # real Chromium; no quota +uv run pytest tests/ -m browser -v # real Chromium, all three suites; no quota uv run pytest -m live -v # one real search; spends 100 quota units uv run ruff check . && uv run ruff format . ``` @@ -64,13 +64,15 @@ npm run deploy # needs Docker; see docs/DEPLOY.md | `static/grid-logic.js` | Pure slot/reserve bookkeeping + config-change classification. Node-testable. | | `static/wallstate.js` | What *this browser* is watching: current query + history in `localStorage`. Every access is defensive. | | `static/socket.js` | WS connect with backoff; re-syncs on every (re)connect. | -| `static/wall-engine.js` | The actual player/DOM/YouTube-API engine (extracted from what used to be `player.js`): builds cells, pre-rolls, mute/audio targeting, zoom/pan, the context menu, the WS-driven config reconciliation. Exports `startWall({computeLayout})`; shared verbatim by `/` and `/layout`. | +| `static/wall-engine.js` | The actual player/DOM/YouTube-API engine (extracted from what used to be `player.js`): builds cells, pre-rolls, mute/audio targeting, zoom/pan, the context menu, the WS-driven config reconciliation. Exports `startWall({computeLayout, controlChannel})`; shared verbatim by `/` and `/layout`. Every interaction — a local DOM event or one relayed from `/layout-control` — funnels through the one `applyIntent()` dispatcher, and with `controlChannel` set it publishes a state snapshot over `BroadcastChannel` after each intent and on a 1s heartbeat. Both are opt-in: `/` passes no channel and behaves exactly as before. | | `static/player.js` | Bootstraps the `/` grid page via `wall-engine.js`'s `startWall()`. Two lines. | | `static/layout-fit.js` | Pure per-screen allocation/fit math for `/layout` — how a total video budget splits across the six real screens and how each screen's cells tile to approximate 16:9. No DOM, no fetch; node-tested like `grid-logic.js`. | | `static/layout-page.js` | Bootstraps `/layout`: loads `static/layout/screens.json`, builds a screens-based `computeLayout` from `layout-fit.js`'s `resolveLayout`, and calls `wall-engine.js`'s `startWall({computeLayout})`. | | `static/layout/screens.json` | Vendored, hand-copied snapshot of `../layout-driver/config/screens.yaml`'s geometry (canvas size, module size, per-screen grid/offset). No live link — see gotcha 39. | +| `static/layout-control.html` | The operator's window: the same header markup `/layout` now hides, plus a grid of plain rectangles — one per cell, no iframes. Loads no YouTube API at all. | +| `static/layout-control.js` | Drives that page. Relays every interaction to `/layout` as an intent over the `yt-matrix-layout-control` `BroadcastChannel`, and renders purely from the snapshots coming back — it fetches nothing and holds no truth of its own (gotcha 40). | | `static/config.js` | The live editor and its quota-cost indicator, including the Layout section (total/max_per_screen/per-screen overrides). | -| `scripts/build-dist.sh` | Assembles `dist/`: `player.html` → `index.html`, `config.html`, `layout.html`, `static/*.js`, `static/layout/screens.json`. Not the test `.mjs` files. | +| `scripts/build-dist.sh` | Assembles `dist/`: `player.html` → `index.html`, `config.html`, `layout.html`, `layout-control.html`, `static/*.js`, `static/layout/screens.json`. Not the test `.mjs` files. | | `tests/conftest.py` | Autouse guard: fails any default-suite test that reaches the live API. | ## Critical gotchas @@ -93,6 +95,18 @@ npm run deploy # needs Docker; see docs/DEPLOY.md or `resync()` — `resync()` also runs on every WebSocket reconnect, which is a network hiccup, not an intent. + **There is now a third trigger: a `newQuery` intent over the + `BroadcastChannel`**, sent by `/layout-control`'s own Query: button. It + reaches the same `requestNewQuery()` and spends the same 100 units. Two + things follow. The `generating` guard that stops a local double-click lives + in the button's `disabled` state, which a relayed click never sees — so + `applyIntent`'s `case "newQuery"` checks `generating` itself, and removing + that check restores a real double-spend. And `BroadcastChannel` is a genuine + broadcast, not a channel to one peer: **two open `/layout` tabs both act on + one control-page click**, which is two generations at 100 units each. One + broadcast page at a time is an operational rule, not something the code + enforces. + **The budget counter is an estimate, not a reading.** Google exposes no remaining-quota field on the YouTube Data API, and an API key cannot reach the Cloud quota APIs. `budget.py` assumes 100 units/search (documented) and @@ -282,11 +296,18 @@ npm run deploy # needs Docker; see docs/DEPLOY.md 14. **Browser bugs need a browser test.** #12 and #13 were both invisible to the Python suite and the node tests — one is script-ordering, the other a serialization gap between two languages. `tests/test_player_smoke.py` - (marked `browser`, covers `/`) and `tests/test_layout_smoke.py` (same - marker, covers `/layout`) are the only things that catch either. Run both - after any change to `wall-engine.js` — it drives both pages — and the - layout one after `layout-fit.js` or `layout-page.js` too. Run either after - a change to the config wire format. + (marked `browser`, covers `/`), `tests/test_layout_smoke.py` (same marker, + covers `/layout`) and `tests/test_layout_control_smoke.py` (same marker, + covers `/layout` and `/layout-control` together, two pages in one + `BrowserContext`) are the only things that catch either. Run **all three** + after any change to `wall-engine.js` — it drives all of it — the layout one + after `layout-fit.js` or `layout-page.js` too, and the control one after + `layout-control.js` or the snapshot/intent shapes. Run any of them after a + change to the config wire format. + + The control suite needs two pages in one context for a reason of its own: + `BroadcastChannel` connects tabs in the same browser profile and nothing + else, so a single-page test could not observe the feature at all. **The browser never loads `static/` — it loads `dist/`,** which `scripts/build-dist.sh` assembles by copying. Both the Worker's asset @@ -481,3 +502,64 @@ npm run deploy # needs Docker; see docs/DEPLOY.md *up front*, with no `onError` involved to catch a still and swap it out. Raising `total` well past `grid.cols × rows` therefore trades some of the motion filtering gotcha 16 exists for. + +40. **`/layout`'s chrome is `display: none`, and it must stay in the DOM.** + The NDI broadcaster captures that window, so the header and the context + menu may never paint — but `startWall()` opens with a dozen + `getElementById` lookups (`play`, `pause`, `mute`, `new-query`, `prompt`, + `follow`, `hover-unmute`, `status`, `audio`, `menu`, …) and every one of + them has to keep succeeding, because `/layout-control` drives those exact + elements remotely: an intent arrives, `applyIntent` runs the same code a + local click would, and the snapshot published afterwards is read straight + off `statusEl.textContent`, `muteButton.disabled` and friends. Delete the + markup instead of hiding it and those lookups return `null`, the engine + throws or silently no-ops partway through startup, and the wall renders + blank **with no console error** — the same class of invisible failure as + gotchas 12 and 13, and invisible to the Python and node suites for the same + reason (gotcha 14). Hiding is a CSS-only change on purpose; there is no + `/layout`-shaped branch anywhere in `wall-engine.js`. + + `#menu` is hidden alongside `header` for the same reason and by the same + rule. `contextmenu`'s `preventDefault()` only suppresses the *browser's* + menu; the app's own would still paint over the video and go out on the + feed. + + The direction of trust runs one way: `/layout` owns every piece of state + and `/layout-control` renders whatever the snapshot says. The control page + fetches nothing — no `/api/config`, no `/api/videos` — so there is no + second copy of the truth to drift. Giving it its own fetch to "make it + load faster" reintroduces exactly that. + +41. **A control-page gesture still starts and unmutes `/layout`, and this was + measured rather than assumed.** The worry is reasonable: Chrome's autoplay + policy gates audible playback on user activation, `BroadcastChannel` + confers no activation on the receiving document, and after `/layout`'s + header was hidden the buttons a human presses moved to a different + document entirely — so it looked like the wall might never be able to make + a sound again. + + It works. Probed against a real server with Playwright Chromium launched + **without** `--autoplay-policy=no-user-gesture-required`, headless and + headed, both identical: after pre-roll all eight players sat at state 2 + (paused, muted); a real click on `/layout-control`'s Play took all 8 to + state 1; a click on its Unmute took all 8 to `isMuted() === false` **and + left all 8 at state 1**; and with hover-to-unmute on, hovering cell 0 gave + `muted=[false, true × 7]` with every player still playing. + + The nuance that makes it safe: the policy gates *starting new audible + playback*, not clearing the mute flag on media that is already running. + `prerollCurrentSet()` (gotcha 18) starts every player muted, which needs no + gesture and is why it is written that way — so by the time any unmute is + relayed, the media element is already playing and past the gate. Gotcha 5's + "unmuting is only permitted off a user gesture" is about *starting* unmuted, + and remains true. + + `test_the_control_page_reflects_mute_state_from_the_broadcast_page`, in + `tests/test_layout_control_smoke.py`, is the standing proof: it launches with no + autoplay flag, plays and unmutes entirely from the control page, and + asserts the players are unmuted **and still at state 1**. That last + assertion is the point — a browser refusing the unmute could pause the + media instead of reporting muted. Do not add `--autoplay-policy` to that + test to "make it match the others": the absence of the flag is what it + tests. The menu test above it does use the flag, for the unrelated reason + that it needs playback started from the page's own script. diff --git a/static/layout-control.html b/static/layout-control.html index 5d0761d..aa5cb61 100644 --- a/static/layout-control.html +++ b/static/layout-control.html @@ -48,9 +48,6 @@ display: flex; align-items: center; gap: 6px; color: #8a8a94; user-select: none; cursor: pointer; white-space: nowrap; } - /* data-audible and data-zoomed carry state for the JS and the browser - tests. They are deliberately not styled: the wall shows video, nothing - painted on top of it. */ #prompt { font: inherit; padding: 7px 11px; border-radius: 6px; min-width: 230px; border: 1px solid #3a3a42; background: #131317; color: #e8e8ea; @@ -77,41 +74,18 @@ #menu button:hover { background: #262630; } #menu button .hint { color: #6f6f7a; font-size: 11px; } a { color: #8a8a94; flex: none; white-space: nowrap; } - #grid { flex: 1; display: grid; gap: 2px; padding: 2px; min-height: 0; } - /* Hidden while pre-rolling. Every cell has to play briefly to buffer, and - without this you watch eight videos flicker on and stop again on every - new query. opacity (not visibility/display) because those let the browser - throttle or discard the very playback we are waiting on. - Hiding is instant and only the reveal fades: a transition on the way out - leaves the outgoing set visible while the new one is already loading. */ - #grid[data-preroll="true"] { opacity: 0; transition: none; } - #grid[data-preroll="false"] { opacity: 1; transition: opacity .22s ease-out; } - /* overflow:hidden is what performs the crop -- player.js oversizes the - iframe past the cell bounds and this clips it. */ - .cell { - position: relative; background: #000; overflow: hidden; - cursor: grab; user-select: none; - } - .cell[data-dragging="true"] { cursor: grabbing; } - /* pointer-events:none is doing real work, not just tidying: YouTube reveals - the title bar, channel avatar and control overlay on mouse-over, and there - is no player parameter that suppresses them. If the cursor can never reach - the iframe, they are never summoned. Everything here is driven through the - JS API, so nothing needs to click the player. */ - .cell iframe { position: absolute; border: 0; display: block; pointer-events: none; } - .cell[data-empty="true"]::after { - content: "no playable result"; - position: absolute; inset: 0; display: grid; place-items: center; - color: #55555e; font-size: 12px; - } - /* .cell here is a plain rectangle, not a player mount: no iframe, no - pointer-events trick needed (there is nothing to protect the cursor - from) -- just a label and a state-driven outline so an operator can see - at a glance which cell is currently audible or locked. */ + /* This page has no iframes and no pre-roll: it draws one plain rectangle + per cell, absolutely positioned from the `rect` the snapshot carries, so + it and /layout can never disagree about geometry. Nothing here needs + player.html's cover-fit crop, its pointer-events:none guard against + YouTube's hover chrome, or its data-preroll opacity gate -- and unlike + /layout, data-audible and data-locked ARE painted here, because seeing at + a glance which cell is currently making the sound is the whole reason an + operator has this window open. */ #grid { flex: 1; position: relative; min-height: 0; } .cell { position: absolute; background: #000; overflow: hidden; - border: 1px solid #26262b; cursor: grab; + border: 1px solid #26262b; cursor: grab; user-select: none; } .cell[data-dragging="true"] { cursor: grabbing; } .cell[data-audible="true"] { outline: 2px solid #5fb87d; outline-offset: -2px; } diff --git a/static/layout-control.js b/static/layout-control.js index 07d8451..75f24e9 100644 --- a/static/layout-control.js +++ b/static/layout-control.js @@ -24,23 +24,41 @@ function send(intent) { channel.postMessage(intent); } +// How long a message this page wrote itself ("copied URL", "clipboard +// blocked") is protected from the next snapshot. /layout's heartbeat arrives +// every second, so without this window a copy confirmation can be painted over +// before it has been read -- and it is the only feedback the action gives. +const LOCAL_STATUS_HOLD_MS = 1500; + +let localStatusAt = 0; + function setStatus(text, state = "") { statusEl.textContent = text; statusEl.dataset.state = state; } +// Status this page owns, rather than /layout's own line relayed through. +function setLocalStatus(text, state = "") { + localStatusAt = Date.now(); + setStatus(text, state); +} + setStatus("waiting for /layout to connect…", "busy"); // The last snapshot's cells, kept only so the context menu can be built // without a round trip -- every mutating action still goes back over the // channel as an intent. let latestCells = []; +// The last snapshot's `global` block, for the same reason: the reserve count +// the menu reports lives there. +let latestGlobal = {}; let staleTimer = null; function renderFromSnapshot(snapshot) { latestCells = snapshot.cells; + latestGlobal = snapshot.global; const g = snapshot.global; - setStatus(g.status, g.statusState); + if (Date.now() - localStatusAt >= LOCAL_STATUS_HOLD_MS) setStatus(g.status, g.statusState); audioEl.textContent = g.audioIndicatorText; audioEl.dataset.locked = String(g.audioLocked); muteButton.textContent = g.muted ? "Unmute" : "Mute"; @@ -103,6 +121,11 @@ promptInput.addEventListener("keydown", (event) => { }); newQueryButton.addEventListener("click", () => { const prompt = promptInput.value.trim(); + // Disable optimistically. /layout's dispatcher is the real guard against a + // double 100-unit spend, but its answer only arrives with the next snapshot + // -- up to a heartbeat away -- and a button that stays live for a second + // after a click invites the second click. + newQueryButton.disabled = true; send({ type: "newQuery", prompt: prompt || null }); }); @@ -190,9 +213,9 @@ gridEl.addEventListener("dblclick", (event) => { async function copyText(text, label) { try { await navigator.clipboard.writeText(text); - setStatus(`copied ${label}`, "busy"); + setLocalStatus(`copied ${label}`, "busy"); } catch { - setStatus("clipboard blocked by the browser", "error"); + setLocalStatus("clipboard blocked by the browser", "error"); } } @@ -222,7 +245,11 @@ function menuItemsFor(cellData) { }, { label: "Restart this cell", run: relay("restart") }, { label: "Reset zoom", hint: `${(zoom ?? 1).toFixed(2)}×`, run: relay("resetZoom") }, - { label: "Replace with next reserve", run: relay("replaceReserve") }, + { + label: "Replace with next reserve", + hint: `${latestGlobal.reservesLeft ?? 0} left`, + run: relay("replaceReserve"), + }, ]; } diff --git a/static/layout.html b/static/layout.html index 7531c54..734b66c 100644 --- a/static/layout.html +++ b/static/layout.html @@ -23,8 +23,13 @@ /* Hidden, not removed: wall-engine.js's getElementById lookups for every header control must keep succeeding, since /layout-control drives them remotely. This is what keeps the NDI broadcast clean -- nothing here is - a JS change. */ - header { display: none; } + a JS change. + + #menu goes with it. contextmenu's preventDefault() only stops the + browser's own menu; the app's custom one would still paint over the wall + and straight into the broadcast feed. Right-clicking is done on + /layout-control, which relays the chosen action as an intent. */ + header, #menu { display: none; } button { font: inherit; font-weight: 600; padding: 7px 13px; border-radius: 6px; border: 1px solid #3a3a42; background: #1c1c21; color: #e8e8ea; cursor: pointer; diff --git a/static/wall-engine.js b/static/wall-engine.js index 335f8bb..f438cdd 100644 --- a/static/wall-engine.js +++ b/static/wall-engine.js @@ -543,6 +543,10 @@ function rebuild() { let broadcastChannel = null; function buildSnapshot() { + // The heartbeat below fires on a timer, which can beat the first resync() to + // the punch -- and computeLayout(config) is the first thing this does. + // Nothing useful to publish yet; the next tick will have it. + if (!config) return null; const layout = computeLayout(config); const cells = slotState.slots.map((videoId, index) => { const player = players[index]; @@ -595,9 +599,15 @@ function buildSnapshot() { function publishSnapshot() { if (!broadcastChannel) return; try { - broadcastChannel.postMessage(buildSnapshot()); - } catch { - // A channel can throw if it has already been closed; nothing useful to do. + const snapshot = buildSnapshot(); + if (!snapshot) return; + broadcastChannel.postMessage(snapshot); + } catch (error) { + // A closed channel is the expected case, but this catch also covers + // anything buildSnapshot() itself threw -- and a control page silently + // frozen on a stale snapshot is exactly the sort of failure that costs an + // afternoon. Say so. + wlog("publishSnapshot failed", error); } } @@ -1112,6 +1122,10 @@ function togglePlayForCell(index) { } function toggleLockedIndex(index) { + // Locking audio to a cell with no player would leave the wall silent with + // an indicator claiming otherwise. The local dblclick handler already + // refuses an empty cell; a relayed cellDblclick reaches here directly. + if (gridEl.children[index]?.dataset.empty === "true") return; lockedIndex = lockedIndex === index ? null : index; for (const other of gridEl.children) delete other.dataset.locked; const cell = gridEl.children[index]; @@ -1344,7 +1358,9 @@ function applyCellWheel(index, deltaY, xFraction, yFraction) { // how big /layout-control's rectangle happens to be. function applyCellPan(index, dxFraction, dyFraction) { const cell = gridEl.children[index]; - if (!cell) return; + // Same guard applyCellWheel has: an empty cell has no iframe to move, and + // storing a view for it would survive into whatever reserve lands there. + if (!cell || cell.dataset.empty === "true") return; const bounds = cell.getBoundingClientRect(); views.set( index, @@ -1379,8 +1395,8 @@ function applyCellMenuAction(index, action) { // on THIS page, or one relayed from /layout-control over BroadcastChannel -- // funnels through here. That symmetry is the whole point: a control-window // message and a local click must produce identical effects, so this is the -// only place either kind is handled. publishSnapshot() (added when -// BroadcastChannel wiring lands) is a no-op until then. +// only place either kind is handled. publishSnapshot() at the end is a no-op +// unless startWall was given a controlChannel, so every path can call it. async function applyIntent(intent) { switch (intent.type) { case "play": @@ -1402,6 +1418,17 @@ async function applyIntent(intent) { resetAllViews(); break; case "newQuery": + // `generating` is already set synchronously by requestNewQuery and + // cleared in its finally, so this is the same guard the local button's + // disabled state gives a local click. A relayed intent never sees that + // disabled state -- /layout-control's own button only learns it is + // disabled when the next snapshot arrives, up to a heartbeat later -- + // so the dispatcher itself has to hold the line. Two fast clicks over + // the channel would otherwise be two generations at 100 units each. + if (generating) { + wlog("ignoring newQuery: one is already in flight"); + break; + } await requestNewQuery(intent.prompt ?? null); break; case "hoverUnmuteToggle": diff --git a/tests/test_layout_control_smoke.py b/tests/test_layout_control_smoke.py index e18580e..3e62508 100644 --- a/tests/test_layout_control_smoke.py +++ b/tests/test_layout_control_smoke.py @@ -178,8 +178,11 @@ def test_a_menu_action_on_the_control_page_pauses_the_real_player(running_server broadcast.goto(f"{running_server}/layout", wait_until="load") broadcast.wait_for_function("window.__prerolled === true", timeout=40_000) broadcast.evaluate("window.__players.forEach(p => p && p.playVideo())") + # Specifically cell 0, which is the one the menu action below targets. + # "some player is playing" would pass with cell 0 never having started, + # and then the assertion at the end would be vacuously true. broadcast.wait_for_function( - "window.__players.some(p => p && p.getPlayerState() === 1)", timeout=10_000 + "window.__players[0] && window.__players[0].getPlayerState() === 1", timeout=10_000 ) control.goto(f"{running_server}/layout-control", wait_until="load") @@ -196,6 +199,15 @@ def test_a_menu_action_on_the_control_page_pauses_the_real_player(running_server def test_the_control_page_reflects_mute_state_from_the_broadcast_page(running_server): + """Play and unmute both survive the gesture moving to another document. + + Launched with NO `--autoplay-policy=no-user-gesture-required`, unlike the + menu test above -- that is the point. The buttons an operator actually + presses now live on /layout-control, and BroadcastChannel confers no user + activation on the receiving document, so /layout starts and unmutes eight + players having never been clicked. This is the test that says that works; + see CLAUDE.md gotcha 41 for why it is allowed to. + """ with sync_playwright() as p: browser = p.chromium.launch() context = browser.new_context(ignore_https_errors=True) @@ -206,12 +218,28 @@ def test_the_control_page_reflects_mute_state_from_the_broadcast_page(running_se broadcast.wait_for_function("window.__prerolled === true", timeout=40_000) control.goto(f"{running_server}/layout-control", wait_until="load") + # The mute button already reads "Unmute" in the page's static HTML, so + # waiting on its text proves nothing. The status line starting out as + # this page's own placeholder and then changing IS proof a snapshot + # arrived -- only renderFromSnapshot ever replaces it. control.wait_for_function( - "document.getElementById('mute').textContent.trim() === 'Unmute'", timeout=10_000 + "document.getElementById('status').textContent !== 'waiting for /layout to connect…'", + timeout=10_000, + ) + + control.click("#play") + broadcast.wait_for_function( + "window.__players.every(p => p && p.getPlayerState() === 1)", timeout=20_000 ) control.click("#mute") broadcast.wait_for_function("window.__players.every(p => !p.isMuted())", timeout=10_000) + # Still playing, specifically. A browser that refused the unmute would + # not necessarily report muted -- it could pause the media instead, and + # a wall that goes silent-and-frozen is the failure worth naming. + broadcast.wait_for_function( + "window.__players.every(p => p && p.getPlayerState() === 1)", timeout=10_000 + ) control.wait_for_function( "document.getElementById('mute').textContent.trim() === 'Mute'", timeout=10_000 ) From b4bce644bee84fc593554b69bad40b2001af177c Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 10:56:19 -0700 Subject: [PATCH 13/14] fix: explain the same-browser BroadcastChannel requirement at startup The only symptom of opening /layout-control in a different browser (or, in production, alongside the broadcaster's own separate off-screen Chrome instance) was this page sitting on "waiting" forever with nothing to click. Say so in the status line and the console. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FEUEMdhXo2SQ4LjtE2twRt --- static/layout-control.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/static/layout-control.js b/static/layout-control.js index 75f24e9..23704d3 100644 --- a/static/layout-control.js +++ b/static/layout-control.js @@ -43,7 +43,19 @@ function setLocalStatus(text, state = "") { setStatus(text, state); } -setStatus("waiting for /layout to connect…", "busy"); +// BroadcastChannel only bridges tabs/windows within the SAME browser process +// and profile -- it cannot reach a /layout running in a different browser, a +// different profile, or (the production case) the broadcaster's own headed- +// but-off-screen Chrome instance. Say so loudly here, since the only other +// symptom is this page sitting on "waiting" forever with nothing to click. +console.log( + `[layout-control] connecting on BroadcastChannel("${CHANNEL_NAME}"). ` + + "This only reaches a /layout tab open in this SAME browser process/profile " + + "-- not a different browser, a different profile, or the NDI broadcaster's " + + "own off-screen Chrome instance, which is a separate process even if it is " + + "the same browser application.", +); +setStatus("waiting for /layout to connect… (must be open in this same browser)", "busy"); // The last snapshot's cells, kept only so the context menu can be built // without a round trip -- every mutating action still goes back over the @@ -94,7 +106,10 @@ function renderFromSnapshot(snapshot) { clearTimeout(staleTimer); staleTimer = setTimeout(() => { - setStatus("no update from /layout in a while — is it still open?", "error"); + setStatus( + "no update from /layout in a while — is it open in this same browser?", + "error", + ); }, STALE_AFTER_MS); } From 3987361e924cce83ecaadb3cf3203a6b97cc3e59 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Sat, 29 Aug 2026 16:03:15 -0700 Subject: [PATCH 14/14] feat: POST /api/intent -- a second front door into applyIntent() For an external controller with no browser to share a BroadcastChannel with (e.g. chasa, an OSC/show-control router), relays a wall-wide control intent over the same /ws connection put_config already uses for config broadcasts. wall-engine.js's socket handler calls applyIntent() on it directly -- no new dispatch logic, one new branch. Deliberately restricted to WALL_WIDE_INTENT_TYPES (play, pause, muteToggle, rewind, shuffle, resetView, newQuery, hoverUnmuteToggle, followToggle), not the cell-indexed intents -- those carry an index tied to whichever page's grid is open, and this broadcasts to every connected tab, not just one wall. Verified against a real chasa config end to end: real OSC UDP cues arrived on yt-matrix's /ws as {"type": "intent", ...} within milliseconds, exactly matching what applyIntent() expects. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FEUEMdhXo2SQ4LjtE2twRt --- CLAUDE.md | 31 +++++++++++++++++++++++ static/wall-engine.js | 16 ++++++++---- tests/test_player_smoke.py | 33 ++++++++++++++++++++++++ tests/test_server.py | 34 +++++++++++++++++++++++++ ytmatrix/server.py | 51 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 160 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 82919f8..b6fdd53 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -563,3 +563,34 @@ npm run deploy # needs Docker; see docs/DEPLOY.md test to "make it match the others": the absence of the flag is what it tests. The menu test above it does use the flag, for the unrelated reason that it needs playback started from the page's own script. + +42. **`POST /api/intent` is a second front door into `applyIntent()`, for a + controller that has no browser to share a `BroadcastChannel` with.** + `/layout-control` relays intents to `/layout` only because both are pages + in the *same* browser profile (gotcha 40's whole premise). An external + show-control/OSC router (e.g. `chasa`, in `../bardo-tools/chasa`) is a + separate process with no browser at all, so it goes in over HTTP instead: + `POST /api/intent` validates `type` against `WALL_WIDE_INTENT_TYPES` + (`ytmatrix/server.py`) and broadcasts `{"type": "intent", "intent": + payload}` over the same `/ws` connection `put_config` already uses for + config pushes. `wall-engine.js`'s socket `onMessage` calls `applyIntent()` + on it directly — one new branch, no new dispatch logic. + + Deliberately restricted to the wall-wide intents (`play`, `pause`, + `muteToggle`, `rewind`, `shuffle`, `resetView`, `newQuery`, + `hoverUnmuteToggle`, `followToggle`) — **not** the cell-indexed ones + (`cellWheel`, `cellDblclick`, `cellMenuAction`, `cellHoverEnter/Leave`, + `cellDragStart/Move/End`). Those carry an index tied to whichever page's + grid is open, and this broadcasts to *every* connected tab, `/` and + `/layout` alike — not just one wall. `newQuery` carries gotcha 29/2's own + quota-multiplication risk for the same reason: every open tab that + receives it independently spends its own 100-unit search. One wall/tab + open at a time when driving this from an external cue source is an + operational rule, not something the endpoint enforces. + + A Python REST client (chasa's `httpx`-based `rest` action) does not trust + this server's local mkcert certificate by default — `httpx` uses + `certifi`, not the macOS keychain, so it rejects the connection even + though a browser or `curl` accepts it. Confirmed live: setting + `SSL_CERT_FILE=$(mkcert -CAROOT)/rootCA.pem` in the caller's environment + fixes it with no code change on either side. diff --git a/static/wall-engine.js b/static/wall-engine.js index f438cdd..89b61ad 100644 --- a/static/wall-engine.js +++ b/static/wall-engine.js @@ -1586,13 +1586,19 @@ if (controlChannel) { connectSocket({ onReconnect: resync, - // Config is the only thing broadcast. Nothing pushes a video set any more: - // the server does not know what query any given browser is watching, so a - // wall only ever changes its own videos -- on its own resync, or off its own - // New query. Deciding whether a config change means this browser has to - // refetch is therefore the client's job, done right here. + // Config and relayed control intents (POST /api/intent, e.g. from an + // external OSC/show-control router) are the only things broadcast. Nothing + // pushes a video set any more: the server does not know what query any + // given browser is watching, so a wall only ever changes its own videos -- + // on its own resync, or off its own New query. Deciding whether a config + // change means this browser has to refetch is therefore the client's job, + // done right here. onMessage: (message) => { wlog(`socket message type=${message.type}`); + if (message.type === "intent") { + applyIntent(message.intent); + return; + } if (message.type !== "config") return; const previous = config; const change = classifyConfigChange(previous, message.config); diff --git a/tests/test_player_smoke.py b/tests/test_player_smoke.py index a601888..3d685e7 100644 --- a/tests/test_player_smoke.py +++ b/tests/test_player_smoke.py @@ -248,6 +248,39 @@ def test_pause_stops_every_player(running_server): browser.close() +def test_a_posted_intent_reaches_the_wall_over_websocket(running_server): + """POST /api/intent is the second front door into applyIntent() -- for a + controller with no browser to share a BroadcastChannel with (e.g. chasa, + an OSC router). This proves the whole relay: HTTP POST -> server + broadcast -> this page's own /ws connection -> applyIntent() -> a real + effect on the players, with no button ever clicked. + """ + with sync_playwright() as p: + browser = p.chromium.launch(args=["--autoplay-policy=no-user-gesture-required"]) + page = browser.new_context(ignore_https_errors=True).new_page() + page.goto(running_server, wait_until="load") + page.wait_for_function( + "window.__players?.every(p => typeof p?.pauseVideo === 'function')", timeout=25_000 + ) + page.wait_for_function("window.__prerolled === true", timeout=40_000) + page.evaluate(""" + window.__pauseCalls = 0; + for (const player of window.__players) { + const original = player.pauseVideo.bind(player); + player.pauseVideo = () => { window.__pauseCalls += 1; return original(); }; + } + """) + page.click("#play") + + response = httpx.post( + f"{running_server}api/intent", json={"type": "pause"}, verify=False, timeout=5.0 + ) + assert response.status_code == 200 + + page.wait_for_function("window.__pauseCalls >= 8", timeout=15_000) + browser.close() + + def test_a_new_set_stays_paused_until_pre_rolled(running_server): """Nothing starts until every cell has buffered, and not even then.""" with sync_playwright() as p: diff --git a/tests/test_server.py b/tests/test_server.py index caf1b3b..3bbf2b8 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -209,6 +209,40 @@ def test_a_cosmetic_change_broadcasts_config_only(app_env): assert first["config"]["playback"]["start_offset"] == 45 +def test_a_wall_wide_intent_is_broadcast_to_every_connection(app_env): + app, _, _ = app_env + with TestClient(app) as client, client.websocket_connect("/ws") as ws: + response = client.post("/api/intent", json={"type": "muteToggle"}) + assert response.status_code == 200 + message = json.loads(ws.receive_text()) + assert message == {"type": "intent", "intent": {"type": "muteToggle"}} + + +def test_an_intent_with_extra_fields_is_relayed_verbatim(app_env): + """hoverUnmuteToggle/followToggle carry a `checked` flag; newQuery carries + an optional `prompt`. The endpoint does not know or care about a given + type's extra fields -- it relays whatever the caller sent, and the + client-side applyIntent() dispatcher is what interprets them.""" + app, _, _ = app_env + with TestClient(app) as client, client.websocket_connect("/ws") as ws: + client.post("/api/intent", json={"type": "hoverUnmuteToggle", "checked": True}) + message = json.loads(ws.receive_text()) + assert message == {"type": "intent", "intent": {"type": "hoverUnmuteToggle", "checked": True}} + + +def test_an_unknown_intent_type_is_rejected_with_422(app_env): + app, _, _ = app_env + with TestClient(app) as client: + response = client.post("/api/intent", json={"type": "cellMenuAction", "index": 0}) + assert response.status_code == 422 + + +def test_a_missing_intent_type_is_rejected_with_422(app_env): + app, _, _ = app_env + with TestClient(app) as client: + assert client.post("/api/intent", json={}).status_code == 422 + + def test_quota_exhaustion_falls_back_to_stale_cache(app_env, monkeypatch): app, _, store = app_env ids = seed_cache(store) diff --git a/ytmatrix/server.py b/ytmatrix/server.py index c579561..14cb228 100644 --- a/ytmatrix/server.py +++ b/ytmatrix/server.py @@ -30,6 +30,23 @@ logger = logging.getLogger(__name__) +# /api/intent's allowlist. Deliberately excludes every cell-indexed intent +# (cellWheel, cellDblclick, cellMenuAction, cellHoverEnter/Leave, +# cellDragStart/Move/End) -- those carry a cell index tied to whichever +# page's grid is open, and this endpoint broadcasts to every connected tab, +# not just one wall. See post_intent's docstring. +WALL_WIDE_INTENT_TYPES = { + "play", + "pause", + "muteToggle", + "rewind", + "shuffle", + "resetView", + "newQuery", + "hoverUnmuteToggle", + "followToggle", +} + class BudgetExceededError(RuntimeError): """The self-imposed daily ceiling would be crossed by another search.""" @@ -794,6 +811,40 @@ async def new_query(request: Request, payload: dict | None = None) -> dict: extra_timings={"gemini": gemini_secs}, ) + @app.post("/api/intent") + async def post_intent(payload: dict) -> dict: + """Relay a wall-wide control intent to every connected browser tab. + + For an external controller (a show-control/OSC router, e.g. chasa) + that cannot join a browser's own BroadcastChannel the way + /layout-control does -- this is a second front door into the same + applyIntent() dispatcher in wall-engine.js, over the /ws connection + every wall already holds open for config pushes. + + Deliberately restricted to WALL_WIDE_INTENT_TYPES: a cell-indexed + intent (cellWheel, cellDblclick, cellMenuAction, ...) carries an + index tied to whichever page's grid is open, and this broadcasts to + every connected tab -- / and /layout alike -- not just one wall. + + newQuery carries the same quota-multiplication risk documented for + /layout-control's own Query button (gotcha 29/2): every open tab + that receives this independently calls requestNewQuery, each one a + 100-unit search. One wall/tab open at a time when driving this from + an external cue source is an operational rule, not something this + endpoint enforces. + """ + intent_type = payload.get("type") + if intent_type not in WALL_WIDE_INTENT_TYPES: + raise HTTPException( + status_code=422, + detail=( + f"unknown intent type {intent_type!r}; must be one of " + f"{sorted(WALL_WIDE_INTENT_TYPES)}" + ), + ) + await manager.broadcast({"type": "intent", "intent": payload}) + return {"status": "ok"} + @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket) -> None: await manager.connect(websocket)