[codex] Add integrated browser preview, annotations, and agent automation

apps/desktop/package.json

@@ -20,7 +20,8 @@
     "@t3tools/tailscale": "workspace:*",
     "effect": "catalog:",
     "electron": "41.5.0",
-    "electron-updater": "^6.6.2"
+    "electron-updater": "^6.6.2",
+    "react-grab": "^0.1.32"
   },
   "devDependencies": {
     "@effect/vitest": "catalog:",

apps/desktop/scripts/dev-electron.mjs

@@ -2,7 +2,11 @@ import { spawn, spawnSync } from "node:child_process";
 import { watch } from "node:fs";
 import { join } from "node:path";
 
-import { desktopDir, resolveDevProtocolClient, resolveElectronPath } from "./electron-launcher.mjs";
+import {
+  desktopDir,
+  resolveDevProtocolClient,
+  resolveElectronLaunchCommand,
+} from "./electron-launcher.mjs";
 import { waitForResources } from "./wait-for-resources.mjs";
 
 const devServerUrl = process.env.VITE_DEV_SERVER_URL?.trim();
@@ -79,7 +83,8 @@ function startApp() {
   const launchArgs = devProtocolClient
     ? electronArgs
     : [...electronArgs, `--t3code-dev-root=${desktopDir}`, "dist-electron/main.cjs"];
-  const app = spawn(resolveElectronPath(), launchArgs, {
+  const electronCommand = resolveElectronLaunchCommand(launchArgs);
+  const app = spawn(electronCommand.electronPath, electronCommand.args, {
     cwd: desktopDir,
     env: childEnv,
     stdio: "inherit",

apps/desktop/scripts/electron-launcher.mjs

@@ -307,6 +307,31 @@ function buildMacLauncher(electronBinaryPath) {
   return targetBinaryPath;
 }
 
+function isLinuxSetuidSandboxConfigured(electronBinaryPath) {
+  if (process.platform !== "linux") {
+    return true;
+  }
+
+  const sandboxPath = join(dirname(electronBinaryPath), "chrome-sandbox");
+  try {
+    const sandboxStat = statSync(sandboxPath);
+    return sandboxStat.uid === 0 && (sandboxStat.mode & 0o4777) === 0o4755;
+  } catch {
+    return false;
+  }
+}
+
+function resolveLinuxSandboxArgs(electronBinaryPath) {
+  if (isLinuxSetuidSandboxConfigured(electronBinaryPath)) {
+    return [];
+  }
+
+  console.warn(
+    "[desktop-launcher] Electron chrome-sandbox is not root-owned with mode 4755; launching local Electron with --no-sandbox.",
+  );
+  return ["--no-sandbox"];
+}
+
 export function resolveElectronPath() {
   ensureElectronRuntime();
 
@@ -320,6 +345,14 @@ export function resolveElectronPath() {
   return buildMacLauncher(electronBinaryPath);
 }
 
+export function resolveElectronLaunchCommand(args = []) {
+  const electronPath = resolveElectronPath();
+  return {
+    electronPath,
+    args: [...resolveLinuxSandboxArgs(electronPath), ...args],
+  };
+}
+
 export function resolveDevProtocolClient() {
   if (process.platform !== "darwin" || !isDevelopment) {
     return null;

apps/desktop/scripts/smoke-test.mjs

@@ -1,15 +1,16 @@
 import { spawn } from "node:child_process";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { resolveElectronLaunchCommand } from "./electron-launcher.mjs";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const desktopDir = resolve(__dirname, "..");
-const electronBin = resolve(desktopDir, "node_modules/.bin/electron");
 const mainJs = resolve(desktopDir, "dist-electron/main.cjs");
 
 console.log("\nLaunching Electron smoke test...");
 
-const child = spawn(electronBin, [mainJs], {
+const electronCommand = resolveElectronLaunchCommand([mainJs]);
+const child = spawn(electronCommand.electronPath, electronCommand.args, {
   stdio: ["pipe", "pipe", "pipe"],
   env: {
     ...process.env,

apps/desktop/scripts/start-electron.mjs

@@ -1,11 +1,12 @@
 import { spawn } from "node:child_process";
 
-import { desktopDir, resolveElectronPath } from "./electron-launcher.mjs";
+import { desktopDir, resolveElectronLaunchCommand } from "./electron-launcher.mjs";
 
 const childEnv = { ...process.env };
 delete childEnv.ELECTRON_RUN_AS_NODE;
 
-const child = spawn(resolveElectronPath(), ["dist-electron/main.cjs"], {
+const electronCommand = resolveElectronLaunchCommand(["dist-electron/main.cjs"]);
+const child = spawn(electronCommand.electronPath, electronCommand.args, {
   stdio: "inherit",
   cwd: desktopDir,
   env: childEnv,

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -48,6 +48,7 @@ import {
   setTheme,
   showContextMenu,
 } from "./methods/window.ts";
+import { previewMethods } from "./methods/preview.ts";
 
 export const installDesktopIpcHandlers = Effect.gen(function* () {
   const ipc = yield* DesktopIpc.DesktopIpc;
@@ -92,4 +93,7 @@ export const installDesktopIpcHandlers = Effect.gen(function* () {
   yield* ipc.handle(downloadUpdate);
   yield* ipc.handle(installUpdate);
   yield* ipc.handle(checkForUpdate);
+  for (const previewMethod of previewMethods) {
+    yield* ipc.handle(previewMethod);
+  }
 }).pipe(Effect.withSpan("desktop.ipc.installHandlers"));

apps/desktop/src/ipc/channels.ts

@@ -39,3 +39,29 @@ export const SET_SERVER_EXPOSURE_MODE_CHANNEL = "desktop:set-server-exposure-mod
 export const SET_TAILSCALE_SERVE_ENABLED_CHANNEL = "desktop:set-tailscale-serve-enabled";
 export const GET_ADVERTISED_ENDPOINTS_CHANNEL = "desktop:get-advertised-endpoints";
 export const SSH_PASSWORD_PROMPT_CANCELLED_RESULT = "ssh-password-prompt-cancelled";
+export const PREVIEW_CREATE_TAB_CHANNEL = "desktop:preview-create-tab";
+export const PREVIEW_CLOSE_TAB_CHANNEL = "desktop:preview-close-tab";
+export const PREVIEW_REGISTER_WEBVIEW_CHANNEL = "desktop:preview-register-webview";
+export const PREVIEW_NAVIGATE_CHANNEL = "desktop:preview-navigate";
+export const PREVIEW_GO_BACK_CHANNEL = "desktop:preview-go-back";
+export const PREVIEW_GO_FORWARD_CHANNEL = "desktop:preview-go-forward";
+export const PREVIEW_REFRESH_CHANNEL = "desktop:preview-refresh";
+export const PREVIEW_ZOOM_IN_CHANNEL = "desktop:preview-zoom-in";
+export const PREVIEW_ZOOM_OUT_CHANNEL = "desktop:preview-zoom-out";
+export const PREVIEW_RESET_ZOOM_CHANNEL = "desktop:preview-reset-zoom";
+export const PREVIEW_HARD_RELOAD_CHANNEL = "desktop:preview-hard-reload";
+export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
+export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies";
+export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";
+export const PREVIEW_GET_CONFIG_CHANNEL = "desktop:preview-get-config";
+export const PREVIEW_PICK_ELEMENT_CHANNEL = "desktop:preview-pick-element";
+export const PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL = "desktop:preview-cancel-pick-element";
+export const PREVIEW_AUTOMATION_STATUS_CHANNEL = "desktop:preview-automation-status";
+export const PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL = "desktop:preview-automation-snapshot";
+export const PREVIEW_AUTOMATION_CLICK_CHANNEL = "desktop:preview-automation-click";
+export const PREVIEW_AUTOMATION_TYPE_CHANNEL = "desktop:preview-automation-type";
+export const PREVIEW_AUTOMATION_PRESS_CHANNEL = "desktop:preview-automation-press";
+export const PREVIEW_AUTOMATION_SCROLL_CHANNEL = "desktop:preview-automation-scroll";
+export const PREVIEW_AUTOMATION_EVALUATE_CHANNEL = "desktop:preview-automation-evaluate";
+export const PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL = "desktop:preview-automation-wait-for";
+export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";

apps/desktop/src/ipc/methods/preview.test.ts

@@ -0,0 +1,28 @@
+import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
+
+const fromPartition = vi.fn(() => {
+  throw new Error("Session can only be received when app is ready");
+});
+
+vi.mock("electron", () => ({
+  BrowserWindow: {
+    getAllWindows: vi.fn(() => []),
+  },
+  session: {
+    fromPartition,
+  },
+  webContents: {
+    fromId: vi.fn(() => null),
+  },
+}));
+
+describe("preview IPC methods", () => {
+  beforeEach(() => {
+    fromPartition.mockClear();
+  });
+
+  it("does not access the Electron session while the module loads", async () => {
+    await expect(import("./preview.ts")).resolves.toBeDefined();
+    expect(fromPartition).not.toHaveBeenCalled();
+  });
+});

apps/desktop/src/ipc/methods/preview.ts

@@ -0,0 +1,154 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as Data from "effect/Data";
+import * as Effect from "effect/Effect";
+import { BrowserWindow } from "electron";
+import { pathToFileURL } from "node:url";
+
+import { previewViewManager } from "../../preview-view-manager.ts";
+import { PREVIEW_WEBVIEW_PREFERENCES } from "../../preview-webview-preferences.ts";
+import * as IpcChannels from "../channels.ts";
+import type { DesktopIpcMethod } from "../DesktopIpc.ts";
+
+previewViewManager.onStateChange((tabId, state) => {
+  for (const window of BrowserWindow.getAllWindows()) {
+    if (!window.isDestroyed()) {
+      window.webContents.send(IpcChannels.PREVIEW_STATE_CHANGE_CHANNEL, tabId, state);
+    }
+  }
+});
+
+const tabIdFrom = (raw: unknown): string => {
+  if (typeof raw !== "object" || raw === null || !("tabId" in raw)) {
+    throw new Error("preview tab id is required");
+  }
+  const tabId = raw.tabId;
+  if (typeof tabId !== "string" || tabId.trim().length === 0) {
+    throw new Error("preview tab id must be a non-empty string");
+  }
+  return tabId;
+};
+
+const inputFrom = (raw: unknown): unknown => {
+  if (typeof raw !== "object" || raw === null || !("input" in raw)) {
+    throw new Error("preview automation input is required");
+  }
+  return raw.input;
+};
+
+class PreviewIpcError extends Data.TaggedError("PreviewIpcError")<{
+  readonly cause: unknown;
+}> {}
+
+const method = (
+  channel: string,
+  handler: (raw: unknown) => unknown | Promise<unknown>,
+): DesktopIpcMethod<PreviewIpcError, never> => ({
+  channel,
+  handler: (raw) =>
+    Effect.tryPromise({
+      try: () => Promise.resolve(handler(raw)),
+      catch: (cause) => new PreviewIpcError({ cause }),
+    }),
+});
+
+export const previewMethods = [
+  method(IpcChannels.PREVIEW_CREATE_TAB_CHANNEL, (raw) =>
+    previewViewManager.createTab(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_CLOSE_TAB_CHANNEL, (raw) =>
+    previewViewManager.closeTab(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_REGISTER_WEBVIEW_CHANNEL, (raw) => {
+    const tabId = tabIdFrom(raw);
+    const webContentsId =
+      typeof raw === "object" && raw !== null && "webContentsId" in raw ? raw.webContentsId : null;
+    if (
+      typeof webContentsId !== "number" ||
+      !Number.isInteger(webContentsId) ||
+      webContentsId <= 0
+    ) {
+      throw new Error("preview webContentsId must be a positive integer");
+    }
+    return previewViewManager.registerWebview(tabId, webContentsId);
+  }),
+  method(IpcChannels.PREVIEW_NAVIGATE_CHANNEL, (raw) => {
+    const tabId = tabIdFrom(raw);
+    const url = typeof raw === "object" && raw !== null && "url" in raw ? raw.url : null;
+    if (typeof url !== "string") throw new Error("preview url must be a string");
+    return previewViewManager.navigate(tabId, url);
+  }),
+  method(IpcChannels.PREVIEW_GO_BACK_CHANNEL, (raw) => previewViewManager.goBack(tabIdFrom(raw))),
+  method(IpcChannels.PREVIEW_GO_FORWARD_CHANNEL, (raw) =>
+    previewViewManager.goForward(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_REFRESH_CHANNEL, (raw) => previewViewManager.refresh(tabIdFrom(raw))),
+  method(IpcChannels.PREVIEW_ZOOM_IN_CHANNEL, (raw) => previewViewManager.zoomIn(tabIdFrom(raw))),
+  method(IpcChannels.PREVIEW_ZOOM_OUT_CHANNEL, (raw) => previewViewManager.zoomOut(tabIdFrom(raw))),
+  method(IpcChannels.PREVIEW_RESET_ZOOM_CHANNEL, (raw) =>
+    previewViewManager.resetZoom(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_HARD_RELOAD_CHANNEL, (raw) =>
+    previewViewManager.hardReload(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_OPEN_DEVTOOLS_CHANNEL, (raw) =>
+    previewViewManager.openDevTools(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_CLEAR_COOKIES_CHANNEL, () => previewViewManager.clearCookies()),
+  method(IpcChannels.PREVIEW_CLEAR_CACHE_CHANNEL, () => previewViewManager.clearCache()),
+  method(IpcChannels.PREVIEW_GET_CONFIG_CHANNEL, () => {
+    const preloadPath = `${__dirname}/preview-pick-preload.cjs`;
+    return {
+      partition: previewViewManager.getBrowserPartition(),
+      webPreferences: PREVIEW_WEBVIEW_PREFERENCES,
+      preloadUrl: pathToFileURL(preloadPath).href,
+    };
+  }),
+  method(IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, (raw) =>
+    previewViewManager.pickElement(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL, (raw) =>
+    previewViewManager.cancelPickElement(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, (raw) =>
+    previewViewManager.automationStatus(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, (raw) =>
+    previewViewManager.automationSnapshot(tabIdFrom(raw)),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, (raw) =>
+    previewViewManager.automationClick(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationClick>[1],
+    ),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_TYPE_CHANNEL, (raw) =>
+    previewViewManager.automationType(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationType>[1],
+    ),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_PRESS_CHANNEL, (raw) =>
+    previewViewManager.automationPress(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationPress>[1],
+    ),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_SCROLL_CHANNEL, (raw) =>
+    previewViewManager.automationScroll(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationScroll>[1],
+    ),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_EVALUATE_CHANNEL, (raw) =>
+    previewViewManager.automationEvaluate(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationEvaluate>[1],
+    ),
+  ),
+  method(IpcChannels.PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL, (raw) =>
+    previewViewManager.automationWaitFor(
+      tabIdFrom(raw),
+      inputFrom(raw) as Parameters<typeof previewViewManager.automationWaitFor>[1],
+    ),
+  ),
+] as const;