[codex] Add integrated browser preview, annotations, and agent automation

apps/desktop/package.json

@@ -20,13 +20,16 @@
     "@t3tools/tailscale": "workspace:*",
     "effect": "catalog:",
     "electron": "41.5.0",
-    "electron-updater": "^6.6.2"
+    "electron-updater": "^6.6.2",
+    "playwright-core": "1.60.0",
+    "react-grab": "^0.1.32"
   },
   "devDependencies": {
     "@effect/vitest": "catalog:",
     "@types/node": "catalog:",
     "cross-env": "^10.1.0",
     "electron-builder": "26.8.1",
+    "tailwindcss": "^4.0.0",
     "vite-plus": "catalog:"
   },
   "productName": "T3 Code (Alpha)"

apps/desktop/scripts/build-preview-annotation-css.mjs

@@ -0,0 +1,40 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { compile } from "tailwindcss";
+
+const directory = dirname(fileURLToPath(import.meta.url));
+const appRoot = join(directory, "..");
+const sourcePath = join(appRoot, "src", "preview", "Annotation.css");
+const preloadPath = join(appRoot, "src", "preview", "PickPreload.ts");
+const outputPath = join(appRoot, "src", "preview", "AnnotationStyles.generated.ts");
+const require = createRequire(import.meta.url);
+const tailwindRoot = dirname(require.resolve("tailwindcss/package.json"));
+
+const [annotationSource, preloadSource, themeSource, preflightSource] = await Promise.all([
+  readFile(sourcePath, "utf8"),
+  readFile(preloadPath, "utf8"),
+  readFile(join(tailwindRoot, "theme.css"), "utf8"),
+  readFile(join(tailwindRoot, "preflight.css"), "utf8"),
+]);
+
+const candidates = new Set(
+  Array.from(preloadSource.matchAll(/!?-?[A-Za-z0-9_:@/.[\]()%,-]+/g), (match) => match[0]),
+);
+const compilerInput = [
+  themeSource,
+  preflightSource,
+  annotationSource.replace('@import "tailwindcss";', "@tailwind utilities;"),
+].join("\n");
+const compiler = await compile(compilerInput, { base: appRoot });
+const css = compiler.build([...candidates]);
+const encodedCss = `'${css
+  .replaceAll("\\", "\\\\")
+  .replaceAll("'", "\\'")
+  .replaceAll("\r", "\\r")
+  .replaceAll("\n", "\\n")}'`;
+const moduleSource = `// Generated by scripts/build-preview-annotation-css.mjs. Do not edit.\nexport const previewAnnotationStyles =\n  ${encodedCss};\n`;
+
+await writeFile(outputPath, moduleSource);

apps/desktop/scripts/dev-electron.mjs

@@ -2,7 +2,11 @@ import { spawn, spawnSync } from "node:child_process";
 import { watch } from "node:fs";
 import { join } from "node:path";
 
-import { desktopDir, resolveDevProtocolClient, resolveElectronPath } from "./electron-launcher.mjs";
+import {
+  desktopDir,
+  resolveDevProtocolClient,
+  resolveElectronLaunchCommand,
+} from "./electron-launcher.mjs";
 import { waitForResources } from "./wait-for-resources.mjs";
 
 const devServerUrl = process.env.VITE_DEV_SERVER_URL?.trim();
@@ -79,7 +83,8 @@ function startApp() {
   const launchArgs = devProtocolClient
     ? electronArgs
     : [...electronArgs, `--t3code-dev-root=${desktopDir}`, "dist-electron/main.cjs"];
-  const app = spawn(resolveElectronPath(), launchArgs, {
+  const electronCommand = resolveElectronLaunchCommand(launchArgs);
+  const app = spawn(electronCommand.electronPath, electronCommand.args, {
     cwd: desktopDir,
     env: childEnv,
     stdio: "inherit",

apps/desktop/scripts/electron-launcher.mjs

@@ -307,6 +307,31 @@ function buildMacLauncher(electronBinaryPath) {
   return targetBinaryPath;
 }
 
+function isLinuxSetuidSandboxConfigured(electronBinaryPath) {
+  if (process.platform !== "linux") {
+    return true;
+  }
+
+  const sandboxPath = join(dirname(electronBinaryPath), "chrome-sandbox");
+  try {
+    const sandboxStat = statSync(sandboxPath);
+    return sandboxStat.uid === 0 && (sandboxStat.mode & 0o4777) === 0o4755;
+  } catch {
+    return false;
+  }
+}
+
+function resolveLinuxSandboxArgs(electronBinaryPath) {
+  if (isLinuxSetuidSandboxConfigured(electronBinaryPath)) {
+    return [];
+  }
+
+  console.warn(
+    "[desktop-launcher] Electron chrome-sandbox is not root-owned with mode 4755; launching local Electron with --no-sandbox.",
+  );
+  return ["--no-sandbox"];
+}
+
 export function resolveElectronPath() {
   ensureElectronRuntime();
 
@@ -320,6 +345,14 @@ export function resolveElectronPath() {
   return buildMacLauncher(electronBinaryPath);
 }
 
+export function resolveElectronLaunchCommand(args = []) {
+  const electronPath = resolveElectronPath();
+  return {
+    electronPath,
+    args: [...resolveLinuxSandboxArgs(electronPath), ...args],
+  };
+}
+
 export function resolveDevProtocolClient() {
   if (process.platform !== "darwin" || !isDevelopment) {
     return null;

apps/desktop/scripts/smoke-test.mjs

@@ -1,15 +1,16 @@
 import { spawn } from "node:child_process";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { resolveElectronLaunchCommand } from "./electron-launcher.mjs";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const desktopDir = resolve(__dirname, "..");
-const electronBin = resolve(desktopDir, "node_modules/.bin/electron");
 const mainJs = resolve(desktopDir, "dist-electron/main.cjs");
 
 console.log("\nLaunching Electron smoke test...");
 
-const child = spawn(electronBin, [mainJs], {
+const electronCommand = resolveElectronLaunchCommand([mainJs]);
+const child = spawn(electronCommand.electronPath, electronCommand.args, {
   stdio: ["pipe", "pipe", "pipe"],
   env: {
     ...process.env,

apps/desktop/scripts/start-electron.mjs

@@ -1,11 +1,12 @@
 import { spawn } from "node:child_process";
 
-import { desktopDir, resolveElectronPath } from "./electron-launcher.mjs";
+import { desktopDir, resolveElectronLaunchCommand } from "./electron-launcher.mjs";
 
 const childEnv = { ...process.env };
 delete childEnv.ELECTRON_RUN_AS_NODE;
 
-const child = spawn(resolveElectronPath(), ["dist-electron/main.cjs"], {
+const electronCommand = resolveElectronLaunchCommand(["dist-electron/main.cjs"]);
+const child = spawn(electronCommand.electronPath, electronCommand.args, {
   stdio: "inherit",
   cwd: desktopDir,
   env: childEnv,

apps/desktop/src/app/DesktopApp.ts

@@ -176,7 +176,7 @@ const bootstrap = Effect.gen(function* () {
     );
   }
 
-  yield* installDesktopIpcHandlers;
+  yield* installDesktopIpcHandlers();
   yield* logBootstrapInfo("bootstrap ipc handlers registered");
 
   if (!(yield* Ref.get(state.quitting))) {

apps/desktop/src/app/DesktopEnvironment.test.ts

@@ -59,6 +59,7 @@ describe("DesktopEnvironment", () => {
       assert.equal(environment.savedEnvironmentRegistryPath, "/tmp/t3/dev/saved-environments.json");
       assert.equal(environment.serverSettingsPath, "/tmp/t3/dev/settings.json");
       assert.equal(environment.logDir, "/tmp/t3/dev/logs");
+      assert.equal(environment.browserArtifactsDir, "/tmp/t3/dev/browser-artifacts");
       assert.equal(environment.rootDir, "/repo");
       assert.equal(environment.appRoot, "/repo");
       assert.equal(environment.backendEntryPath, "/repo/apps/server/dist/bin.mjs");
@@ -89,6 +90,7 @@ describe("DesktopEnvironment", () => {
       assert.equal(environment.isDevelopment, false);
       assert.equal(environment.stateDir, "/tmp/t3/userdata");
       assert.equal(environment.logDir, "/tmp/t3/userdata/logs");
+      assert.equal(environment.browserArtifactsDir, "/tmp/t3/userdata/browser-artifacts");
       assert.equal(environment.serverSettingsPath, "/tmp/t3/userdata/settings.json");
     }),
   );

apps/desktop/src/app/DesktopEnvironment.ts

@@ -49,6 +49,7 @@ export interface DesktopEnvironmentShape {
   readonly savedEnvironmentRegistryPath: string;
   readonly serverSettingsPath: string;
   readonly logDir: string;
+  readonly browserArtifactsDir: string;
   readonly rootDir: string;
   readonly appRoot: string;
   readonly backendEntryPath: string;
@@ -183,6 +184,7 @@ const makeDesktopEnvironment = Effect.fn("desktop.environment.make")(function* (
     savedEnvironmentRegistryPath: path.join(stateDir, "saved-environments.json"),
     serverSettingsPath: path.join(stateDir, "settings.json"),
     logDir: path.join(stateDir, "logs"),
+    browserArtifactsDir: path.join(stateDir, "browser-artifacts"),
     rootDir,
     appRoot,
     backendEntryPath: path.join(appRoot, "apps/server/dist/bin.mjs"),

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -48,9 +48,11 @@ import {
   setTheme,
   showContextMenu,
 } from "./methods/window.ts";
+import * as PreviewIpc from "./methods/preview.ts";
 
-export const installDesktopIpcHandlers = Effect.gen(function* () {
+export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers")(function* () {
   const ipc = yield* DesktopIpc.DesktopIpc;
+  yield* PreviewIpc.installPreviewEventForwarding();
 
   yield* ipc.handleSync(getAppBranding);
   yield* ipc.handleSync(getLocalEnvironmentBootstrap);
@@ -92,4 +94,7 @@ export const installDesktopIpcHandlers = Effect.gen(function* () {
   yield* ipc.handle(downloadUpdate);
   yield* ipc.handle(installUpdate);
   yield* ipc.handle(checkForUpdate);
-}).pipe(Effect.withSpan("desktop.ipc.installHandlers"));
+  for (const previewMethod of PreviewIpc.methods) {
+    yield* ipc.handle(previewMethod);
+  }
+});

apps/desktop/src/ipc/channels.ts

@@ -39,3 +39,38 @@ export const SET_SERVER_EXPOSURE_MODE_CHANNEL = "desktop:set-server-exposure-mod
 export const SET_TAILSCALE_SERVE_ENABLED_CHANNEL = "desktop:set-tailscale-serve-enabled";
 export const GET_ADVERTISED_ENDPOINTS_CHANNEL = "desktop:get-advertised-endpoints";
 export const SSH_PASSWORD_PROMPT_CANCELLED_RESULT = "ssh-password-prompt-cancelled";
+export const PREVIEW_CREATE_TAB_CHANNEL = "desktop:preview-create-tab";
+export const PREVIEW_CLOSE_TAB_CHANNEL = "desktop:preview-close-tab";
+export const PREVIEW_REGISTER_WEBVIEW_CHANNEL = "desktop:preview-register-webview";
+export const PREVIEW_NAVIGATE_CHANNEL = "desktop:preview-navigate";
+export const PREVIEW_GO_BACK_CHANNEL = "desktop:preview-go-back";
+export const PREVIEW_GO_FORWARD_CHANNEL = "desktop:preview-go-forward";
+export const PREVIEW_REFRESH_CHANNEL = "desktop:preview-refresh";
+export const PREVIEW_ZOOM_IN_CHANNEL = "desktop:preview-zoom-in";
+export const PREVIEW_ZOOM_OUT_CHANNEL = "desktop:preview-zoom-out";
+export const PREVIEW_RESET_ZOOM_CHANNEL = "desktop:preview-reset-zoom";
+export const PREVIEW_HARD_RELOAD_CHANNEL = "desktop:preview-hard-reload";
+export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
+export const PREVIEW_CLEAR_COOKIES_CHANNEL = "desktop:preview-clear-cookies";
+export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";
+export const PREVIEW_GET_CONFIG_CHANNEL = "desktop:preview-get-config";
+export const PREVIEW_SET_ANNOTATION_THEME_CHANNEL = "desktop:preview-set-annotation-theme";
+export const PREVIEW_PICK_ELEMENT_CHANNEL = "desktop:preview-pick-element";
+export const PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL = "desktop:preview-cancel-pick-element";
+export const PREVIEW_CAPTURE_SCREENSHOT_CHANNEL = "desktop:preview-capture-screenshot";
+export const PREVIEW_REVEAL_ARTIFACT_CHANNEL = "desktop:preview-reveal-artifact";
+export const PREVIEW_COPY_ARTIFACT_CHANNEL = "desktop:preview-copy-artifact";
+export const PREVIEW_AUTOMATION_STATUS_CHANNEL = "desktop:preview-automation-status";
+export const PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL = "desktop:preview-automation-snapshot";
+export const PREVIEW_AUTOMATION_CLICK_CHANNEL = "desktop:preview-automation-click";
+export const PREVIEW_AUTOMATION_TYPE_CHANNEL = "desktop:preview-automation-type";
+export const PREVIEW_AUTOMATION_PRESS_CHANNEL = "desktop:preview-automation-press";
+export const PREVIEW_AUTOMATION_SCROLL_CHANNEL = "desktop:preview-automation-scroll";
+export const PREVIEW_AUTOMATION_EVALUATE_CHANNEL = "desktop:preview-automation-evaluate";
+export const PREVIEW_AUTOMATION_WAIT_FOR_CHANNEL = "desktop:preview-automation-wait-for";
+export const PREVIEW_RECORDING_START_CHANNEL = "desktop:preview-recording-start";
+export const PREVIEW_RECORDING_STOP_CHANNEL = "desktop:preview-recording-stop";
+export const PREVIEW_RECORDING_SAVE_CHANNEL = "desktop:preview-recording-save";
+export const PREVIEW_RECORDING_FRAME_CHANNEL = "desktop:preview-recording-frame";
+export const PREVIEW_STATE_CHANGE_CHANNEL = "desktop:preview-state-change";
+export const PREVIEW_POINTER_EVENT_CHANNEL = "desktop:preview-pointer-event";

apps/desktop/src/ipc/methods/preview.test.ts

@@ -0,0 +1,54 @@
+import { it as effectIt } from "@effect/vitest";
+import * as Cause from "effect/Cause";
+import * as Effect from "effect/Effect";
+import * as Exit from "effect/Exit";
+import * as Option from "effect/Option";
+import * as Schema from "effect/Schema";
+import { beforeEach, describe, expect, it, vi } from "vite-plus/test";
+
+import * as PreviewManager from "../../preview/Manager.ts";
+import * as PreviewIpc from "./preview.ts";
+
+const { fromPartition } = vi.hoisted(() => ({
+  fromPartition: vi.fn(() => {
+    throw new Error("Session can only be received when app is ready");
+  }),
+}));
+
+vi.mock("electron", () => ({
+  BrowserWindow: {
+    getAllWindows: vi.fn(() => []),
+  },
+  session: {
+    fromPartition,
+  },
+  webContents: {
+    fromId: vi.fn(() => null),
+  },
+}));
+
+describe("preview IPC methods", () => {
+  beforeEach(() => {
+    fromPartition.mockClear();
+  });
+
+  it("does not access the Electron session while the module loads", async () => {
+    await expect(import("./preview.ts")).resolves.toBeDefined();
+    expect(fromPartition).not.toHaveBeenCalled();
+  });
+
+  effectIt.effect("rejects invalid webContents ids before resolving the preview service", () =>
+    Effect.map(
+      PreviewIpc.registerWebview
+        .handler({ tabId: "tab-1", webContentsId: 0 })
+        .pipe(Effect.provideService(PreviewManager.PreviewManager, null as never), Effect.exit),
+      (exit) => {
+        expect(Exit.isFailure(exit)).toBe(true);
+        if (Exit.isSuccess(exit)) return;
+        const error = Cause.findErrorOption(exit.cause);
+        expect(Option.isSome(error) && Schema.isSchemaError(error.value)).toBe(true);
+        expect(fromPartition).not.toHaveBeenCalled();
+      },
+    ),
+  );
+});