diff --git a/javascript/examples/openai-realtime-demo/package.json b/javascript/examples/openai-realtime-demo/package.json index 2759d0d60..12f70184d 100644 --- a/javascript/examples/openai-realtime-demo/package.json +++ b/javascript/examples/openai-realtime-demo/package.json @@ -11,7 +11,8 @@ "format": "pnpm lint --fix" }, "dependencies": { - "@openai/agents": "^0.16.0" + "@openai/agents": "^0.16.0", + "zod": "^4.1.13" }, "devDependencies": { "@types/node": "^26.2.0", diff --git a/javascript/pnpm-lock.yaml b/javascript/pnpm-lock.yaml index b3cf79bf7..5a0ccc278 100644 --- a/javascript/pnpm-lock.yaml +++ b/javascript/pnpm-lock.yaml @@ -196,7 +196,10 @@ importers: dependencies: '@openai/agents': specifier: ^0.16.0 - version: 0.16.0(ws@8.21.3)(zod@3.25.76) + version: 0.16.0(ws@8.21.3)(zod@4.4.3) + zod: + specifier: ^4.1.13 + version: 4.4.3 devDependencies: '@types/node': specifier: ^26.2.0 @@ -1668,42 +1671,36 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.2.4': resolution: {integrity: sha512-Ql1Q0EQqVThvn9VAVlwNzsUvbSFtCMGjLpRRi4pk5i7NZZ4n5ISiLMjHYtus4VQ2PvkSw24zyaCVsiS+sXPj1w==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] - libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.2.4': resolution: {integrity: sha512-GjbjXD4XXfN19D0LZNbmiCBUoDiRACsYHr0yaIbbn8aFsXjHZifcYqu/W5Er5X2X990WjHXFrxarn5chzItorQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.2.4': resolution: {integrity: sha512-p5WR0NOwaRmJ/B1b6IjEFLLivwEsf3PrdBIhRbhTCQisbo2SvHHpG4ELB/+FgQNnB88LTOF86upmJmbvZdQ2lw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.2.4': resolution: {integrity: sha512-4/GyVjmhR+Tc6HLJvwc1sOhPqAZtySiSMesOZyX6JQ5XBxoTDEMKQzvo07NIK6nTon/SivlZqvhzvuVBNQhObQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [glibc] '@rolldown/binding-linux-x64-musl@1.2.4': resolution: {integrity: sha512-l9eeLsCNvPpmSXUej0etw/J1eqV0Jj1D5G/xG6YTijmE6dkv6E2QezgWbTfQk63v952DPqrjOCoiqxq7Bw0YUQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] - libc: [musl] '@rolldown/binding-openharmony-arm64@1.2.4': resolution: {integrity: sha512-e0F355MSTMm3+UOqtV3L24gFUp2N5m1f8L/7d56deik6va+AXdrt9F8LbzGpeWGWRbZEDq4m8NVnJDeBtf9DZg==} @@ -1760,79 +1757,66 @@ packages: resolution: {integrity: sha512-EIPRXTVQpHyF8WOo219AD2yEltPehLTcTMz2fn6JsatLYSzQf00hj3rulF+yauOlF9/FtM2WpkT/hJh/KJFGhA==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.60.4': resolution: {integrity: sha512-J3Yh9PzzF1Ovah2At+lHiGQdsYgArxBbXv/zHfSyaiFQEqvNv7DcW98pCrmdjCZBrqBiKrKKe2V+aaSGWuBe/w==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.60.4': resolution: {integrity: sha512-BFDEZMYfUvLn37ONE1yMBojPxnMlTFsdyNoqncT0qFq1mAfllL+ATMMJd8TeuVMiX84s1KbcxcZbXInmcO2mRg==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.60.4': resolution: {integrity: sha512-pc9EYOSlOgdQ2uPl1o9PF6/kLSgaUosia7gOuS8mB69IxJvlclko1MECXysjs5ryez1/5zjYqx3+xYU0TU6R1A==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.60.4': resolution: {integrity: sha512-NxnomyxYerDh5n4iLrNa+sH+Z+U4BMEE46V2PgQ/hoB909i8gV1M5wPojWg9fk1jWpO3IQnOs20K4wyZuFLEFQ==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.60.4': resolution: {integrity: sha512-nbJnQ8a3z1mtmrwImCYhc6BGpThAyYVRQxw9uKSKG4wR6aAYno9sVjJ0zaZcW9BPJX1GbrDPf+SvdWjgTuDmnw==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.60.4': resolution: {integrity: sha512-2EU6acNrQLd8tYvo/LXW535wupT3m6fo7HKo6lr7ktQoItxTyOL1ZCR/GfGCuXl2vR+zmfI6eRXkSemafv+iVg==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.60.4': resolution: {integrity: sha512-WeBtoMuaMxiiIrO2IYP3xs6GMWkJP2C0EoT8beTLkUPmzV1i/UcOSVw1d5r9KBODtHKilG5yFxsGRnBbK3wJ4A==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.60.4': resolution: {integrity: sha512-FJHFfqpKUI3A10WrWKiFbBZ7yVbGT4q4B5o1qKFFojqpaYoh9LrQgqWCmmcxQzVSXYtyB5bzkXrYzlHTs21MYA==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.60.4': resolution: {integrity: sha512-mcEl6CUT5IAUmQf1m9FYSmVqCJlpQ8r8eyftFUHG8i9OhY7BkBXSUdnLH5DOf0wCOjcP9v/QO93zpmF1SptCCw==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.60.4': resolution: {integrity: sha512-ynt3JxVd2w2buzoKDWIyiV1pJW93xlQic1THVLXilz429oijRpSHivZAgp65KBu+cMcgf1eVVjdnTLvPxgCuoQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.60.4': resolution: {integrity: sha512-Boiz5+MsaROEWDf+GGEwF8VMHGhlUoQMtIPjOgA5fv4osupqTVnJteQNKJwUcnUog2G55jYXH7KZFFiJe0TEzQ==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.60.4': resolution: {integrity: sha512-+qfSY27qIrFfI/Hom04KYFw3GKZSGU4lXus51wsb5EuySfFlWRwjkKWoE9emgRw/ukoT4Udsj4W/+xxG8VbPKg==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.60.4': resolution: {integrity: sha512-VpTfOPHgVXEBeeR8hZ2O0F3aSso+JDWqTWmTmzcQKted54IAdUVbxE+j/MVxUsKa8L20HJhv3vUezVPoquqWjA==} @@ -1917,28 +1901,24 @@ packages: engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-arm64-musl@4.3.3': resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] - libc: [musl] '@tailwindcss/oxide-linux-x64-gnu@4.3.3': resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [glibc] '@tailwindcss/oxide-linux-x64-musl@4.3.3': resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==} engines: {node: '>= 20'} cpu: [x64] os: [linux] - libc: [musl] '@tailwindcss/oxide-wasm32-wasi@4.3.3': resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==} @@ -2221,61 +2201,51 @@ packages: resolution: {integrity: sha512-zJc0H99FEPoFfSrNpa91HYfxzfAJCr502oxNK1cfdC9hlaFI43RT+JFCann9JUgZmLzzntChHyn13Sgn9ljHNg==} cpu: [arm64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.12.2': resolution: {integrity: sha512-KQ3Lki6l+Pz1k/eBipN41ES+YUK30beLGb9YqcB1O542cyLCNE6GaxrfcY3T6EezmGGk84wb5XyO9loTM9tkcA==} cpu: [arm64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-loong64-gnu@1.12.2': resolution: {integrity: sha512-3SJGEh1DborhG6pyxvhPzCT4bbSIVihsvgJc13P1bHG7KLdNDaF9T3gsTwFc7Jw/5Y5/iWOjkEx7Zy0NvCGX3Q==} cpu: [loong64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-loong64-musl@1.12.2': resolution: {integrity: sha512-jiuG/Obbel7uw1PwHNFfrkiKhLAF6mnyZ6aWlOAVN9WqKm8v0OFGnciJIHu8+CMvXLQ8AD51LPzAoUfT21D5Ew==} cpu: [loong64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.12.2': resolution: {integrity: sha512-q7xRvVpmcfeL+LlZg8Pbbo6QaTZwDU5BaGZbwfhkEsXJn3Was8xYfE0RBH266xZt0rM6B7i8xAYIvjthuUIWHg==} cpu: [ppc64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.12.2': resolution: {integrity: sha512-0CVdx6lcnT3Q9inOH8tsMIOJ6ImndllMjqJHg8RLVdB7Vq4SfkEXl9mCSsVNuNA4MCYycRicCUxPCabVHJRr6A==} cpu: [riscv64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.12.2': resolution: {integrity: sha512-iOwlRo9vnp6R6ohHQS11n0NnfdXx/omhkocmIfaPRpQhKZ+3BDMkkdRVh53qjkFkpPddf+FETA28NwGN7l5l+w==} cpu: [riscv64] os: [linux] - libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.12.2': resolution: {integrity: sha512-HYJtLfXq94q8iZNFT1lknx258wlkkWhZeUXJRqzKBBUJ00CvZ+N33zgbCqimLjsyw5Va6uUxhVa12mI+kaveEw==} cpu: [s390x] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.12.2': resolution: {integrity: sha512-mPsUhunKKDih5O96Y6enDQyHc1SqBPlY1E/SfMWDM3EdJ95Z9CArPeCVwCCqbP45ljvivdEk8Fxn+SIb1rDAJQ==} cpu: [x64] os: [linux] - libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.12.2': resolution: {integrity: sha512-azrt6+5ydLd8Vt210AAFis/lZevSfPw93EJRIJG+xPu4WCJ8K0kppCTpMyLPcKT7H15M4Jnt2tMp5bOvCkRC6A==} cpu: [x64] os: [linux] - libc: [musl] '@unrs/resolver-binding-openharmony-arm64@1.12.2': resolution: {integrity: sha512-YZ9hP4O0X9PQb8eO980qmLNGH4zT3I9+SZTdt0Pr0YyuGQhYKoOZkV02VzrzyOZJ5xIJ3UFIenKkUkGg8GjgWQ==} @@ -4068,56 +4038,48 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-gnu@1.33.0: resolution: {integrity: sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-arm64-musl@1.33.0: resolution: {integrity: sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] - libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-gnu@1.33.0: resolution: {integrity: sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-linux-x64-musl@1.33.0: resolution: {integrity: sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] - libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -5363,7 +5325,7 @@ packages: '@vitest/ui': 4.1.10 happy-dom: '*' jsdom: '*' - vite: '>=7.3.2' + vite: '>=8.0.16' peerDependenciesMeta: '@edge-runtime/vm': optional: true diff --git a/javascript/src/voice/__tests__/elevenlabs-sdk.test.ts b/javascript/src/voice/__tests__/elevenlabs-sdk.test.ts new file mode 100644 index 000000000..e2841fdd5 --- /dev/null +++ b/javascript/src/voice/__tests__/elevenlabs-sdk.test.ts @@ -0,0 +1,76 @@ +/** + * The ElevenLabs SDK must stay out of the module graph until voice runs. + * + * A module-scope `import ... from "@elevenlabs/elevenlabs-js"` anywhere under + * `src/` puts it back into every consumer of the package, because `index.ts` + * re-exports the voice namespace. That cost 4,972 modules and 237MB of RSS on + * a plain `import "@langwatch/scenario"`, paid by text-only scenarios too. + * + * This is a source scan rather than a runtime check because the regression is + * a static import, and a static import is exactly what a runtime probe of an + * already-bundled build can no longer distinguish. + */ +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +/** javascript/src, from src/voice/__tests__/. */ +const SRC = resolve(__dirname, "../.."); + +/** The seam is the one module allowed to name the SDK, and only in a body. */ +const SEAM = "voice/elevenlabs-sdk.ts"; + +function sourceFiles(dir: string): string[] { + return readdirSync(dir).flatMap((entry) => { + const path = join(dir, entry); + if (statSync(path).isDirectory()) { + return entry === "__tests__" ? [] : sourceFiles(path); + } + return path.endsWith(".ts") && !path.endsWith(".d.ts") ? [path] : []; + }); +} + +/** + * Every specifier reached by a static import, in all the forms that load a + * module: named, type-only, re-export, and bare side-effect. + * + * A named import may wrap over several lines, so the `from` pattern has to + * cross newlines. What stops it running into a function body and finding the + * seam's own `import(...)` is excluding `;`, `(` and `)`: every dynamic import + * sits inside a call, and no static import declaration contains one. + * + * `matchAll` rather than `.test()`, because a `g`-flagged regex carries + * `lastIndex` between calls and would skip every second offending file. + */ +const IMPORT_FROM = /^[ \t]*(?:import|export)\s[^;()]*?\bfrom\s*["']([^"']+)["']/gm; +const IMPORT_BARE = /^[ \t]*import\s+["']([^"']+)["']/gm; + +function staticallyImportsSdk(text: string): boolean { + for (const pattern of [IMPORT_FROM, IMPORT_BARE]) { + for (const [, specifier] of text.matchAll(pattern)) { + if (specifier?.startsWith("@elevenlabs/")) return true; + } + } + return false; +} + +describe("given the package is imported without running voice", () => { + describe("when a consumer imports the entry point", () => { + it("loads no ElevenLabs module, because nothing imports the SDK statically", () => { + const offenders = sourceFiles(SRC) + .map((path) => ({ path, text: readFileSync(path, "utf8") })) + .filter(({ text }) => staticallyImportsSdk(text)) + .map(({ path }) => relative(SRC, path)); + + expect( + offenders, + `these modules import the ElevenLabs SDK at module scope, which pulls ~5k modules into every consumer. Route the call through ${SEAM} instead`, + ).toEqual([]); + }); + + it("keeps the SDK reachable through the seam, so voice still works", () => { + const seam = readFileSync(resolve(SRC, SEAM), "utf8"); + expect(seam).toMatch(/await import\(\s*\n?\s*["']@elevenlabs\//); + }); + }); +}); diff --git a/javascript/src/voice/adapters/elevenlabs.ts b/javascript/src/voice/adapters/elevenlabs.ts index 836169c47..47b40b8fc 100644 --- a/javascript/src/voice/adapters/elevenlabs.ts +++ b/javascript/src/voice/adapters/elevenlabs.ts @@ -69,11 +69,6 @@ import { openai } from "@ai-sdk/openai"; // map, so Node resolves these deep paths as literal files. Extensionless they // only resolve under bundler semantics (vitest/tsx/webpack) and crash plain // `node` consumers of the published dist/index.mjs with ERR_MODULE_NOT_FOUND. -import { AudioInterface } from "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/AudioInterface.js"; -import { Conversation } from "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/Conversation.js"; -import type { ConversationClient } from "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/interfaces/ConversationClient"; -import type { WebSocketFactory } from "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/interfaces/WebSocketInterface"; -import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js/Client.js"; import type { LanguageModel } from "ai"; import { AgentRole } from "../../domain/agents"; @@ -81,6 +76,14 @@ import { Logger } from "../../utils/logger"; import { VoiceAgentAdapter } from "../adapter"; import { AudioChunk } from "../audio-chunk"; import { AdapterCapabilities } from "../capabilities"; +import { + type ElevenLabsAudioInterface, + type ElevenLabsAudioInterfaceCtor, + type ElevenLabsConversation, + type ElevenLabsConversationClient, + type ElevenLabsWebSocketFactory, + loadElevenLabsConversationRuntime, +} from "../elevenlabs-sdk"; import { currentSpan, setSpanAttributes } from "../telemetry"; import { COMPOSABLE_VOICE_LLM_MODEL, @@ -206,33 +209,41 @@ const SILENCE_FRAME = Buffer.alloc(PUMP_FRAME_BYTES); * members private — the closures are created inside the adapter and close over * `this`. */ -class BridgeAudioInterface extends AudioInterface { - constructor( - private readonly hooks: { - onStart: (inputCallback: (audio: Buffer) => void) => void; - onStop: () => void; - onOutput: (audio: Buffer) => void; - onInterrupt: () => void; - }, - ) { - super(); - } +interface BridgeAudioHooks { + onStart: (inputCallback: (audio: Buffer) => void) => void; + onStop: () => void; + onOutput: (audio: Buffer) => void; + onInterrupt: () => void; +} - override start(inputCallback: (audio: Buffer) => void): void { - this.hooks.onStart(inputCallback); - } +/** + * Built rather than declared, because the base class arrives with the SDK and + * the SDK is only loaded once a session actually starts. A module-scope + * `class ... extends AudioInterface` would need it at import time, which is + * what used to put 4,549 ElevenLabs modules into every consumer's graph. + */ +function createBridgeAudioInterface( + AudioInterfaceBase: ElevenLabsAudioInterfaceCtor, + hooks: BridgeAudioHooks, +): ElevenLabsAudioInterface { + class BridgeAudioInterface extends AudioInterfaceBase { + start(inputCallback: (audio: Buffer) => void): void { + hooks.onStart(inputCallback); + } - override stop(): void { - this.hooks.onStop(); - } + stop(): void { + hooks.onStop(); + } - override output(audio: Buffer): void { - this.hooks.onOutput(audio); - } + output(audio: Buffer): void { + hooks.onOutput(audio); + } - override interrupt(): void { - this.hooks.onInterrupt(); + interrupt(): void { + hooks.onInterrupt(); + } } + return new BridgeAudioInterface(); } /** A non-null, non-array object — the only value kind {@link deepMerge} recurses into. */ @@ -332,14 +343,14 @@ export interface ElevenLabsAgentAdapterOptions { * runs against an in-memory socket (no network). Production callers leave this * unset; the SDK's `DefaultWebSocketFactory` (the `ws` package) is used. */ - webSocketFactory?: WebSocketFactory; + webSocketFactory?: ElevenLabsWebSocketFactory; /** * SDK conversation client used ONLY for the `requiresAuth` signed-URL handshake * — injected for unit tests so `startSession()` does not make a real * `getSignedUrl` HTTP call. Production callers leave this unset; the adapter's - * authenticated {@link ElevenLabsClient} is used. + * own authenticated `ElevenLabsClient` is used. */ - conversationClient?: ConversationClient; + conversationClient?: ElevenLabsConversationClient; } /** @@ -366,11 +377,11 @@ export class ElevenLabsAgentAdapter extends VoiceAgentAdapter { private readonly firstMessageOverride?: string; private readonly dynamicVariables?: Record; private readonly overrides?: Record; - private readonly webSocketFactory?: WebSocketFactory; - private readonly conversationClient?: ConversationClient; + private readonly webSocketFactory?: ElevenLabsWebSocketFactory; + private readonly conversationClient?: ElevenLabsConversationClient; /** Live SDK session; null whenever disconnected (or before the first connect). */ - private conversation: Conversation | null = null; + private conversation: ElevenLabsConversation | null = null; /** * The SDK's mic-input sink, captured when the SDK calls `AudioInterface.start` * on session open. The continuous mic pump feeds it one frame per tick; the SDK @@ -474,6 +485,8 @@ export class ElevenLabsAgentAdapter extends VoiceAgentAdapter { setSpanAttributes(currentSpan(), { "voice.elevenlabs.agent_id": this.agentId, }); + const { AudioInterface, Conversation, ElevenLabsClient } = + await loadElevenLabsConversationRuntime(); const client = new ElevenLabsClient({ apiKey: this.apiKey }); // The adapter's NARROW prompt/first-message knobs build an `agent` override @@ -497,7 +510,7 @@ export class ElevenLabsAgentAdapter extends VoiceAgentAdapter { agent: agentOverride, }); - const audioInterface = new BridgeAudioInterface({ + const audioInterface = createBridgeAudioInterface(AudioInterface, { onStart: (inputCallback) => this.onAudioStart(inputCallback), onStop: () => this.onAudioStop(), onOutput: (audio) => this.onAgentAudio(audio), @@ -525,10 +538,10 @@ export class ElevenLabsAgentAdapter extends VoiceAgentAdapter { // and to `client` for getSignedUrl. Set only by unit tests. webSocketFactory: this.webSocketFactory, conversationClient: this.conversationClient, - callbackUserTranscript: (transcript) => { + callbackUserTranscript: (transcript: string) => { this.lastUserTranscript = transcript; }, - callbackAgentResponse: (response) => { + callbackAgentResponse: (response: string) => { // #734 (AC4) — measure how far this transcript event lags the audio it // describes. A lag exceeding `responseTailSilence` is a turn whose // transcript would have LOST the drain-close race pre-fix; the grace-wait @@ -544,13 +557,13 @@ export class ElevenLabsAgentAdapter extends VoiceAgentAdapter { }); this.lastAgentTranscript = response; }, - callbackAgentResponseCorrection: (_original, corrected) => { + callbackAgentResponseCorrection: (_original: string, corrected: string) => { // Post-barge-in correction replaces the agent transcript. this.lastAgentTranscript = corrected; }, // Fires for EVERY inbound message (ping included) AFTER the SDK has routed it // — our universal liveness + terminal-turn hook. See onMessage. - callbackMessageReceived: (message) => this.onMessage(message), + callbackMessageReceived: (message: unknown) => this.onMessage(message), }); // The SDK re-emits WS errors as an `error` event on the Conversation itself; an diff --git a/javascript/src/voice/elevenlabs-sdk.ts b/javascript/src/voice/elevenlabs-sdk.ts new file mode 100644 index 000000000..058ce3cd1 --- /dev/null +++ b/javascript/src/voice/elevenlabs-sdk.ts @@ -0,0 +1,154 @@ +/** + * The one place `@elevenlabs/elevenlabs-js` is named, and the only place it is + * loaded. + * + * The SDK is large: importing `@langwatch/scenario` used to pull 5,732 modules + * and 225MB of RSS, and 4,549 of those modules were ElevenLabs. Every consumer + * paid that, including a text-only scenario in a server that never does voice, + * because `index.ts` re-exports the voice namespace and the three ElevenLabs + * leaves imported the SDK at module scope. + * + * Two rules keep it out: + * + * 1. **Nothing here re-exports an SDK type.** The interfaces below are + * structural views of the parts we actually call, so the published + * `index.d.ts` never imports the SDK and a consumer's typecheck never loads + * its 2,605 declaration files. + * 2. **The SDK is imported inside a function body.** Declaration emit only + * describes exported signatures, so a dynamic import in a body reaches + * neither the types nor the module graph until someone runs voice. + * + * Adding a call to a new SDK method means widening the structural type here + * rather than importing the SDK at a call site. + */ + +/** The `speechToText` surface the STT leaf uses. */ +export interface ElevenLabsSpeechToText { + convert(request: { + file: Blob; + modelId: string; + }): Promise<{ text?: string }>; +} + +/** The `textToSpeech` surface the TTS leaf uses. */ +export interface ElevenLabsTextToSpeech { + convert( + voiceId: string, + request: { text: string; modelId: string; outputFormat: string }, + ): Promise>; +} + +/** + * The client as this SDK uses it. A real `ElevenLabsClient` satisfies it, so + * callers injecting one through a `clientFactory` need no change. + */ +export interface ElevenLabsClientLike { + speechToText: ElevenLabsSpeechToText; + textToSpeech: ElevenLabsTextToSpeech; +} + +/** Factory for the client. Injectable, so a test can supply a fake. */ +export type ElevenLabsClientFactory = (apiKey: string) => ElevenLabsClientLike; + +/** + * Opens the session socket. Forwarded to the SDK unchanged; described here + * structurally so a caller still gets checked without the SDK's own + * `WebSocketFactory` entering the published types. + */ +export interface ElevenLabsWebSocketFactory { + create(url: string, ...rest: never[]): unknown; +} + +/** + * Signs the session URL for the `requiresAuth` handshake. Structural for the + * same reason as {@link ElevenLabsWebSocketFactory}; a real SDK client + * satisfies it. + */ +export interface ElevenLabsConversationClient { + conversationalAi: { + conversations: { + getSignedUrl(request: { agentId: string }): Promise; + }; + }; +} + +/** The audio sink the adapter subclasses to bridge SDK audio to the run. */ +export interface ElevenLabsAudioInterface { + start(inputCallback: (audio: Buffer) => void): void; + stop(): void; + output(audio: Buffer): void; + interrupt(): void; +} + +/** Constructor for {@link ElevenLabsAudioInterface}, used as a base class. */ +export type ElevenLabsAudioInterfaceCtor = new () => ElevenLabsAudioInterface; + +/** The live session surface the agent adapter drives. */ +export interface ElevenLabsConversation { + startSession(): Promise; + endSession(): Promise; + isSessionActive(): boolean; + on(event: string, listener: (...args: never[]) => void): unknown; +} + +/** Names the missing package rather than letting a bare resolution error escape. */ +function missingSdk(cause: unknown): Error { + return new Error( + "The ElevenLabs voice backend needs '@elevenlabs/elevenlabs-js'. Install it to use elevenlabs voices, speech-to-text or the ElevenLabs agent adapter.", + { cause }, + ); +} + +/** Builds an authenticated client, loading the SDK on first use. */ +export async function loadElevenLabsClient( + apiKey: string, +): Promise { + let sdk: { ElevenLabsClient: new (o: { apiKey: string }) => unknown }; + try { + sdk = (await import("@elevenlabs/elevenlabs-js")) as unknown as typeof sdk; + } catch (cause) { + throw missingSdk(cause); + } + return new sdk.ElevenLabsClient({ apiKey }) as ElevenLabsClientLike; +} + +/** + * The conversational-AI runtime the agent adapter drives. + * + * Deliberately untyped at this boundary: the adapter subclasses `AudioInterface` + * and constructs `Conversation`, both of which would drag the SDK's types into + * the published declarations if they were named in an exported signature. The + * adapter keeps the loaded values inside its own module. + */ +interface ElevenLabsRuntimeModule { + AudioInterface: ElevenLabsAudioInterfaceCtor; + /** Its option bag is wide and SDK-shaped, so it stays a bare constructor. */ + Conversation: new (options: Record) => ElevenLabsConversation; + ElevenLabsClient: new (options: { apiKey: string }) => unknown; +} + +export async function loadElevenLabsConversationRuntime(): Promise { + try { + const [audio, conversation, client] = await Promise.all([ + import( + "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/AudioInterface.js" + ), + import( + "@elevenlabs/elevenlabs-js/api/resources/conversationalAi/conversation/Conversation.js" + ), + import("@elevenlabs/elevenlabs-js/Client.js"), + ]); + // The casts are the seam itself: the SDK's own declarations would come + // with the SDK, so each export is narrowed to the shape we drive it by. + return { + AudioInterface: (audio as unknown as ElevenLabsRuntimeModule) + .AudioInterface, + Conversation: (conversation as unknown as ElevenLabsRuntimeModule) + .Conversation, + ElevenLabsClient: (client as unknown as ElevenLabsRuntimeModule) + .ElevenLabsClient, + }; + } catch (cause) { + throw missingSdk(cause); + } +} diff --git a/javascript/src/voice/stt/elevenlabs-stt.ts b/javascript/src/voice/stt/elevenlabs-stt.ts index 65426bfac..2286ddaf9 100644 --- a/javascript/src/voice/stt/elevenlabs-stt.ts +++ b/javascript/src/voice/stt/elevenlabs-stt.ts @@ -12,9 +12,11 @@ * copy that used to live in `adapters/composable.ts` is gone; composable and * the branded preset import this leaf. */ -import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; - import { AudioChunk } from "../audio-chunk"; +import { + type ElevenLabsClientLike, + loadElevenLabsClient, +} from "../elevenlabs-sdk"; import { ELEVENLABS_STT_MODEL } from "../voice-models"; import type { STTProvider } from "./stt-provider"; import { pcm16ToWav } from "./wav"; @@ -28,7 +30,7 @@ export interface ElevenLabsSTTProviderOptions { /** API key; falls back to `process.env.ELEVENLABS_API_KEY`. */ apiKey?: string; /** Test seam — override the SDK client constructor. */ - clientFactory?: (apiKey: string) => ElevenLabsClient; + clientFactory?: (apiKey: string) => ElevenLabsClientLike; } /** @@ -36,12 +38,11 @@ export interface ElevenLabsSTTProviderOptions { */ export class ElevenLabsSTTProvider implements STTProvider { private readonly apiKey: string; - private readonly clientFactory: (apiKey: string) => ElevenLabsClient; + private readonly clientFactory?: (apiKey: string) => ElevenLabsClientLike; constructor(options: ElevenLabsSTTProviderOptions = {}) { this.apiKey = options.apiKey ?? process.env.ELEVENLABS_API_KEY ?? ""; - this.clientFactory = - options.clientFactory ?? ((apiKey) => new ElevenLabsClient({ apiKey })); + this.clientFactory = options.clientFactory; } toString(): string { @@ -49,7 +50,11 @@ export class ElevenLabsSTTProvider implements STTProvider { } async transcribe(audio: AudioChunk): Promise { - const client = this.clientFactory(this.apiKey); + // Loaded here rather than at module scope: the SDK is 4,549 modules and + // only a run that actually transcribes should pay for them. + const client = this.clientFactory + ? this.clientFactory(this.apiKey) + : await loadElevenLabsClient(this.apiKey); const wav = pcm16ToWav(audio.data); // The SDK accepts Blob/File/ReadStream. Node 20+ supplies Blob globally so // we don't need a polyfill. diff --git a/javascript/src/voice/tts/elevenlabs-tts.ts b/javascript/src/voice/tts/elevenlabs-tts.ts index 0683e8e9e..9b10639a6 100644 --- a/javascript/src/voice/tts/elevenlabs-tts.ts +++ b/javascript/src/voice/tts/elevenlabs-tts.ts @@ -14,13 +14,15 @@ */ import { Buffer } from "node:buffer"; -import { ElevenLabsClient } from "@elevenlabs/elevenlabs-js"; - +import { + type ElevenLabsClientLike, + loadElevenLabsClient, +} from "../elevenlabs-sdk"; import { ELEVENLABS_TTS_MODEL } from "../voice-models"; import type { TTSCallable } from "./tts"; /** Factory for the ElevenLabs SDK client — injectable for tests. */ -export type ElevenLabsClientFactory = (apiKey: string) => ElevenLabsClient; +export type ElevenLabsClientFactory = (apiKey: string) => ElevenLabsClientLike; /** Construction / per-call options for {@link ElevenLabsTtsProvider}. */ export interface ElevenLabsTtsOptions { @@ -30,9 +32,6 @@ export interface ElevenLabsTtsOptions { clientFactory?: ElevenLabsClientFactory; } -const defaultClientFactory: ElevenLabsClientFactory = (apiKey) => - new ElevenLabsClient({ apiKey }); - /** * Synthesize `text` to raw PCM16/24 kHz bytes via the ElevenLabs SDK. * @@ -45,8 +44,11 @@ export async function elevenLabsSynthesizeBytes( options: ElevenLabsTtsOptions = {}, ): Promise { const apiKey = options.apiKey ?? process.env.ELEVENLABS_API_KEY ?? ""; - const factory = options.clientFactory ?? defaultClientFactory; - const client = factory(apiKey); + // Loaded here rather than at module scope: the SDK is 4,549 modules and + // only a run that actually synthesizes should pay for them. + const client = options.clientFactory + ? options.clientFactory(apiKey) + : await loadElevenLabsClient(apiKey); const stream = await client.textToSpeech.convert(voiceId, { text, modelId: ELEVENLABS_TTS_MODEL,