diff --git a/.gitignore b/.gitignore index b1a9f5a55..f9eb04b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -243,6 +243,17 @@ python/outputs/recordings/background_handoff/segments/ !python/outputs/recordings/stt_swap/ python/outputs/recordings/stt_swap/segments/ +# TypeScript SDK artifacts — same shape as python/outputs/ above. Generated +# per run and NOT committed, except the whitelisted evidence directories +# below, which are the canonical "prove it" artifacts for reviewers. +# See javascript/outputs/README.md. +javascript/outputs/* +!javascript/outputs/README.md +!javascript/outputs/recordings/ +javascript/outputs/recordings/* +!javascript/outputs/recordings/README.md +!javascript/outputs/recordings/issue533_voicestyle/ + # TypeScript voice demo outputs (recordings + manifests) — same policy as # python/outputs/recordings/ above. Regenerated per local run by the @e2e demo tests # under javascript/examples/vitest/tests/voice/; the committed subset below diff --git a/docs/docs/pages/voice/recipes/effects.mdx b/docs/docs/pages/voice/recipes/effects.mdx index e4a7849a4..82ffecc6b 100644 --- a/docs/docs/pages/voice/recipes/effects.mdx +++ b/docs/docs/pages/voice/recipes/effects.mdx @@ -95,10 +95,15 @@ scenario.user("I'm really upset!", voice_style="angry") ``` -When `voice_style` is set, the SDK emits a warning if the active TTS backend -does not yet wire prosody — the audio is still synthesized, but the style hint -is not applied. `audio_effects` post-processing is always applied regardless of -the backend. +In **TypeScript**, `voiceStyle` reaches the TTS provider: ElevenLabs prepends an +inline `[angry]` marker (honoured by the pinned `eleven_v3` model) and OpenAI +passes it as `instructions` on `gpt-4o-mini-tts`. A provider that has not +declared style support warns once and synthesizes unstyled. + +In **Python**, `voice_style` is accepted but not yet routed to a provider — the +SDK emits a warning and the audio is synthesized without the style hint. + +`audio_effects` post-processing is always applied regardless of the backend. ## Worked example diff --git a/javascript/outputs/README.md b/javascript/outputs/README.md new file mode 100644 index 000000000..0127e730f --- /dev/null +++ b/javascript/outputs/README.md @@ -0,0 +1,24 @@ +# TypeScript SDK artifacts + +Sibling of `python/outputs/` — the parent directory for checked-in "prove it" +artifacts produced by the TypeScript SDK. Room for `traces/`, `logs/`, +`screenshots/` later; `recordings/` is the audio subdir and the only one +whitelisted in `.gitignore` today. + +Everything under here is generated. Only the explicitly whitelisted +subdirectories are committed, and only because a reviewer needs to hear or see +what a change actually produced against a live provider — a passing test cannot +carry that. + +## `recordings/` + +| Directory | Recorded | What it proves | +|---|---|---| +| `issue533_voicestyle/` | 2026-07-31 | A `voiceStyle` threaded through the user simulator reaches the real ElevenLabs API and is honoured as a delivery directive rather than spoken aloud (issue #533). Play `bare.wav` against `angry.wav`; `manifest.json` carries the requests, hashes, and Scribe transcripts. | + +Regenerate with the harness named in each `manifest.json` — for +`issue533_voicestyle/` that is: + +```bash +cd javascript && ELEVENLABS_API_KEY=... npx tsx scripts/voice-style-live-proof.ts +``` diff --git a/javascript/outputs/recordings/README.md b/javascript/outputs/recordings/README.md new file mode 100644 index 000000000..4b8e19359 --- /dev/null +++ b/javascript/outputs/recordings/README.md @@ -0,0 +1,44 @@ +# TypeScript SDK recordings + +Audio evidence produced by the TypeScript SDK against **live** providers, kept +because a reviewer needs to hear what a change actually produced. Mirrors +`python/outputs/recordings/` — see that directory's README for the original +statement of the policy. + +Everything here is generated. Only the directories whitelisted in `.gitignore` +are committed, and each one exists to answer a specific reviewer question that a +passing test cannot. + +| Directory | Recorded | What it proves | +|---|---|---| +| `issue533_voicestyle/` | 2026-07-31 | A `voiceStyle` threaded through the user simulator reaches the real ElevenLabs API and is honoured as a *delivery directive* rather than spoken aloud (issue #533). | + +## What each directory contains + +- The clips themselves, named for the condition they capture (here: `bare.wav` + vs `angry.wav` — same sentence, same voice, same model, one variable). +- `manifest.json` — the exact request text put on the wire, byte counts and + hashes, independent STT transcripts, the checks that ran, and a **`not_proven` + field** naming what the evidence does *not* establish. +- Screenshots of the harness output, so the PR body can embed proof that lives + in-tree rather than on a link that rots. + +## Reading the evidence + +**Byte-difference is not proof.** ElevenLabs `eleven_v3` is non-deterministic: +two identical requests already return different bytes. Any recording here that +claims "styled differs from unstyled" must discriminate on something stronger — +for `issue533_voicestyle/` that is the STT transcript, because `eleven_v3` is the +only EL model that *consumes* an inline `[angry]` marker as a directive instead +of reading it aloud. + +## Regenerating + +Each `manifest.json` names its `harness`. For `issue533_voicestyle/`: + +```bash +cd javascript && ELEVENLABS_API_KEY=... npx tsx scripts/voice-style-live-proof.ts +``` + +These harnesses spend real provider credits, so they are hand-run and never part +of `vitest run`. diff --git a/javascript/outputs/recordings/issue533_voicestyle/angry.wav b/javascript/outputs/recordings/issue533_voicestyle/angry.wav new file mode 100644 index 000000000..1af30c90b Binary files /dev/null and b/javascript/outputs/recordings/issue533_voicestyle/angry.wav differ diff --git a/javascript/outputs/recordings/issue533_voicestyle/bare.wav b/javascript/outputs/recordings/issue533_voicestyle/bare.wav new file mode 100644 index 000000000..b41d92835 Binary files /dev/null and b/javascript/outputs/recordings/issue533_voicestyle/bare.wav differ diff --git a/javascript/outputs/recordings/issue533_voicestyle/live-el-proof.png b/javascript/outputs/recordings/issue533_voicestyle/live-el-proof.png new file mode 100644 index 000000000..d68d2ac02 Binary files /dev/null and b/javascript/outputs/recordings/issue533_voicestyle/live-el-proof.png differ diff --git a/javascript/outputs/recordings/issue533_voicestyle/manifest.json b/javascript/outputs/recordings/issue533_voicestyle/manifest.json new file mode 100644 index 000000000..b568afd42 --- /dev/null +++ b/javascript/outputs/recordings/issue533_voicestyle/manifest.json @@ -0,0 +1,58 @@ +{ + "issue": 533, + "recorded": "2026-07-31", + "what_it_proves": "A voiceStyle threaded through the TypeScript simulator reaches the real ElevenLabs API and is honoured as a delivery directive, not spoken as text.", + "provider": "elevenlabs", + "tts_model": "eleven_v3", + "voice_id": "EXAVITQu4vr4xnSDxMaL", + "source_text": "I have been on hold for forty minutes and nobody has helped me.", + "harness": "javascript/scripts/voice-style-live-proof.ts", + "why_bytes_are_not_the_proof": "eleven_v3 is non-deterministic: two identical unstyled requests already return different bytes (168960 vs 161280 in this run). A byte-difference between styled and unstyled therefore proves nothing on its own. The discriminator is the transcript: eleven_v3 is the only ElevenLabs model that CONSUMES an inline [angry] marker as a delivery instruction \u2014 every other model reads it aloud. The styled clip never says 'angry', so the marker was consumed, not voiced.", + "clips": [ + { + "file": "bare.wav", + "request_text": "I have been on hold for forty minutes and nobody has helped me.", + "voice_style": null, + "pcm_bytes": 168960, + "pcm_sha256_16": "10bf6a62071bcee5", + "transcript_scribe_v1": "I have been on hold for 40 minutes and nobody has helped me." + }, + { + "file": "angry.wav", + "request_text": "[angry] I have been on hold for forty minutes and nobody has helped me.", + "voice_style": "angry", + "pcm_bytes": 195840, + "pcm_sha256_16": "e97d57ff6c8ac594", + "transcript_scribe_v1": "I have been on hold for 40 minutes and nobody has helped me" + } + ], + "checks": [ + { + "name": "marker not voiced", + "result": "PASS", + "detail": "the styled clip never says \"angry\" or \"[\" \u2014 the marker was consumed as a delivery directive" + }, + { + "name": "line still intact", + "result": "PASS", + "detail": "the styled clip still speaks the source line, so nothing was swallowed" + }, + { + "name": "cache does not collide", + "result": "PASS", + "detail": "router bare sha=d448e4345a1e1170 vs angry sha=5eb13fc16bbbd2c7" + }, + { + "name": "unstyled entry survives the styled call", + "result": "PASS", + "detail": "re-read of the unstyled key returned the cached bare bytes, not the styled ones" + } + ], + "format": { + "container": "wav", + "encoding": "pcm_s16le", + "sample_rate_hz": 24000, + "channels": 1 + }, + "not_proven": "Whether the styled clip SOUNDS angry to a human ear. No automated check can assert timbre \u2014 play both clips to judge that." +} diff --git a/javascript/outputs/recordings/issue533_voicestyle/mutation-proof.png b/javascript/outputs/recordings/issue533_voicestyle/mutation-proof.png new file mode 100644 index 000000000..447b05358 Binary files /dev/null and b/javascript/outputs/recordings/issue533_voicestyle/mutation-proof.png differ diff --git a/javascript/scripts/voice-style-live-proof.ts b/javascript/scripts/voice-style-live-proof.ts new file mode 100644 index 000000000..280f9ed99 --- /dev/null +++ b/javascript/scripts/voice-style-live-proof.ts @@ -0,0 +1,163 @@ +/** + * Live ElevenLabs proof for issue #533 — `voiceStyle` on the wire. + * + * Not a test: it costs real ElevenLabs credits, so it is a hand-run harness, + * never part of `vitest run`. Regenerates the checked-in evidence under + * `javascript/outputs/recordings/issue533_voicestyle/`. + * + * ELEVENLABS_API_KEY=... npx tsx scripts/voice-style-live-proof.ts + * + * Same text, same voice, twice: bare vs voiceStyle="angry", against the REAL + * ElevenLabs API. + * + * Byte-difference alone proves nothing: `eleven_v3` is non-deterministic, so + * two identical requests already return different bytes. The real + * discriminator is what the audio SAYS. `eleven_v3` is the only EL model that + * CONSUMES an inline `[angry]` marker as a delivery instruction; every other + * model READS IT ALOUD as text. So: + * + * - if the styled clip's transcript contains "angry" -> the marker was + * spoken, i.e. treated as text, i.e. the mapping is wrong; + * - if it does not, and the transcript still matches the source line, the + * marker was consumed as a directive and never voiced -> the style + * reached the wire and was honoured as a style. + * + * Transcription is done by ElevenLabs Scribe (the SDK's own STT leaf), so the + * check is independent of the TTS request we are asserting about. + */ +import { createHash } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { AudioChunk } from "../src/voice/audio-chunk"; +import { ElevenLabsSTTProvider } from "../src/voice/stt"; +import { synthesize, clearTtsCache } from "../src/voice/tts"; +import { elevenLabsSynthesizeBytes } from "../src/voice/tts/elevenlabs-tts"; + +const OUT_DIR = resolve( + dirname(fileURLToPath(import.meta.url)), + "..", + "outputs", + "recordings", + "issue533_voicestyle", +); +const BARE_OUT = resolve(OUT_DIR, "bare.wav"); +const ANGRY_OUT = resolve(OUT_DIR, "angry.wav"); + +const TEXT = "I have been on hold for forty minutes and nobody has helped me."; +const VOICE_ID = process.env.ELEVENLABS_VOICE_ID ?? "EXAVITQu4vr4xnSDxMaL"; + +function sha(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex").slice(0, 16); +} + +// PCM16/24kHz mono -> minimal WAV so GitHub renders an inline player. +function toWav(pcm: Uint8Array, sampleRate = 24000): Buffer { + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); // PCM + header.writeUInt16LE(1, 22); // mono + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(sampleRate * 2, 28); + header.writeUInt16LE(2, 32); + header.writeUInt16LE(16, 34); + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + return Buffer.concat([header, Buffer.from(pcm)]); +} + +let failures = 0; +function check(label: string, ok: boolean, detail: string) { + console.log(`${ok ? "PASS" : "FAIL"} — ${label}: ${detail}`); + if (!ok) failures += 1; +} + +async function main() { + if (!process.env.ELEVENLABS_API_KEY) { + console.error("ELEVENLABS_API_KEY is not set — cannot run the live proof"); + process.exit(2); + } + console.log(`text : ${JSON.stringify(TEXT)}`); + console.log(`voice : elevenlabs/${VOICE_ID} (model eleven_v3)`); + console.log(""); + + // --- Control: is the API even deterministic? ----------------------------- + const bareA = await elevenLabsSynthesizeBytes(TEXT, VOICE_ID, {}); + const bareB = await elevenLabsSynthesizeBytes(TEXT, VOICE_ID, {}); + console.log( + `control bare#1 bytes=${bareA.length} sha=${sha(bareA)} | ` + + `bare#2 bytes=${bareB.length} sha=${sha(bareB)}`, + ); + console.log( + sha(bareA) === sha(bareB) + ? " (deterministic — byte-diff would be meaningful)" + : " (NON-deterministic — byte-diff proves nothing; see transcripts below)", + ); + console.log(""); + + // --- The styled call ----------------------------------------------------- + const angry = await elevenLabsSynthesizeBytes(TEXT, VOICE_ID, { + voiceStyle: "angry", + }); + mkdirSync(OUT_DIR, { recursive: true }); + writeFileSync(BARE_OUT, toWav(bareA)); + writeFileSync(ANGRY_OUT, toWav(angry)); + console.log(`bare bytes=${bareA.length} sha=${sha(bareA)} -> ${BARE_OUT}`); + console.log(`angry bytes=${angry.length} sha=${sha(angry)} -> ${ANGRY_OUT}`); + console.log(""); + + // --- The real discriminator: what does the audio SAY? -------------------- + const stt = new ElevenLabsSTTProvider(); + const bareText = await stt.transcribe(new AudioChunk({ data: bareA })); + const angryText = await stt.transcribe(new AudioChunk({ data: angry })); + console.log(`transcript(bare) = ${JSON.stringify(bareText)}`); + console.log(`transcript(angry) = ${JSON.stringify(angryText)}`); + console.log(""); + + const spokeMarker = /angry|\[/i.test(angryText); + check( + "marker not voiced", + !spokeMarker, + spokeMarker + ? "the styled clip SPEAKS the marker — eleven_v3 read it as text, not as a directive" + : 'the styled clip never says "angry" or "[" — the marker was consumed as a delivery directive', + ); + // Scribe normalizes number words to digits ("forty" -> "40"), so accept both. + const carriesLine = /hold for (forty|40) minutes/i.test(angryText); + check( + "line still intact", + carriesLine, + carriesLine + ? "the styled clip still speaks the source line, so nothing was swallowed" + : "the styled clip lost the source line", + ); + + // --- Router level: does the cache keep styled and unstyled apart? -------- + clearTtsCache(); + const rBare = await synthesize(TEXT, `elevenlabs/${VOICE_ID}`); + const rAngry = await synthesize(TEXT, `elevenlabs/${VOICE_ID}`, undefined, { + voiceStyle: "angry", + }); + const rBareAgain = await synthesize(TEXT, `elevenlabs/${VOICE_ID}`); + check( + "cache does not collide", + sha(rBare.data) !== sha(rAngry.data), + `router bare sha=${sha(rBare.data)} vs angry sha=${sha(rAngry.data)}`, + ); + check( + "unstyled entry survives the styled call", + sha(rBareAgain.data) === sha(rBare.data), + `re-read of the unstyled key sha=${sha(rBareAgain.data)} (cache hit, not overwritten by the styled call)`, + ); + + console.log(""); + console.log(failures === 0 ? "ALL CHECKS PASSED" : `${failures} CHECK(S) FAILED`); + process.exit(failures === 0 ? 0 : 1); +} + +void main(); diff --git a/javascript/src/agents/__tests__/user-simulator-voice-style.test.ts b/javascript/src/agents/__tests__/user-simulator-voice-style.test.ts new file mode 100644 index 000000000..0ea53cc1d --- /dev/null +++ b/javascript/src/agents/__tests__/user-simulator-voice-style.test.ts @@ -0,0 +1,202 @@ +/** + * `voiceStyle` threading through the user simulator (issue #533). + * + * Before #533 the style stopped AT the simulator: `setOneShotOverride({ + * voiceStyle })` installed it, `synthesizeToAudioMessage` emitted a "no TTS + * provider currently honours it" warning, and `_synthesize` was then called + * with `(text, voice)` only — the style was dropped on the floor. + * + * These tests pin the wiring at the simulator boundary: every source of a + * style (per-step one-shot override / simulator `voiceStyle` / per-run + * `voice.tts.voiceStyle`) must reach `_synthesize`'s third argument with the + * documented precedence, and a one-shot override must not leak into the next + * turn. Offline — both the LLM and TTS are stubs; no network, no keys. + */ + +import { describe, it, expect, vi } from "vitest"; + +import type { AgentInput } from "../../domain"; +import { AudioChunk } from "../../voice/audio-chunk"; +import { + userSimulatorAgent, + type UserSimulatorAgentConfig, +} from "../user-simulator-agent"; + +// Mock getProjectConfig to avoid filesystem dependency in unit tests. +vi.mock("../../config", () => ({ + getProjectConfig: vi.fn().mockResolvedValue({ + defaultModel: { model: "openai/gpt-4.1-mini", temperature: 0 }, + }), +})); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** One recorded `_synthesize(text, voice, voiceStyle)` invocation. */ +interface SynthCall { + text: string; + voice: string; + voiceStyle?: string; +} + +/** + * Build a minimal {@link AgentInput} stub, optionally carrying the per-run + * `voice.tts` carrier (`run({ voice: { tts: { voice, voiceStyle } } })`). + */ +function makeInput(tts?: { voice: string; voiceStyle?: string }): AgentInput { + return { + threadId: "voice-style-thread", + messages: [], + newMessages: [], + requestedRole: "User" as AgentInput["requestedRole"], + scenarioConfig: { + name: "test", + description: "A test scenario description", + voice: tts ? { tts } : undefined, + } as AgentInput["scenarioConfig"], + scenarioState: {} as AgentInput["scenarioState"], + } as AgentInput; +} + +/** Wire a stub LLM into the simulator that always returns the given text. */ +function stubLlm(sim: ReturnType, text: string) { + ( + sim as unknown as { + invokeLLM: ( + p: unknown, + ) => Promise<{ text: string; toolCalls: []; steps: [] }>; + } + ).invokeLLM = async () => ({ text, toolCalls: [], steps: [] }); +} + +/** + * Replace `_synthesize` with a recorder that appends every `(text, voice, + * voiceStyle)` triple to `calls` and returns a 4-byte chunk. + */ +function recordSynth( + sim: ReturnType, + calls: SynthCall[], +) { + ( + sim as unknown as { + _synthesize: ( + text: string, + voice: string, + voiceStyle?: string, + ) => Promise; + } + )._synthesize = async (text, voice, voiceStyle) => { + calls.push({ text, voice, voiceStyle }); + return new AudioChunk({ data: new Uint8Array(4), transcript: text }); + }; +} + +// --------------------------------------------------------------------------- + +describe("UserSimulatorAgent voiceStyle → _synthesize (#533)", () => { + it("passes a one-shot override to _synthesize and reverts on the next turn", async () => { + const config: UserSimulatorAgentConfig = { voice: "openai/nova" }; + const sim = userSimulatorAgent(config); + stubLlm(sim, "I'm really upset about this!"); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + const restore = sim.setOneShotOverride({ voiceStyle: "angry" }); + await sim.call(makeInput()); + restore(); + await sim.call(makeInput()); + + expect(calls).toHaveLength(2); + expect(calls[0].voiceStyle).toBe("angry"); + // Restored — the style must NOT bleed into the following turn. + expect(calls[1].voiceStyle).toBeUndefined(); + expect(calls[1].voice).toBe("openai/nova"); + }); + + it("passes the simulator's configured voiceStyle when no override is installed", async () => { + const config: UserSimulatorAgentConfig = { + voice: "openai/nova", + voiceStyle: "cheerful", + }; + const sim = userSimulatorAgent(config); + stubLlm(sim, "hi there"); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + await sim.call(makeInput()); + + expect(calls).toHaveLength(1); + expect(calls[0].voiceStyle).toBe("cheerful"); + }); + + it("lets a one-shot override beat the configured voiceStyle for one turn only", async () => { + const config: UserSimulatorAgentConfig = { + voice: "openai/nova", + voiceStyle: "cheerful", + }; + const sim = userSimulatorAgent(config); + stubLlm(sim, "I'm really upset about this!"); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + const restore = sim.setOneShotOverride({ voiceStyle: "angry" }); + await sim.call(makeInput()); + restore(); + await sim.call(makeInput()); + + expect(calls[0].voiceStyle).toBe("angry"); + // `_voiceStyleOverride === null` means "no override installed", NOT + // "force unstyled" — the configured default resumes. + expect(calls[1].voiceStyle).toBe("cheerful"); + }); + + it("passes the per-run voice.tts.voiceStyle when the simulator has none", async () => { + const sim = userSimulatorAgent({}); + stubLlm(sim, "i guess so"); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + await sim.call(makeInput({ voice: "openai/nova", voiceStyle: "sad" })); + + expect(calls).toHaveLength(1); + expect(calls[0].voice).toBe("openai/nova"); + expect(calls[0].voiceStyle).toBe("sad"); + }); + + it("lets the simulator's voiceStyle beat the per-run voice.tts.voiceStyle", async () => { + const config: UserSimulatorAgentConfig = { + voice: "openai/nova", + voiceStyle: "cheerful", + }; + const sim = userSimulatorAgent(config); + stubLlm(sim, "hello"); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + await sim.call(makeInput({ voice: "openai/alloy", voiceStyle: "sad" })); + + expect(calls).toHaveLength(1); + // Simulator wins on BOTH axes — voice and style resolve the same way. + expect(calls[0].voice).toBe("openai/nova"); + expect(calls[0].voiceStyle).toBe("cheerful"); + }); + + it("passes the run-level style through voiceifyText (scripted content path)", async () => { + const sim = userSimulatorAgent({}); + const calls: SynthCall[] = []; + recordSynth(sim, calls); + + await sim.voiceifyText("I'm really upset about this!", { + tts: { voice: "openai/nova", voiceStyle: "sad" }, + }); + + expect(calls).toEqual([ + { + text: "I'm really upset about this!", + voice: "openai/nova", + voiceStyle: "sad", + }, + ]); + }); +}); diff --git a/javascript/src/agents/__tests__/user-simulator-voice.test.ts b/javascript/src/agents/__tests__/user-simulator-voice.test.ts index e8ae6b6fe..6b37350e7 100644 --- a/javascript/src/agents/__tests__/user-simulator-voice.test.ts +++ b/javascript/src/agents/__tests__/user-simulator-voice.test.ts @@ -1,7 +1,7 @@ /** * User simulator voice-path tests — PR4 of issue #372. * - * Binds 4 scenarios from `specs/voice-agents.feature` tagged `@ts-simulator`. + * Binds 5 scenarios from `specs/voice-agents.feature` tagged `@ts-simulator`. * Each scenario exercises the voice path of {@link UserSimulatorAgent} without * making real LLM or TTS calls — all external I/O is replaced by vitest spies. * @@ -9,12 +9,10 @@ * over-matching PR1's voice-contract-surface.test.ts which uses * `includeTags: ["ts-bound"]`. See issue #523 for the tag-convention decision. * - * Note: the `Per-step voice override applies to only that step` scenario in - * specs/voice-agents.feature stays @todo for the AUDIBLE voiceStyle effect — - * no TTS backend changes timbre by style yet (the simulator emits a one-shot - * warning). The per-step override PLUMBING (voiceStyle/audioEffects installed - * for one turn, then reverted) is wired and covered by - * `script/__tests__/interrupt-after-and-user-overrides.test.ts`. + * Note: `Per-step voice override applies to only that step` left @todo for the + * AUDIBLE voiceStyle effect until #533 gave the TTS providers a real style + * channel (ElevenLabs inline markers, OpenAI `instructions`). It is now + * `@ts-simulator` and bound here alongside the rest. */ import { dirname, resolve } from "node:path"; @@ -23,11 +21,11 @@ import { fileURLToPath } from "node:url"; import { loadFeature, describeFeature } from "@amiceli/vitest-cucumber"; import { expect, vi } from "vitest"; -import { AudioChunk } from "../../voice/audio-chunk"; import { makeChunk } from "./fixtures/make-chunk"; +import type { AgentInput } from "../../domain"; +import { AudioChunk } from "../../voice/audio-chunk"; import { extractAudio, messageHasAudio } from "../../voice/messages"; import { userSimulatorAgent, type UserSimulatorAgentConfig } from "../user-simulator-agent"; -import type { AgentInput } from "../../domain"; // Mock getProjectConfig to avoid filesystem dependency in unit tests. vi.mock("../../config", () => ({ @@ -99,10 +97,17 @@ function stubLlm(sim: ReturnType, text: string) { /** * Wire a stub synthesize function into the simulator. * Returns an AudioChunk with the given text as the transcript and 4 zero bytes. + * + * `fn` takes the full `_synthesize` signature — `voiceStyle` is the per-turn + * delivery style resolved by the simulator (#533). */ function stubSynth( sim: ReturnType, - fn?: (text: string, voice: string) => Promise + fn?: ( + text: string, + voice: string, + voiceStyle?: string + ) => Promise ) { const defaultFn = async (text: string): Promise => makeChunk(text); @@ -186,6 +191,60 @@ describeFeature( } ); + // ----------------------------------------------------------------------- + // Scenario: Per-step voice override applies to only that step (line 315) + // ----------------------------------------------------------------------- + Scenario( + "Per-step voice override applies to only that step", + ({ Given, When, Then, And }) => { + let sim: ReturnType; + const synthCalls: Array<{ + text: string; + voice: string; + voiceStyle?: string; + }> = []; + + Given( + "scenario.user(\"I'm really upset about this!\", voice_style=\"angry\")", + () => { + synthCalls.length = 0; + sim = userSimulatorAgent({ voice: "openai/nova" } as UserSimulatorAgentConfig); + stubLlm(sim, "I'm really upset about this!"); + // Record what the TTS layer is actually asked for — the style must + // reach `_synthesize`, not stop at the simulator (#533). + stubSynth(sim, async (text, voice, voiceStyle) => { + synthCalls.push({ text, voice, voiceStyle }); + return makeChunk(text); + }); + } + ); + + When("the step runs", async () => { + // The executor installs the one-shot override around exactly one + // turn (script/voice-steps.ts:withUserStepOverride). + const restore = sim.setOneShotOverride({ voiceStyle: "angry" }); + await sim.call(makeInput()); + restore(); + + // A following turn runs with no override installed. + await sim.call(makeInput()); + }); + + Then('the style "angry" is applied only to that turn', () => { + expect(synthCalls).toHaveLength(2); + expect(synthCalls[0].voiceStyle).toBe("angry"); + }); + + And( + "the simulator's default voice/effects resume on subsequent turns", + () => { + expect(synthCalls[1].voiceStyle).toBeUndefined(); + expect(synthCalls[1].voice).toBe("openai/nova"); + } + ); + } + ); + // ----------------------------------------------------------------------- // Scenario: Per-step audio_effects override applies to only that step (line 196) // ----------------------------------------------------------------------- diff --git a/javascript/src/agents/user-simulator-agent.ts b/javascript/src/agents/user-simulator-agent.ts index aea2bfcce..91a8d932a 100644 --- a/javascript/src/agents/user-simulator-agent.ts +++ b/javascript/src/agents/user-simulator-agent.ts @@ -32,6 +32,20 @@ export interface UserSimulatorVoiceConfig { */ persona?: string; + /** + * Default delivery style for every synthesized turn, e.g. `"angry"`, + * `"whispering"`. Distinct from {@link voice}: the same voice speaks it. + * + * Precedence: a per-step `user("…", { voiceStyle })` override wins for its + * one turn, then this value, then the per-run `voice.tts.voiceStyle`. + * + * Mapping is provider-specific — ElevenLabs prepends an inline `[angry]` + * marker (honoured by `eleven_v3`), OpenAI passes it as `instructions`. A + * provider that has not declared `supportsVoiceStyle` warns once and + * synthesizes unstyled. + */ + voiceStyle?: string; + /** * Optional array of audio effect functions applied to each synthesized audio * turn AFTER the TTS cache hit (effects are never baked into the cache key). @@ -198,19 +212,23 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { * Synthesize TTS audio. Exposed as a property so tests can replace it with * a stub without subclassing. * - * Signature mirrors `python/scenario/voice/tts.py:synthesize(text, voice)`. + * Signature mirrors `python/scenario/voice/tts.py:synthesize(text, voice)` + * plus a third `voiceStyle` argument — TS leads Python here (#533): the + * Python port still has no style channel, so this is NOT parity. * Returns an {@link AudioChunk} with the synthesized PCM16 data. * * Default: routes through the per-run TTS module (`voice/tts#synthesize`), * whose `"provider/voice"` router resolves the backend and whose LRU cache - * is keyed on `(sha256(text), voice)` — effects apply AFTER the cache read - * (in {@link voiceify}), never baked into the key. Per-run, not a module - * global. Tests inject a stub via `_synthesize`. + * is keyed on `(sha256(text), voice, voiceStyle)` — effects apply AFTER the + * cache read (in {@link synthesizeToAudioMessage}), never baked into the + * key. Per-run, not a module global. Tests inject a stub via `_synthesize`. */ - _synthesize: (text: string, voice: string) => Promise = ( - text, - voice, - ) => synthesize(text, voice); + _synthesize: ( + text: string, + voice: string, + voiceStyle?: string, + ) => Promise = (text, voice, voiceStyle) => + synthesize(text, voice, undefined, { voiceStyle }); // ---------------------------------------------- per-step overrides (§4.2) // Mirror of python/scenario/user_simulator_agent.py _one_shot_override. @@ -220,9 +238,6 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { _voiceStyleOverride: string | null = null; _audioEffectsOverride: Array<(audio: Uint8Array) => Uint8Array> | null = null; - /** Class-level one-shot warning flag (mirrors Python's class attribute). */ - private static _voiceStyleWarningEmitted = false; - constructor(private readonly cfg?: UserSimulatorAgentConfig) { super(); } @@ -248,18 +263,22 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { } /** - * Emit a one-shot UserWarning if voice_style was passed (not yet wired). + * Returns the effective delivery style for this turn. + * + * Precedence mirrors {@link effectiveAudioEffects} / {@link effectiveVoice}: + * the per-step one-shot override wins, then the simulator's own + * `cfg.voiceStyle`, then the per-run `voice.tts.voiceStyle`. + * + * `_voiceStyleOverride === null` means "no override installed" — NOT "force + * unstyled" — so between steps the configured defaults still apply. + * + * @param runVoiceStyle The per-run `voice.tts.voiceStyle`, when set. */ - private warnVoiceStyleOnce(): void { - if (UserSimulatorAgent._voiceStyleWarningEmitted) return; - UserSimulatorAgent._voiceStyleWarningEmitted = true; - // Node doesn't have a process.emitWarning parity issue — warn to console. - console.warn( - "UserSimulatorAgent: voice_style=... is accepted for forward " + - "compatibility but no TTS provider currently honours it. The " + - "simulator will synthesise without style modification. This will land " + - "as a per-provider instructions channel in a follow-up." - ); + private effectiveVoiceStyle(runVoiceStyle?: string): string | undefined { + if (this._voiceStyleOverride !== null) { + return this._voiceStyleOverride; + } + return this.cfg?.voiceStyle ?? runVoiceStyle; } /** @@ -330,7 +349,7 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { */ async voiceifyText( text: string, - runVoiceConfig?: { tts?: { voice?: string } }, + runVoiceConfig?: { tts?: { voice?: string; voiceStyle?: string } }, ): Promise { const voice = this.cfg?.voice ?? runVoiceConfig?.tts?.voice; const textMessage: ModelMessage = { role: "user", content: text }; @@ -345,7 +364,11 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { } return textMessage; } - return this.synthesizeToAudioMessage(text, voice); + return this.synthesizeToAudioMessage( + text, + voice, + this.effectiveVoiceStyle(runVoiceConfig?.tts?.voiceStyle), + ); } /** @@ -373,26 +396,32 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { typeof textMessage.content === "string" ? textMessage.content : ""; if (!content) return textMessage; - return this.synthesizeToAudioMessage(content, voice); + return this.synthesizeToAudioMessage( + content, + voice, + this.effectiveVoiceStyle(input.scenarioConfig.voice?.tts?.voiceStyle), + ); } /** * Shared TTS pipeline behind {@link voiceifyText} (scripted content) and - * {@link voiceify} (auto-generated turns): synthesize `text` with `voice`, - * apply any active one-shot warning + audio effects, and wrap the result in - * the canonical audio {@link ModelMessage}. Callers own the - * "should this turn be voiced?" decision (voice/empty-content guards) so - * this stays a pure text→audio-message transform. + * {@link voiceify} (auto-generated turns): synthesize `text` with `voice` in + * `voiceStyle`, apply any active audio effects, and wrap the result in the + * canonical audio {@link ModelMessage}. Callers own the + * "should this turn be voiced?" decision (voice/empty-content guards) and + * resolve the style via {@link effectiveVoiceStyle}, so this stays a pure + * text→audio-message transform. + * + * `voiceStyle` reaches the provider through the TTS options channel (#533) + * and participates in the TTS cache key, so a styled turn never serves (or + * poisons) the unstyled entry for the same text+voice. */ private async synthesizeToAudioMessage( text: string, voice: string, + voiceStyle?: string, ): Promise { - if (this._voiceStyleOverride !== null) { - this.warnVoiceStyleOnce(); - } - - const chunk = await this._synthesize(text, voice); + const chunk = await this._synthesize(text, voice, voiceStyle); let audioBytes = chunk.data; const effects = this.effectiveAudioEffects(); for (const effect of effects) { @@ -519,6 +548,9 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { * When set, each simulator turn is synthesized to audio. * @param config.persona Optional persona description appended to the system prompt. * @param config.audioEffects Optional audio effect functions applied after TTS synthesis. + * @param config.voiceStyle Optional default delivery style (e.g. `"angry"`) passed to the + * TTS provider for every turn. A per-step + * `user("…", { voiceStyle })` override wins for its one turn. * * @throws {Error} If no model is configured either in parameters or global config. * @@ -566,7 +598,7 @@ class UserSimulatorAgent extends UserSimulatorAgentAdapter { * **Implementation Notes:** * - Uses role reversal internally to work around LLM biases toward assistant roles * - Audio content is stripped from messages sent to the text LLM - * - TTS synthesis is applied AFTER the LLM generates text (cache key = (text, voice)) + * - TTS synthesis is applied AFTER the LLM generates text (cache key = (text, voice, voiceStyle)) * - Audio effects are applied AFTER any TTS cache hit (effects never enter the cache) */ export const userSimulatorAgent = (config?: UserSimulatorAgentConfig) => { diff --git a/javascript/src/voice/config.ts b/javascript/src/voice/config.ts index 25ba38c63..49d738a6b 100644 --- a/javascript/src/voice/config.ts +++ b/javascript/src/voice/config.ts @@ -58,6 +58,16 @@ export interface SttConfig { export interface TtsConfig { /** litellm-style `"provider/voice"`, e.g. `"openai/nova"`. */ voice: string; + /** + * Per-run default delivery style for every synthesized user turn, e.g. + * `"angry"`. Separate from {@link voice} because the same voice can speak in + * many styles. Mapping is provider-specific — ElevenLabs prepends an inline + * `[angry]` marker, OpenAI passes it as `instructions`; see the provider + * leaves under `voice/tts/`. Lowest-precedence source: the simulator's own + * `voiceStyle` and a per-step `user("…", { voiceStyle })` override both win + * over it. + */ + voiceStyle?: string; /** Output format hint. */ format?: AudioFormat; /** Per-run API key override. */ diff --git a/javascript/src/voice/index.ts b/javascript/src/voice/index.ts index fa6ce078c..2ed3e5723 100644 --- a/javascript/src/voice/index.ts +++ b/javascript/src/voice/index.ts @@ -120,7 +120,8 @@ export { } from "./voice-models"; // TTS subtree (split from the flat tts.ts; one file per provider — EDR §5.3). -// LRU cache invariant preserved (key = sha256(text)+voice; effects after read). +// LRU cache invariant preserved (key = sha256(text)+voice+voiceStyle; effects +// after read). export { clearTtsCache, listTtsProviders, @@ -129,6 +130,7 @@ export { type TTSCallable, type TtsEffectFn, type TtsProvider, + type TtsSynthesisOptions, } from "./tts"; // STT subtree (Gap #1 — split from the flat stt.ts; one file per provider). diff --git a/javascript/src/voice/tts/__tests__/elevenlabs-voice-style.test.ts b/javascript/src/voice/tts/__tests__/elevenlabs-voice-style.test.ts new file mode 100644 index 000000000..eac214e9e --- /dev/null +++ b/javascript/src/voice/tts/__tests__/elevenlabs-voice-style.test.ts @@ -0,0 +1,75 @@ +/** + * ElevenLabs `voiceStyle` mapping (issue #533). + * + * EL has no named-style field — `voiceSettings.style` is a numeric 0–1 + * exaggeration knob — so a style like `"angry"` is carried as an inline + * paralinguistic marker on the text, which the pinned `eleven_v3` model reads + * as a delivery instruction. These tests pin the marker actually reaching the + * wire (`textToSpeech.convert`'s `text` field), offline via the injected + * client factory. + */ + +import { describe, it, expect, vi } from "vitest"; + +import { elevenLabsSynthesizeBytes } from "../elevenlabs-tts"; + +/** + * Fake ElevenLabs SDK client whose `convert` records its arguments and returns + * a single PCM frame. + */ +function makeFakeClient() { + const convert = vi.fn(async () => { + async function* frames() { + yield new Uint8Array([1, 2, 3, 4]); + } + return frames(); + }); + return { convert, client: { textToSpeech: { convert } } }; +} + +describe("elevenLabsSynthesizeBytes voiceStyle → inline marker (#533)", () => { + it("prepends the [style] marker to the text when a style is supplied", async () => { + const { convert, client } = makeFakeClient(); + + await elevenLabsSynthesizeBytes("I'm upset", "voice-id", { + apiKey: "sk_fake", + clientFactory: () => client as never, + voiceStyle: "angry", + }); + + expect(convert).toHaveBeenCalledTimes(1); + expect(convert).toHaveBeenCalledWith( + "voice-id", + expect.objectContaining({ text: "[angry] I'm upset" }), + ); + }); + + it("sends the bare text when no style is supplied", async () => { + const { convert, client } = makeFakeClient(); + + await elevenLabsSynthesizeBytes("I'm upset", "voice-id", { + apiKey: "sk_fake", + clientFactory: () => client as never, + }); + + expect(convert).toHaveBeenCalledWith( + "voice-id", + expect.objectContaining({ text: "I'm upset" }), + ); + }); + + it("does not double-prepend a marker the text already opens with", async () => { + const { convert, client } = makeFakeClient(); + + await elevenLabsSynthesizeBytes("[angry] I'm upset", "voice-id", { + apiKey: "sk_fake", + clientFactory: () => client as never, + voiceStyle: "angry", + }); + + expect(convert).toHaveBeenCalledWith( + "voice-id", + expect.objectContaining({ text: "[angry] I'm upset" }), + ); + }); +}); diff --git a/javascript/src/voice/tts/__tests__/openai-voice-style.test.ts b/javascript/src/voice/tts/__tests__/openai-voice-style.test.ts new file mode 100644 index 000000000..7d3211155 --- /dev/null +++ b/javascript/src/voice/tts/__tests__/openai-voice-style.test.ts @@ -0,0 +1,59 @@ +/** + * OpenAI `voiceStyle` mapping (issue #533). + * + * `gpt-4o-mini-tts` takes an `instructions` parameter for delivery control, so + * a named style maps straight onto it. These tests pin BOTH directions: the + * instruction is present (and carries the style) when one is set, and the key + * is absent entirely when it is not — an unstyled request must stay exactly + * the request this callable always sent. + * + * The `openai` SDK is module-mocked; no network, no key. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; + +// `vi.mock` factories are hoisted above imports — `vi.hoisted` gives the spy a +// definition that exists by the time the factory runs. +const { speechCreate } = vi.hoisted(() => ({ speechCreate: vi.fn() })); + +vi.mock("openai", () => ({ + default: class FakeOpenAI { + audio = { speech: { create: speechCreate } }; + }, +})); + +import { openaiTts } from "../openai-tts"; + +describe("openaiTts voiceStyle → instructions (#533)", () => { + beforeEach(() => { + speechCreate.mockReset(); + speechCreate.mockResolvedValue({ + arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer, + }); + }); + + it("sends an instructions prompt carrying the style", async () => { + await openaiTts("I'm upset", "nova", { voiceStyle: "angry" }); + + expect(speechCreate).toHaveBeenCalledTimes(1); + const body = speechCreate.mock.calls[0][0] as Record; + expect(body["instructions"]).toBe("Speak in a angry tone."); + // The style must not leak into the spoken text. + expect(body["input"]).toBe("I'm upset"); + expect(body["voice"]).toBe("nova"); + }); + + it("omits the instructions key entirely when no style is set", async () => { + await openaiTts("I'm upset", "nova"); + + const body = speechCreate.mock.calls[0][0] as Record; + expect(body).not.toHaveProperty("instructions"); + }); + + it("omits the instructions key when options carry no style", async () => { + await openaiTts("I'm upset", "nova", {}); + + const body = speechCreate.mock.calls[0][0] as Record; + expect(body).not.toHaveProperty("instructions"); + }); +}); diff --git a/javascript/src/voice/tts/__tests__/synthesize-voice-style.test.ts b/javascript/src/voice/tts/__tests__/synthesize-voice-style.test.ts new file mode 100644 index 000000000..808146e96 --- /dev/null +++ b/javascript/src/voice/tts/__tests__/synthesize-voice-style.test.ts @@ -0,0 +1,82 @@ +/** + * TTS router `voiceStyle` plumbing (issue #533). + * + * Two invariants, both offline against a fake registered provider: + * 1. `synthesize(..., { voiceStyle })` reaches the provider callable's third + * argument — the style is a per-call option, not part of the voice id. + * 2. The LRU cache key includes the style. Without it, `(text, voice)` alone + * would let an angry turn and a neutral turn share one entry, so whichever + * ran first would silently serve its audio to the other. + * + * Complements `voice/__tests__/tts.test.ts`, which binds the `@ts-tts` spec + * scenario for the (text, voice) + effects-after-cache invariants. + */ + +import { describe, it, expect, beforeEach } from "vitest"; + +import { + clearTtsCache, + registerTtsProvider, + synthesize, + type TtsSynthesisOptions, +} from "../tts"; + +const PREFIX = "style-test-tts"; + +describe("synthesize(..., { voiceStyle }) (#533)", () => { + beforeEach(() => clearTtsCache()); + + it("forwards voiceStyle to the registered provider callable", async () => { + const seen: Array = []; + registerTtsProvider({ + prefix: PREFIX, + supportsVoiceStyle: true, + synth: async (_text, _name, options) => { + seen.push(options); + return new Uint8Array([1, 2, 3, 4]); + }, + }); + + await synthesize("hello", `${PREFIX}/alice`, undefined, { + voiceStyle: "angry", + }); + + expect(seen).toEqual([{ voiceStyle: "angry" }]); + }); + + it("keys the cache on voiceStyle so styled and unstyled turns never collide", async () => { + const styles: Array = []; + registerTtsProvider({ + prefix: PREFIX, + supportsVoiceStyle: true, + // Distinct payload per style so a cache collision is visible in the + // returned BYTES, not just in the call count. + synth: async (_text, _name, options) => { + styles.push(options?.voiceStyle); + return options?.voiceStyle === "angry" + ? new Uint8Array([9, 9, 9, 9]) + : new Uint8Array([1, 2, 3, 4]); + }, + }); + + const angry = await synthesize("same text", `${PREFIX}/alice`, undefined, { + voiceStyle: "angry", + }); + const neutral = await synthesize("same text", `${PREFIX}/alice`); + + // Same text + same voice, different style → two provider calls, two payloads. + expect(styles).toEqual(["angry", undefined]); + expect(Array.from(angry.data)).toEqual([9, 9, 9, 9]); + expect(Array.from(neutral.data)).toEqual([1, 2, 3, 4]); + + // Repeating the SAME (text, voice, style) triple is served from cache. + const angryAgain = await synthesize( + "same text", + `${PREFIX}/alice`, + undefined, + { voiceStyle: "angry" }, + ); + expect(styles).toHaveLength(2); + expect(Array.from(angryAgain.data)).toEqual([9, 9, 9, 9]); + }); +}); diff --git a/javascript/src/voice/tts/__tests__/voice-style-unsupported-warning.test.ts b/javascript/src/voice/tts/__tests__/voice-style-unsupported-warning.test.ts new file mode 100644 index 000000000..f1887487b --- /dev/null +++ b/javascript/src/voice/tts/__tests__/voice-style-unsupported-warning.test.ts @@ -0,0 +1,110 @@ +/** + * Router warning for providers that cannot honour `voiceStyle` (issue #533). + * + * A style the backend drops must be LOUD, not silent: otherwise a scripted + * `user("…", { voiceStyle: "angry" })` yields cheerful audio and nothing says + * why. The router warns once per provider — a styled multi-turn run must not + * repeat the line on every turn — and strips the style so the backend + * synthesizes unstyled rather than receiving an option it ignores. + */ + +import { describe, it, expect, beforeEach, vi } from "vitest"; + +import { + __resetVoiceStyleWarnings, + clearTtsCache, + registerTtsProvider, + synthesize, +} from "../tts"; + +const UNSUPPORTED = "style-unaware-tts"; +const OTHER_UNSUPPORTED = "style-unaware-tts-two"; +const SUPPORTED = "style-aware-tts"; + +/** Register a provider that records the style it was handed, if any. */ +function registerRecorder(prefix: string, supportsVoiceStyle: boolean) { + const styles: Array = []; + registerTtsProvider({ + prefix, + supportsVoiceStyle, + synth: async (_text, _name, options) => { + styles.push(options?.voiceStyle); + return new Uint8Array([1, 2, 3, 4]); + }, + }); + return styles; +} + +describe("synthesize() voiceStyle on a provider that does not support it (#533)", () => { + beforeEach(() => { + clearTtsCache(); + __resetVoiceStyleWarnings(); + }); + + it("warns exactly once per provider, names it and the style, and synthesizes unstyled", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const styles = registerRecorder(UNSUPPORTED, false); + + // DIFFERENT text on the second call so it misses the cache and genuinely + // reaches the router again — otherwise a cache hit, not the one-shot + // ledger, would be what suppresses the second warning. + await synthesize("first line", `${UNSUPPORTED}/alice`, undefined, { + voiceStyle: "angry", + }); + await synthesize("second line", `${UNSUPPORTED}/alice`, undefined, { + voiceStyle: "angry", + }); + + expect(warn).toHaveBeenCalledTimes(1); + const message = String(warn.mock.calls[0]?.[0]); + expect(message).toContain(UNSUPPORTED); + expect(message).toContain("angry"); + // Both turns synthesized unstyled — the style is stripped, not forwarded. + expect(styles).toEqual([undefined, undefined]); + + warn.mockRestore(); + }); + + it("warns per provider, not once globally", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + registerRecorder(UNSUPPORTED, false); + registerRecorder(OTHER_UNSUPPORTED, false); + + await synthesize("a line", `${UNSUPPORTED}/alice`, undefined, { + voiceStyle: "angry", + }); + await synthesize("a line", `${OTHER_UNSUPPORTED}/alice`, undefined, { + voiceStyle: "angry", + }); + + expect(warn).toHaveBeenCalledTimes(2); + expect(String(warn.mock.calls[1]?.[0])).toContain(OTHER_UNSUPPORTED); + + warn.mockRestore(); + }); + + it("never warns for a provider that declares supportsVoiceStyle", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const styles = registerRecorder(SUPPORTED, true); + + await synthesize("a line", `${SUPPORTED}/alice`, undefined, { + voiceStyle: "angry", + }); + + expect(warn).not.toHaveBeenCalled(); + expect(styles).toEqual(["angry"]); + + warn.mockRestore(); + }); + + it("does not warn when no style is requested", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + registerRecorder(UNSUPPORTED, false); + + await synthesize("a line", `${UNSUPPORTED}/alice`); + + expect(warn).not.toHaveBeenCalled(); + + warn.mockRestore(); + }); +}); diff --git a/javascript/src/voice/tts/elevenlabs-tts.ts b/javascript/src/voice/tts/elevenlabs-tts.ts index 0683e8e9e..c220fd5bf 100644 --- a/javascript/src/voice/tts/elevenlabs-tts.ts +++ b/javascript/src/voice/tts/elevenlabs-tts.ts @@ -8,7 +8,8 @@ * * Wire: `client.textToSpeech.convert(voiceId, { modelId: eleven_v3, * outputFormat: "pcm_24000" })` → raw PCM16/24 kHz mono, matching the - * canonical {@link AudioChunk}. + * canonical {@link AudioChunk}. A `voiceStyle` rides on the text as an inline + * `[angry]`-style marker — see {@link applyVoiceStyle}. * * Registered under the `elevenlabs` prefix by `tts/index.ts` (side effect). */ @@ -28,11 +29,40 @@ export interface ElevenLabsTtsOptions { apiKey?: string; /** Test seam — override the SDK client constructor. */ clientFactory?: ElevenLabsClientFactory; + /** + * Named delivery style for this utterance, e.g. `"angry"` (#533). Applied as + * an inline paralinguistic marker on the text — see {@link applyVoiceStyle} + * for why that, and not `voiceSettings.style`. + */ + voiceStyle?: string; } const defaultClientFactory: ElevenLabsClientFactory = (apiKey) => new ElevenLabsClient({ apiKey }); +/** + * Prefix `text` with the inline `[style]` marker that {@link + * ELEVENLABS_TTS_MODEL} reads as a delivery instruction. + * + * **Design decision (#533).** ElevenLabs has no named-style field: its + * `voiceSettings.style` is a NUMERIC 0–1 *exaggeration* knob, so a string like + * `"angry"` has nowhere to go — passing it there is a type error, and mapping + * it to a number would be inventing a meaning the API never promised. What + * `eleven_v3` (already the pinned model, for exactly this reason) DOES honour + * is inline paralinguistic markers — `[angry]`, `[whispering]`, `[laughs]`. + * Prepending the marker is therefore the mechanism that actually makes a named + * style audible, so that is what a `voiceStyle` maps to here. + * + * Idempotent: text that already opens with the same marker is returned + * unchanged, so a caller that hand-wrote `[angry] …` is not double-prepended. + */ +function applyVoiceStyle(text: string, voiceStyle?: string): string { + if (!voiceStyle) return text; + const marker = `[${voiceStyle}]`; + if (text.startsWith(marker)) return text; + return `${marker} ${text}`; +} + /** * Synthesize `text` to raw PCM16/24 kHz bytes via the ElevenLabs SDK. * @@ -48,7 +78,7 @@ export async function elevenLabsSynthesizeBytes( const factory = options.clientFactory ?? defaultClientFactory; const client = factory(apiKey); const stream = await client.textToSpeech.convert(voiceId, { - text, + text: applyVoiceStyle(text, options.voiceStyle), modelId: ELEVENLABS_TTS_MODEL, outputFormat: "pcm_24000", }); @@ -71,11 +101,17 @@ export class ElevenLabsTtsProvider { this.options = options; } - /** `(text, voiceId) => PCM16/24kHz bytes`, bound to this provider's options. */ - readonly synth: TTSCallable = (text, voiceId) => - elevenLabsSynthesizeBytes(text, voiceId, this.options); + /** + * `(text, voiceId, options?) => PCM16/24kHz bytes`, bound to this provider's + * options. A per-call `voiceStyle` wins over a construction-time one. + */ + readonly synth: TTSCallable = (text, voiceId, options) => + elevenLabsSynthesizeBytes(text, voiceId, { + ...this.options, + voiceStyle: options?.voiceStyle ?? this.options.voiceStyle, + }); } /** Registry callable — uses the env API key (the `elevenlabs/...` router path). */ -export const elevenLabsTts: TTSCallable = (text, voiceId) => - elevenLabsSynthesizeBytes(text, voiceId); +export const elevenLabsTts: TTSCallable = (text, voiceId, options) => + elevenLabsSynthesizeBytes(text, voiceId, { voiceStyle: options?.voiceStyle }); diff --git a/javascript/src/voice/tts/index.ts b/javascript/src/voice/tts/index.ts index b49cfb61e..aad24accc 100644 --- a/javascript/src/voice/tts/index.ts +++ b/javascript/src/voice/tts/index.ts @@ -6,8 +6,8 @@ * re-exports the interface, router, cache, and provider leaves. * * Replaces the flat `voice/tts.ts` (one file per provider — EDR §5.3). The LRU - * cache invariant is preserved (key = sha256(text)+voice; effects applied AFTER - * cache read) — see `./tts`. + * cache invariant is preserved (key = sha256(text)+voice+voiceStyle; effects + * applied AFTER cache read) — see `./tts`. */ export { @@ -15,9 +15,11 @@ export { listTtsProviders, registerTtsProvider, synthesize, + __resetVoiceStyleWarnings, type TTSCallable, type TtsEffectFn, type TtsProvider, + type TtsSynthesisOptions, } from "./tts"; export { openaiTts } from "./openai-tts"; @@ -35,5 +37,15 @@ import { elevenLabsTts } from "./elevenlabs-tts"; import { openaiTts } from "./openai-tts"; import { registerTtsProvider } from "./tts"; -registerTtsProvider({ prefix: "openai", synth: openaiTts }); -registerTtsProvider({ prefix: "elevenlabs", synth: elevenLabsTts }); +// Both leaves honour `voiceStyle` (#533): OpenAI via the `instructions` +// parameter, ElevenLabs via an inline `[angry]` marker on `eleven_v3`. +registerTtsProvider({ + prefix: "openai", + synth: openaiTts, + supportsVoiceStyle: true, +}); +registerTtsProvider({ + prefix: "elevenlabs", + synth: elevenLabsTts, + supportsVoiceStyle: true, +}); diff --git a/javascript/src/voice/tts/openai-tts.ts b/javascript/src/voice/tts/openai-tts.ts index 15d6d3621..fce3f612b 100644 --- a/javascript/src/voice/tts/openai-tts.ts +++ b/javascript/src/voice/tts/openai-tts.ts @@ -13,19 +13,35 @@ * design choice is that callers swap the whole callable (i.e. supply a * different {@link TTSCallable}) rather than parameterising this one. * See {@link OPENAI_TTS_MODEL} for why it is the current-gen default. + * + * That rule has exactly one documented exception: `voiceStyle` (#533). It is + * per-utterance, not per-backend — the same voice on the same model speaks an + * angry turn and a calm one — so swapping the callable would be the wrong + * granularity. `gpt-4o-mini-tts` exposes an `instructions` parameter for + * precisely this, so the style maps onto it. */ import { OPENAI_TTS_MODEL } from "../voice-models"; import type { TTSCallable } from "./tts"; -/** OpenAI TTS callable — `(text, voiceName) => PCM16/24kHz bytes`. */ -export const openaiTts: TTSCallable = async (text, voiceName) => { +/** + * OpenAI TTS callable — `(text, voiceName, options?) => PCM16/24kHz bytes`. + * + * `options.voiceStyle` becomes an `instructions` prompt. The key is omitted + * entirely when no style is set, so the unstyled request is byte-for-byte the + * one this callable always sent. + */ +export const openaiTts: TTSCallable = async (text, voiceName, options) => { const { default: OpenAI } = await import("openai"); const client = new OpenAI(); + const voiceStyle = options?.voiceStyle; const response = await client.audio.speech.create({ model: OPENAI_TTS_MODEL, voice: voiceName, input: text, response_format: "pcm", + ...(voiceStyle + ? { instructions: `Speak in a ${voiceStyle} tone.` } + : {}), }); const arrayBuffer = await response.arrayBuffer(); let bytes = new Uint8Array(arrayBuffer); diff --git a/javascript/src/voice/tts/tts.ts b/javascript/src/voice/tts/tts.ts index 808589832..8d4142252 100644 --- a/javascript/src/voice/tts/tts.ts +++ b/javascript/src/voice/tts/tts.ts @@ -3,9 +3,13 @@ * * Python parity: `python/scenario/voice/tts.py`. Litellm-style routing — voice * strings are `provider/name` (e.g. `openai/nova`, `elevenlabs/rachel`). The - * TTS cache key is `(sha256(text), voice)` so raw user-supplied text never - * reaches the cache payload; audio effects apply AFTER cache hit and are never - * baked into stored audio. + * TTS cache key is `(sha256(text), voice, voiceStyle)` so raw user-supplied + * text never reaches the cache payload; audio effects apply AFTER cache hit + * and are never baked into stored audio. + * + * TS leads Python here: `voiceStyle` (issue #533) is a per-call synthesis + * option threaded from the user simulator down to the provider. Python's + * `synthesize(text, voice)` still has no style channel — this is NOT parity. * * Concrete providers (OpenAI, ElevenLabs) are one-file-per-provider leaves that * self-register via `tts/index.ts` (mirrors the `stt/` subtree, EDR §5.3). This @@ -16,8 +20,35 @@ import { createHash } from "node:crypto"; import { AudioChunk } from "../audio-chunk"; -/** A TTS backend: takes (text, voiceName) and returns PCM16/24kHz mono bytes. */ -export type TTSCallable = (text: string, voiceName: string) => Promise; +/** + * Per-call synthesis options that are NOT part of the voice identity. + * + * Kept separate from the `provider/name` voice string because the SAME voice + * can speak in many styles — the style varies per utterance, the voice does + * not. Optional on {@link TTSCallable} so existing two-argument provider + * callables stay assignable. + */ +export interface TtsSynthesisOptions { + /** + * Named delivery style for this utterance, e.g. `"angry"`, `"whispering"`. + * Providers map it onto their own style channel (ElevenLabs: an inline + * `[angry]` marker; OpenAI: the `instructions` parameter) and opt in via + * {@link TtsProvider.supportsVoiceStyle}. A provider that has not opted in + * gets the style stripped, and the router warns once. + */ + voiceStyle?: string; +} + +/** + * A TTS backend: takes (text, voiceName, options?) and returns PCM16/24kHz + * mono bytes. `options` carries per-call, non-identity knobs such as + * {@link TtsSynthesisOptions.voiceStyle}. + */ +export type TTSCallable = ( + text: string, + voiceName: string, + options?: TtsSynthesisOptions, +) => Promise; /** * A TTS provider registration — a litellm-style prefix and the function that @@ -26,6 +57,13 @@ export type TTSCallable = (text: string, voiceName: string) => Promise AudioChunk | Promise; -const PROVIDERS = new Map(); +/** Registry entry — the callable plus the capabilities it declared. */ +interface RegisteredTtsProvider { + synth: TTSCallable; + supportsVoiceStyle: boolean; +} + +const PROVIDERS = new Map(); /** - * In-process LRU cache keyed on (sha256(text), voice) → PCM16 bytes. Bounded to - * prevent unbounded memory growth — a 5-minute clip is ~14 MB, so 64 entries - * caps the cache at ~900 MB even for long utterances. (Mirrors the Python - * tuning.) + * In-process LRU cache keyed on (sha256(text), voice, voiceStyle) → PCM16 + * bytes. Bounded to prevent unbounded memory growth — a 5-minute clip is + * ~14 MB, so 64 entries caps the cache at ~900 MB even for long utterances. + * (Mirrors the Python tuning.) */ const CACHE_MAX_ENTRIES = 64; const CACHE = new Map(); @@ -50,9 +94,45 @@ export function clearTtsCache(): void { CACHE.clear(); } +/** + * Prefixes already warned about an ignored `voiceStyle`. One warning per + * provider for the life of the process — a styled multi-turn run would + * otherwise repeat the same line on every turn. + */ +const VOICE_STYLE_WARNED = new Set(); + +/** + * Reset the one-shot "provider ignores voiceStyle" warning ledger. + * + * @internal Test-only seam. Deliberately NOT folded into + * {@link clearTtsCache}: the two have unrelated lifetimes, and a suite that + * clears the cache between cases must not silently re-arm the warning it is + * asserting fires exactly once. + */ +export function __resetVoiceStyleWarnings(): void { + VOICE_STYLE_WARNED.clear(); +} + +function warnVoiceStyleUnsupportedOnce( + provider: string, + voiceStyle: string, +): void { + if (VOICE_STYLE_WARNED.has(provider)) return; + VOICE_STYLE_WARNED.add(provider); + console.warn( + `[scenario] TTS provider ${JSON.stringify(provider)} does not support ` + + `voiceStyle — the style ${JSON.stringify(voiceStyle)} is ignored and ` + + "this turn is synthesized unstyled. Providers declare support with " + + "registerTtsProvider({ …, supportsVoiceStyle: true }).", + ); +} + /** Register a TTS backend under the given provider prefix. */ export function registerTtsProvider(provider: TtsProvider): void { - PROVIDERS.set(provider.prefix.toLowerCase(), provider.synth); + PROVIDERS.set(provider.prefix.toLowerCase(), { + synth: provider.synth, + supportsVoiceStyle: provider.supportsVoiceStyle ?? false, + }); } /** Test-only: enumerate registered provider prefixes. */ @@ -78,44 +158,82 @@ function hashText(text: string): string { return createHash("sha256").update(text, "utf8").digest("hex"); } -function cacheKey(textHash: string, voice: string): string { - // Composite key — text hash and voice are both load-bearing; "voice" is the - // full provider/name string so two providers can't collide. - return `${textHash}:${voice}`; +function cacheKey(textHash: string, voice: string, voiceStyle?: string): string { + // Composite key — text hash, voice and voice style are all load-bearing. + // "voice" is the full provider/name string so two providers can't collide. + // `voiceStyle` is in the key because it changes the synthesized AUDIO, not + // just the request: without it, an angry turn and a neutral turn with the + // same (text, voice) would share one entry, so whichever ran first would + // silently serve its bytes to the other (issue #533). Effects stay OUT of + // the key — they are applied after the cache read, never baked in. + return `${textHash}:${voice}:${voiceStyle ?? ""}`; } -async function synthesizeRaw(text: string, voice: string): Promise { +/** True when `options` carries at least one set value (all fields optional). */ +function hasAnyOption(options?: TtsSynthesisOptions): boolean { + return ( + options !== undefined && Object.values(options).some((v) => v !== undefined) + ); +} + +async function synthesizeRaw( + text: string, + voice: string, + options?: TtsSynthesisOptions, +): Promise { const { provider, name } = splitVoice(voice); - const fn = PROVIDERS.get(provider); - if (!fn) { + const registered = PROVIDERS.get(provider); + if (!registered) { throw new Error( `Unknown TTS provider ${JSON.stringify(provider)}. Known: ${listTtsProviders().join(", ") || "(none)"}`, ); } - return fn(text, name); + + // A style the backend cannot honour must be loud, not silent — otherwise a + // scripted `user("…", { voiceStyle: "angry" })` produces cheerful audio and + // nothing says why. Warn once per provider, then strip ONLY the style so + // any other per-call option still reaches the backend. + let effective = options; + const voiceStyle = options?.voiceStyle; + if (voiceStyle && !registered.supportsVoiceStyle) { + warnVoiceStyleUnsupportedOnce(provider, voiceStyle); + effective = { ...options, voiceStyle: undefined }; + } + + // Providers registered before per-call options existed are two-argument + // callables. Only widen the call when an option is actually set, so the + // (dominant) unstyled path keeps its historical arity — a third `undefined` + // argument is observable to callables that inspect `arguments.length`. + return hasAnyOption(effective) + ? registered.synth(text, name, effective) + : registered.synth(text, name); } /** * Synthesize `text` into an {@link AudioChunk} using the voice provider. * - * Cache key is `(sha256(text), voice)` — equivalent determinism to - * `(text, voice)` without pinning raw text in the cache payload. Effects pass - * through `effectFn` AFTER a cache hit and are never part of the key, matching - * the locked-decision invariant from the Python port. + * Cache key is `(sha256(text), voice, voiceStyle)` — equivalent determinism to + * `(text, voice, voiceStyle)` without pinning raw text in the cache payload. + * Effects pass through `effectFn` AFTER a cache hit and are never part of the + * key, matching the locked-decision invariant from the Python port. + * + * @param options Per-call synthesis knobs (currently + * {@link TtsSynthesisOptions.voiceStyle}) forwarded to the provider. */ export async function synthesize( text: string, voice: string, effectFn?: TtsEffectFn, + options?: TtsSynthesisOptions, ): Promise { - const key = cacheKey(hashText(text), voice); + const key = cacheKey(hashText(text), voice, options?.voiceStyle); let pcm = CACHE.get(key); if (pcm !== undefined) { // LRU touch — delete + set re-inserts at the tail of insertion order. CACHE.delete(key); CACHE.set(key, pcm); } else { - pcm = await synthesizeRaw(text, voice); + pcm = await synthesizeRaw(text, voice, options); CACHE.set(key, pcm); while (CACHE.size > CACHE_MAX_ENTRIES) { const oldest = CACHE.keys().next().value; diff --git a/specs/voice-agents.feature b/specs/voice-agents.feature index 0148ac88a..6d4893be9 100644 --- a/specs/voice-agents.feature +++ b/specs/voice-agents.feature @@ -311,13 +311,10 @@ Feature: Voice agent testing in Scenario SDK Then the TTS synthesis is cached on (text, voice) and only called once And effects are applied to the cached audio after retrieval, never baked in - @unit @todo + @unit @ts-simulator Scenario: Per-step voice override applies to only that step # Source §4.2, L290-294 - # @todo for the AUDIBLE voiceStyle effect — no TTS backend changes timbre - # by style yet (the simulator emits a one-shot warning). The per-step - # override PLUMBING (one-turn install + revert) IS wired; covered by - # script/__tests__/interrupt-after-and-user-overrides.test.ts. + # Bound by javascript/src/agents/__tests__/user-simulator-voice.test.ts Given scenario.user("I'm really upset about this!", voice_style="angry") When the step runs Then the style "angry" is applied only to that turn