|
1 | 1 | import { describe, expect, it } from "vitest"; |
2 | | -import { snapWordBoundariesToAudio } from "./snapWordBoundaries"; |
| 2 | +import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries"; |
3 | 3 | import type { SttWordSegment } from "./transcriptionContract"; |
4 | 4 |
|
5 | | -const SAMPLE_RATE = 16_000; |
6 | | - |
7 | | -/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */ |
8 | | -function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array { |
9 | | - const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE)); |
10 | | - for (let i = 0; i < samples.length; i++) { |
11 | | - const t = i / SAMPLE_RATE; |
12 | | - const silent = silences.some(([from, to]) => t >= from && t < to); |
13 | | - // Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone. |
14 | | - samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5; |
15 | | - } |
16 | | - return samples; |
17 | | -} |
18 | | - |
19 | | -const word = (w: Partial<SttWordSegment> = {}): SttWordSegment => ({ |
20 | | - word: "w", |
21 | | - startSec: 0, |
22 | | - endSec: 0.1, |
23 | | - ...w, |
| 5 | +/** A helper word; its anchor defaults to its start, as for a request's first word. */ |
| 6 | +const word = ( |
| 7 | + text: string, |
| 8 | + startSec: number, |
| 9 | + endSec: number, |
| 10 | + anchorSec = startSec, |
| 11 | +): HelperWord => ({ |
| 12 | + word: text, |
| 13 | + startSec, |
| 14 | + endSec, |
| 15 | + anchorSec, |
24 | 16 | }); |
25 | | - |
26 | | -describe("snapWordBoundariesToAudio", () => { |
27 | | - it("pulls a late boundary back into the silence that precedes it", () => { |
28 | | - // Speech stops at 1.0 and resumes at 1.2; whisper reports the next word |
29 | | - // starting at 1.3 — 100 ms after the audio actually resumed. |
30 | | - const samples = audioWithSilences(3, [[1.0, 1.2]]); |
31 | | - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples); |
32 | | - expect(snapped.startSec).toBeGreaterThanOrEqual(1.0); |
33 | | - expect(snapped.startSec).toBeLessThan(1.2); |
34 | | - }); |
35 | | - |
36 | | - it("leaves a boundary alone when nothing quieter precedes it", () => { |
37 | | - // A word ending a phrase: whisper is already right, the frames before the |
38 | | - // boundary are all speech, so the quietest frame in the window is the |
39 | | - // boundary itself and it must not drift. |
40 | | - const samples = audioWithSilences(3, [[1.5, 2.0]]); |
41 | | - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples); |
42 | | - expect(snapped.endSec).toBeCloseTo(1.5, 2); |
43 | | - }); |
44 | | - |
45 | | - it("never moves a boundary more than the lookback window", () => { |
46 | | - const samples = audioWithSilences(3, [[0.0, 1.0]]); |
47 | | - const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples); |
48 | | - expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15); |
49 | | - }); |
50 | | - |
51 | | - it("keeps a boundary shared by two words shared", () => { |
52 | | - const samples = audioWithSilences(3, [[1.0, 1.2]]); |
53 | | - const [first, second] = snapWordBoundariesToAudio( |
54 | | - [word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })], |
55 | | - samples, |
56 | | - ); |
57 | | - expect(first.endSec).toBeCloseTo(second.startSec, 6); |
58 | | - }); |
59 | | - |
60 | | - it("leaves boundaries that fall outside the decoded audio alone", () => { |
61 | | - // Clamping these into range would collapse every boundary onto the end of |
62 | | - // the buffer instead of leaving the unmeasurable ones untouched. |
63 | | - const samples = audioWithSilences(0.1, []); |
64 | | - const words = [word({ startSec: 5.51, endSec: 6.85 })]; |
65 | | - expect(snapWordBoundariesToAudio(words, samples)).toEqual(words); |
66 | | - }); |
67 | | - |
68 | | - it("keeps degenerate words non-empty and passes words through without audio", () => { |
69 | | - const samples = audioWithSilences(3, [[1.0, 1.2]]); |
70 | | - const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples); |
71 | | - expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec); |
72 | | - |
73 | | - const untouched = [word({ startSec: 1.3, endSec: 1.8 })]; |
74 | | - expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched); |
| 17 | +const ms = (sec: number) => Math.round(sec * 1000) / 1000; |
| 18 | +const times = (words: SttWordSegment[]) => words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]); |
| 19 | + |
| 20 | +describe("anchorWordsOnSpeech", () => { |
| 21 | + it("returns the helper's times without speech intervals, minus the anchor", () => { |
| 22 | + const out = anchorWordsOnSpeech([word("a", 1, 1.4, 1.2), word("b", 1.4, 1.4, 1.4)]); |
| 23 | + expect(out).toEqual([ |
| 24 | + { word: "a", startSec: 1, endSec: 1.4 }, |
| 25 | + { word: "b", startSec: 1.4, endSec: 1.42 }, |
| 26 | + ]); |
75 | 27 | }); |
76 | | -}); |
77 | 28 |
|
78 | | -describe("snapWordBoundariesToAudio with speech intervals", () => { |
79 | | - // Loud and flat everywhere, so the RMS snap moves nothing and only the |
80 | | - // anchoring on `speech` is under test. |
81 | | - const flat = audioWithSilences(6, []); |
82 | | - const ms = (sec: number) => Math.round(sec * 1000) / 1000; |
83 | | - const times = (words: SttWordSegment[]) => |
84 | | - words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]); |
85 | | - |
86 | | - it("puts each phrase's first word on its onset and its closing punctuation on its end", () => { |
87 | | - // The shape of a real French take: whisper put "Salut" 0.58 s and "Bah" |
88 | | - // 0.25 s after the speech started, ran "Salut" on through the pause, and |
89 | | - // dropped the "!" closing the first phrase just after the second began. |
| 29 | + it("puts a phrase's first word on its onset whether DTW put it late or early", () => { |
| 30 | + // "Salut" opens the request, so it has no previous token and starts late, |
| 31 | + // on its own first token. "Bah" starts where the previous token ended, at |
| 32 | + // the end of the first phrase: in the pause, before its own speech. |
90 | 33 | const words = [ |
91 | | - word({ word: "Salut", startSec: 2.15, endSec: 3.37 }), |
92 | | - word({ word: "!", startSec: 3.37, endSec: 3.39 }), |
93 | | - word({ word: "Bah", startSec: 3.61, endSec: 4.01 }), |
94 | | - word({ word: "voilà", startSec: 4.01, endSec: 4.35 }), |
| 34 | + word("Salut", 2.15, 2.5), |
| 35 | + word("!", 2.5, 2.6, 2.6), |
| 36 | + word("Bah", 2.6, 3.8, 3.61), |
| 37 | + word("voilà", 3.8, 4.35, 4.01), |
95 | 38 | ]; |
96 | 39 | const speech = [ |
97 | 40 | { startSec: 1.57, endSec: 2.56 }, |
98 | 41 | { startSec: 3.36, endSec: 4.35 }, |
99 | 42 | ]; |
100 | | - expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([ |
| 43 | + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ |
101 | 44 | ["Salut", 1.57, 2.56], |
102 | 45 | ["!", 2.56, 2.56], |
103 | | - ["Bah", 3.36, 4.01], |
104 | | - ["voilà", 4.01, 4.35], |
| 46 | + ["Bah", 3.36, 3.8], |
| 47 | + ["voilà", 3.8, 4.35], |
105 | 48 | ]); |
106 | 49 | }); |
107 | 50 |
|
108 | 51 | it("ends each phrase's last word where its speech stops, stretched or cut back", () => { |
109 | | - // The end of the same take: "tic," ran on into the pause, whisper closed |
110 | | - // the last segment at 2.79 while "tac" ran to 3.04, and dropped the "!" on |
111 | | - // the word itself. |
112 | 52 | const words = [ |
113 | | - word({ word: "tic,", startSec: 1.95, endSec: 2.59 }), |
114 | | - word({ word: "tac", startSec: 2.59, endSec: 2.79 }), |
115 | | - word({ word: "!", startSec: 2.62, endSec: 2.79 }), |
| 53 | + word("tic,", 1.95, 2.59), |
| 54 | + word("tac", 2.59, 2.79, 2.7), |
| 55 | + word("!", 2.79, 2.9, 2.9), |
116 | 56 | ]; |
117 | 57 | const speech = [ |
118 | 58 | { startSec: 1.95, endSec: 2.37 }, |
119 | 59 | { startSec: 2.59, endSec: 3.04 }, |
120 | 60 | ]; |
121 | | - expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([ |
| 61 | + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ |
122 | 62 | ["tic,", 1.95, 2.37], |
123 | 63 | ["tac", 2.59, 3.04], |
124 | 64 | ["!", 3.04, 3.04], |
125 | 65 | ]); |
126 | 66 | }); |
127 | 67 |
|
128 | | - it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => { |
129 | | - const onTime = [word({ startSec: 0.95, endSec: 3 })]; |
130 | | - expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual( |
131 | | - times(onTime), |
132 | | - ); |
| 68 | + it("leaves an edge alone when the word is too far past it to be that edge", () => { |
133 | 69 | // 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour |
134 | 70 | // of words whisper dropped than the phrase's own edges. |
135 | | - const tooFar = [word({ startSec: 2.2, endSec: 2.5 })]; |
136 | | - expect(times(snapWordBoundariesToAudio(tooFar, flat, [{ startSec: 1, endSec: 3.6 }]))).toEqual( |
| 71 | + const tooFar = [word("w", 2.2, 2.5)]; |
| 72 | + expect(times(anchorWordsOnSpeech(tooFar, [{ startSec: 1, endSec: 3.6 }]))).toEqual( |
137 | 73 | times(tooFar), |
138 | 74 | ); |
139 | 75 | }); |
140 | 76 |
|
141 | | - it("never mistakes a phrase's second word for its first", () => { |
142 | | - // "y" opens the second phrase but was reported just before its onset; |
143 | | - // "z" must not be dragged back over it. |
144 | | - const words = [ |
145 | | - word({ word: "x", startSec: 0.5, endSec: 1.95 }), |
146 | | - word({ word: "y", startSec: 1.95, endSec: 2.4 }), |
147 | | - word({ word: "z", startSec: 2.4, endSec: 3 }), |
148 | | - ]; |
149 | | - const speech = [ |
150 | | - { startSec: 0.5, endSec: 1 }, |
151 | | - { startSec: 2, endSec: 3 }, |
152 | | - ]; |
153 | | - expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([ |
154 | | - ["x", 0.5, 1], |
155 | | - ["y", 1.95, 2.4], |
156 | | - ["z", 2.4, 3], |
157 | | - ]); |
158 | | - }); |
159 | | - |
160 | | - it("does not take a word in the previous phrase's tail for the next one's first", () => { |
161 | | - // The helper keeps 0.1 s past each offset. "b" was reported in the first |
162 | | - // phrase's tail, and "c" still has to reach the second onset. |
| 77 | + it("gives a word to the stretch its anchor falls in, the kept tail included", () => { |
| 78 | + // "b" is anchored in the first stretch's 0.1 s tail: it closes that phrase. |
| 79 | + // "c" starts in the first stretch (the previous token's end) but is |
| 80 | + // anchored in the second, so it opens the second. |
163 | 81 | const words = [ |
164 | | - word({ word: "a", startSec: 1, endSec: 2.05 }), |
165 | | - word({ word: "b", startSec: 2.05, endSec: 3.4 }), |
166 | | - word({ word: "c", startSec: 3.4, endSec: 4 }), |
| 82 | + word("a", 1, 1.9, 1.5), |
| 83 | + word("b", 1.9, 2.05, 2.05), |
| 84 | + word("c", 2.05, 3.5, 3.2), |
| 85 | + word("d", 3.5, 4, 3.8), |
167 | 86 | ]; |
168 | 87 | const speech = [ |
169 | 88 | { startSec: 1, endSec: 2 }, |
170 | 89 | { startSec: 3, endSec: 4 }, |
171 | 90 | ]; |
172 | | - expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([ |
173 | | - ["a", 1, 2.05], |
174 | | - ["b", 2.05, 2.07], |
175 | | - ["c", 3, 4], |
176 | | - ]); |
177 | | - }); |
178 | | - |
179 | | - it("keeps a word with the stretch whisper heard it in, once the snap pulls it before the onset", () => { |
180 | | - // 80 ms between two stretches: the snap moves "b" from 2.10 to the silence |
181 | | - // at 2.07, inside what would be the first stretch's tail. It still opens |
182 | | - // the second stretch and keeps its audio. |
183 | | - const samples = audioWithSilences(6, [[2, 2.08]]); |
184 | | - const words = [ |
185 | | - word({ word: "a", startSec: 1, endSec: 2.1 }), |
186 | | - word({ word: "b", startSec: 2.1, endSec: 2.4 }), |
187 | | - word({ word: "c", startSec: 2.4, endSec: 3 }), |
188 | | - ]; |
189 | | - const speech = [ |
190 | | - { startSec: 1, endSec: 2 }, |
191 | | - { startSec: 2.08, endSec: 3 }, |
192 | | - ]; |
193 | | - expect(times(snapWordBoundariesToAudio(words, samples, speech))).toEqual([ |
194 | | - ["a", 1, 2], |
195 | | - ["b", 2.07, 2.4], |
196 | | - ["c", 2.4, 3], |
| 91 | + expect(times(anchorWordsOnSpeech(words, speech))).toEqual([ |
| 92 | + ["a", 1, 1.9], |
| 93 | + ["b", 1.9, 2], |
| 94 | + ["c", 3, 3.5], |
| 95 | + ["d", 3.5, 4], |
197 | 96 | ]); |
198 | 97 | }); |
199 | 98 | }); |
0 commit comments