Skip to content

Commit 019e198

Browse files
committed
fix(stt): start each word at the end of the token before it
whisper.cpp's t_dtw marks where a token ends, so taking a word's first-token t_dtw as its start put every word one token late (+175 ms median). The helper now starts a word at the previous token's t_dtw, ends it at its own last token's, and sends the first-token time as `anchor` to tell which stretch of speech owns it. The RMS lookback snap goes; phrase edges are anchored on the VAD in both directions. Adds the word-timing harness used to measure it (issue #948).
1 parent 2ad2293 commit 019e198

18 files changed

Lines changed: 1311 additions & 360 deletions

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,9 @@ result-*
9595
/tools/stt-eval/whispercpp-dtw-poc/fixtures/*.wav
9696
/tools/stt-eval/whispercpp-dtw-poc/results/
9797

98+
# Word-timing harness (tools/stt-eval/word-timing): TTS corpus and results.
99+
/tools/stt-eval/word-timing/data/
100+
98101
opencode.json
99102
opencode.json
100103

‎electron/native/whisper-stt/src/main.cpp‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -223,9 +223,12 @@ std::string detect_active_backend() {
223223
}
224224

225225
struct Word {
226-
double start = 0.0;
227-
double end = 0.0;
228-
double prob = 0.0;
226+
double start = 0.0;
227+
double end = 0.0;
228+
// t_dtw of the word's first token: the end of that token, so always inside
229+
// the word. Only used to tell which stretch of speech a word belongs to.
230+
double anchor = 0.0;
231+
double prob = 0.0;
229232
std::string text;
230233
};
231234

@@ -517,18 +520,28 @@ int main(int argc, char** argv) {
517520
};
518521
std::vector<Segment> segments;
519522

523+
// whisper.cpp writes a token's t_dtw when the DTW path enters the decoder
524+
// row that PREDICTS the next token (whisper_exp_compute_token_level_
525+
// timestamps_dtw, v1.9.1): it marks the END of the token, not its start.
526+
// So a word runs from the t_dtw of the text token before it to the t_dtw
527+
// of its own last token. Taking its first token's t_dtw as the start put
528+
// every word one token late (+175 ms median, tools/stt-eval/word-timing).
529+
// The previous token carries across segments; the request's very first
530+
// word has none and starts on its first token's time (the Node side
531+
// anchors it on the speech onset anyway).
532+
double prev_tok_end = -1.0;
520533
const int n_segments = input.empty() ? 0 : whisper_full_n_segments(ctx);
521534
for (int si = 0; si < n_segments; ++si) {
522535
Segment seg;
523536
seg.start = to_original_sec(whisper_full_get_segment_t0(ctx, si), kept);
524537
seg.end = to_original_sec(whisper_full_get_segment_t1(ctx, si), kept);
525538
if (const char* t = whisper_full_get_segment_text(ctx, si)) seg.text = t;
526539

527-
struct W { double t_dtw_first; double p_sum; int p_n; std::string text; };
540+
struct W { double start; double end; double anchor; double p_sum; int p_n; std::string text; };
528541
std::vector<W> word_buf;
529542
std::string cur_text;
530543
bool in_word = false;
531-
double w_first_t_dtw = 0;
544+
double w_start = 0, w_end = 0, w_anchor = 0;
532545
double w_p_sum = 0; int w_p_n = 0;
533546

534547
const int n_tokens = whisper_full_n_tokens(ctx, si);
@@ -555,30 +568,32 @@ int main(int argc, char** argv) {
555568
const double td_dtw = to_original_sec(td.t_dtw >= 0 ? td.t_dtw : 0, kept);
556569

557570
if (starts_word && in_word) {
558-
word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });
571+
word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text });
559572
w_p_sum = 0; w_p_n = 0; cur_text.clear();
560573
}
561574
if (starts_word) {
562575
in_word = true;
563-
w_first_t_dtw = td_dtw;
576+
w_start = prev_tok_end >= 0 ? prev_tok_end : td_dtw;
577+
w_anchor = td_dtw;
564578
cur_text = (!raw.empty() && raw[0] == ' ') ? raw.substr(1) : raw;
565579
} else {
566580
cur_text += raw;
567581
}
582+
w_end = td_dtw;
583+
prev_tok_end = td_dtw;
568584
w_p_sum += td.p;
569585
w_p_n += 1;
570586
}
571587
if (in_word) {
572-
word_buf.push_back({ w_first_t_dtw, w_p_sum, w_p_n, cur_text });
588+
word_buf.push_back({ w_start, w_end, w_anchor, w_p_sum, w_p_n, cur_text });
573589
}
574-
for (size_t wi = 0; wi < word_buf.size(); ++wi) {
590+
for (const W& b : word_buf) {
575591
Word w;
576-
w.start = word_buf[wi].t_dtw_first;
577-
w.end = (wi + 1 < word_buf.size())
578-
? word_buf[wi + 1].t_dtw_first
579-
: seg.end;
580-
w.prob = word_buf[wi].p_sum / std::max(1, word_buf[wi].p_n);
581-
w.text = word_buf[wi].text;
592+
w.start = b.start;
593+
w.end = b.end;
594+
w.anchor = b.anchor;
595+
w.prob = b.p_sum / std::max(1, b.p_n);
596+
w.text = b.text;
582597
seg.words.push_back(w);
583598
}
584599
segments.push_back(std::move(seg));
@@ -642,6 +657,7 @@ int main(int argc, char** argv) {
642657
{"word", w.text},
643658
{"start", w.start},
644659
{"end", w.end},
660+
{"anchor", w.anchor},
645661
{"probability", w.prob},
646662
});
647663
}
Lines changed: 53 additions & 154 deletions
Original file line numberDiff line numberDiff line change
@@ -1,199 +1,98 @@
11
import { describe, expect, it } from "vitest";
2-
import { snapWordBoundariesToAudio } from "./snapWordBoundaries";
2+
import { anchorWordsOnSpeech, type HelperWord } from "./snapWordBoundaries";
33
import type { SttWordSegment } from "./transcriptionContract";
44

5-
const SAMPLE_RATE = 16_000;
6-
7-
/** Mono 16 kHz buffer that is loud everywhere except the given silent spans. */
8-
function audioWithSilences(durationSec: number, silences: Array<[number, number]>): Float32Array {
9-
const samples = new Float32Array(Math.round(durationSec * SAMPLE_RATE));
10-
for (let i = 0; i < samples.length; i++) {
11-
const t = i / SAMPLE_RATE;
12-
const silent = silences.some(([from, to]) => t >= from && t < to);
13-
// Alternating ±0.5 gives a flat, non-zero RMS without needing a real tone.
14-
samples[i] = silent ? 0 : i % 2 === 0 ? 0.5 : -0.5;
15-
}
16-
return samples;
17-
}
18-
19-
const word = (w: Partial<SttWordSegment> = {}): SttWordSegment => ({
20-
word: "w",
21-
startSec: 0,
22-
endSec: 0.1,
23-
...w,
5+
/** A helper word; its anchor defaults to its start, as for a request's first word. */
6+
const word = (
7+
text: string,
8+
startSec: number,
9+
endSec: number,
10+
anchorSec = startSec,
11+
): HelperWord => ({
12+
word: text,
13+
startSec,
14+
endSec,
15+
anchorSec,
2416
});
25-
26-
describe("snapWordBoundariesToAudio", () => {
27-
it("pulls a late boundary back into the silence that precedes it", () => {
28-
// Speech stops at 1.0 and resumes at 1.2; whisper reports the next word
29-
// starting at 1.3 — 100 ms after the audio actually resumed.
30-
const samples = audioWithSilences(3, [[1.0, 1.2]]);
31-
const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.8 })], samples);
32-
expect(snapped.startSec).toBeGreaterThanOrEqual(1.0);
33-
expect(snapped.startSec).toBeLessThan(1.2);
34-
});
35-
36-
it("leaves a boundary alone when nothing quieter precedes it", () => {
37-
// A word ending a phrase: whisper is already right, the frames before the
38-
// boundary are all speech, so the quietest frame in the window is the
39-
// boundary itself and it must not drift.
40-
const samples = audioWithSilences(3, [[1.5, 2.0]]);
41-
const [snapped] = snapWordBoundariesToAudio([word({ startSec: 1.0, endSec: 1.5 })], samples);
42-
expect(snapped.endSec).toBeCloseTo(1.5, 2);
43-
});
44-
45-
it("never moves a boundary more than the lookback window", () => {
46-
const samples = audioWithSilences(3, [[0.0, 1.0]]);
47-
const [snapped] = snapWordBoundariesToAudio([word({ startSec: 2.0, endSec: 2.5 })], samples);
48-
expect(snapped.startSec).toBeGreaterThanOrEqual(2.0 - 0.15);
49-
});
50-
51-
it("keeps a boundary shared by two words shared", () => {
52-
const samples = audioWithSilences(3, [[1.0, 1.2]]);
53-
const [first, second] = snapWordBoundariesToAudio(
54-
[word({ startSec: 0.5, endSec: 1.3 }), word({ startSec: 1.3, endSec: 1.8 })],
55-
samples,
56-
);
57-
expect(first.endSec).toBeCloseTo(second.startSec, 6);
58-
});
59-
60-
it("leaves boundaries that fall outside the decoded audio alone", () => {
61-
// Clamping these into range would collapse every boundary onto the end of
62-
// the buffer instead of leaving the unmeasurable ones untouched.
63-
const samples = audioWithSilences(0.1, []);
64-
const words = [word({ startSec: 5.51, endSec: 6.85 })];
65-
expect(snapWordBoundariesToAudio(words, samples)).toEqual(words);
66-
});
67-
68-
it("keeps degenerate words non-empty and passes words through without audio", () => {
69-
const samples = audioWithSilences(3, [[1.0, 1.2]]);
70-
const [degenerate] = snapWordBoundariesToAudio([word({ startSec: 1.3, endSec: 1.3 })], samples);
71-
expect(degenerate.endSec).toBeGreaterThan(degenerate.startSec);
72-
73-
const untouched = [word({ startSec: 1.3, endSec: 1.8 })];
74-
expect(snapWordBoundariesToAudio(untouched, new Float32Array(0))).toEqual(untouched);
17+
const ms = (sec: number) => Math.round(sec * 1000) / 1000;
18+
const times = (words: SttWordSegment[]) => words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);
19+
20+
describe("anchorWordsOnSpeech", () => {
21+
it("returns the helper's times without speech intervals, minus the anchor", () => {
22+
const out = anchorWordsOnSpeech([word("a", 1, 1.4, 1.2), word("b", 1.4, 1.4, 1.4)]);
23+
expect(out).toEqual([
24+
{ word: "a", startSec: 1, endSec: 1.4 },
25+
{ word: "b", startSec: 1.4, endSec: 1.42 },
26+
]);
7527
});
76-
});
7728

78-
describe("snapWordBoundariesToAudio with speech intervals", () => {
79-
// Loud and flat everywhere, so the RMS snap moves nothing and only the
80-
// anchoring on `speech` is under test.
81-
const flat = audioWithSilences(6, []);
82-
const ms = (sec: number) => Math.round(sec * 1000) / 1000;
83-
const times = (words: SttWordSegment[]) =>
84-
words.map((w) => [w.word, ms(w.startSec), ms(w.endSec)]);
85-
86-
it("puts each phrase's first word on its onset and its closing punctuation on its end", () => {
87-
// The shape of a real French take: whisper put "Salut" 0.58 s and "Bah"
88-
// 0.25 s after the speech started, ran "Salut" on through the pause, and
89-
// dropped the "!" closing the first phrase just after the second began.
29+
it("puts a phrase's first word on its onset whether DTW put it late or early", () => {
30+
// "Salut" opens the request, so it has no previous token and starts late,
31+
// on its own first token. "Bah" starts where the previous token ended, at
32+
// the end of the first phrase: in the pause, before its own speech.
9033
const words = [
91-
word({ word: "Salut", startSec: 2.15, endSec: 3.37 }),
92-
word({ word: "!", startSec: 3.37, endSec: 3.39 }),
93-
word({ word: "Bah", startSec: 3.61, endSec: 4.01 }),
94-
word({ word: "voilà", startSec: 4.01, endSec: 4.35 }),
34+
word("Salut", 2.15, 2.5),
35+
word("!", 2.5, 2.6, 2.6),
36+
word("Bah", 2.6, 3.8, 3.61),
37+
word("voilà", 3.8, 4.35, 4.01),
9538
];
9639
const speech = [
9740
{ startSec: 1.57, endSec: 2.56 },
9841
{ startSec: 3.36, endSec: 4.35 },
9942
];
100-
expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
43+
expect(times(anchorWordsOnSpeech(words, speech))).toEqual([
10144
["Salut", 1.57, 2.56],
10245
["!", 2.56, 2.56],
103-
["Bah", 3.36, 4.01],
104-
["voilà", 4.01, 4.35],
46+
["Bah", 3.36, 3.8],
47+
["voilà", 3.8, 4.35],
10548
]);
10649
});
10750

10851
it("ends each phrase's last word where its speech stops, stretched or cut back", () => {
109-
// The end of the same take: "tic," ran on into the pause, whisper closed
110-
// the last segment at 2.79 while "tac" ran to 3.04, and dropped the "!" on
111-
// the word itself.
11252
const words = [
113-
word({ word: "tic,", startSec: 1.95, endSec: 2.59 }),
114-
word({ word: "tac", startSec: 2.59, endSec: 2.79 }),
115-
word({ word: "!", startSec: 2.62, endSec: 2.79 }),
53+
word("tic,", 1.95, 2.59),
54+
word("tac", 2.59, 2.79, 2.7),
55+
word("!", 2.79, 2.9, 2.9),
11656
];
11757
const speech = [
11858
{ startSec: 1.95, endSec: 2.37 },
11959
{ startSec: 2.59, endSec: 3.04 },
12060
];
121-
expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
61+
expect(times(anchorWordsOnSpeech(words, speech))).toEqual([
12262
["tic,", 1.95, 2.37],
12363
["tac", 2.59, 3.04],
12464
["!", 3.04, 3.04],
12565
]);
12666
});
12767

128-
it("leaves a phrase alone when its words are on time, or too far off to be its edges", () => {
129-
const onTime = [word({ startSec: 0.95, endSec: 3 })];
130-
expect(times(snapWordBoundariesToAudio(onTime, flat, [{ startSec: 1, endSec: 3 }]))).toEqual(
131-
times(onTime),
132-
);
68+
it("leaves an edge alone when the word is too far past it to be that edge", () => {
13369
// 1.2 s after the onset and 1.1 s before the offset: more likely a neighbour
13470
// of words whisper dropped than the phrase's own edges.
135-
const tooFar = [word({ startSec: 2.2, endSec: 2.5 })];
136-
expect(times(snapWordBoundariesToAudio(tooFar, flat, [{ startSec: 1, endSec: 3.6 }]))).toEqual(
71+
const tooFar = [word("w", 2.2, 2.5)];
72+
expect(times(anchorWordsOnSpeech(tooFar, [{ startSec: 1, endSec: 3.6 }]))).toEqual(
13773
times(tooFar),
13874
);
13975
});
14076

141-
it("never mistakes a phrase's second word for its first", () => {
142-
// "y" opens the second phrase but was reported just before its onset;
143-
// "z" must not be dragged back over it.
144-
const words = [
145-
word({ word: "x", startSec: 0.5, endSec: 1.95 }),
146-
word({ word: "y", startSec: 1.95, endSec: 2.4 }),
147-
word({ word: "z", startSec: 2.4, endSec: 3 }),
148-
];
149-
const speech = [
150-
{ startSec: 0.5, endSec: 1 },
151-
{ startSec: 2, endSec: 3 },
152-
];
153-
expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
154-
["x", 0.5, 1],
155-
["y", 1.95, 2.4],
156-
["z", 2.4, 3],
157-
]);
158-
});
159-
160-
it("does not take a word in the previous phrase's tail for the next one's first", () => {
161-
// The helper keeps 0.1 s past each offset. "b" was reported in the first
162-
// phrase's tail, and "c" still has to reach the second onset.
77+
it("gives a word to the stretch its anchor falls in, the kept tail included", () => {
78+
// "b" is anchored in the first stretch's 0.1 s tail: it closes that phrase.
79+
// "c" starts in the first stretch (the previous token's end) but is
80+
// anchored in the second, so it opens the second.
16381
const words = [
164-
word({ word: "a", startSec: 1, endSec: 2.05 }),
165-
word({ word: "b", startSec: 2.05, endSec: 3.4 }),
166-
word({ word: "c", startSec: 3.4, endSec: 4 }),
82+
word("a", 1, 1.9, 1.5),
83+
word("b", 1.9, 2.05, 2.05),
84+
word("c", 2.05, 3.5, 3.2),
85+
word("d", 3.5, 4, 3.8),
16786
];
16887
const speech = [
16988
{ startSec: 1, endSec: 2 },
17089
{ startSec: 3, endSec: 4 },
17190
];
172-
expect(times(snapWordBoundariesToAudio(words, flat, speech))).toEqual([
173-
["a", 1, 2.05],
174-
["b", 2.05, 2.07],
175-
["c", 3, 4],
176-
]);
177-
});
178-
179-
it("keeps a word with the stretch whisper heard it in, once the snap pulls it before the onset", () => {
180-
// 80 ms between two stretches: the snap moves "b" from 2.10 to the silence
181-
// at 2.07, inside what would be the first stretch's tail. It still opens
182-
// the second stretch and keeps its audio.
183-
const samples = audioWithSilences(6, [[2, 2.08]]);
184-
const words = [
185-
word({ word: "a", startSec: 1, endSec: 2.1 }),
186-
word({ word: "b", startSec: 2.1, endSec: 2.4 }),
187-
word({ word: "c", startSec: 2.4, endSec: 3 }),
188-
];
189-
const speech = [
190-
{ startSec: 1, endSec: 2 },
191-
{ startSec: 2.08, endSec: 3 },
192-
];
193-
expect(times(snapWordBoundariesToAudio(words, samples, speech))).toEqual([
194-
["a", 1, 2],
195-
["b", 2.07, 2.4],
196-
["c", 2.4, 3],
91+
expect(times(anchorWordsOnSpeech(words, speech))).toEqual([
92+
["a", 1, 1.9],
93+
["b", 1.9, 2],
94+
["c", 3, 3.5],
95+
["d", 3.5, 4],
19796
]);
19897
});
19998
});

0 commit comments

Comments
 (0)