diff --git a/audio/audio.go b/audio/audio.go index 2d83e0c..3483150 100644 --- a/audio/audio.go +++ b/audio/audio.go @@ -145,6 +145,9 @@ type DeviceInfo struct { type Context interface { Devices() ([]DeviceInfo, error) + // NewCapture does not fail because no input device can be opened yet (no + // mic attached): the device is then opened at Start, and Start reports + // the error if there is still none. NewCapture(device *DeviceInfo, config CaptureConfig) (CaptureDevice, error) Close() } diff --git a/audio/capture_darwin.go b/audio/capture_darwin.go index 36242fc..0eaff6e 100644 --- a/audio/capture_darwin.go +++ b/audio/capture_darwin.go @@ -79,9 +79,10 @@ func (m *malgoContext) NewCapture(device *DeviceInfo, config CaptureConfig) (Cap err := c.initDevice() deviceMu.Unlock() if err != nil { - return nil, err + // No usable input right now — e.g. a Mac mini or Studio with no mic + // attached at login. Not fatal: Start builds the device on first use. + log.Warnf("capture device init failed (%v); retrying at the first recording", err) } - return c, nil } diff --git a/docs/design-notes.md b/docs/design-notes.md index 3220993..79e15d0 100644 --- a/docs/design-notes.md +++ b/docs/design-notes.md @@ -2203,3 +2203,50 @@ clip" in the Qwen entry above, may be English speech. whisper `-l en`, Nemotron and NeMo-Speech give the same coherent English, and whisper-auto's Turkish loops. If so, Qwen did not translate; whisper-auto did. Listen to the clip before editing that entry. + +## The silence warning clears on a 1 s window, not the 8 s one that raised it (2026-10-20) + +The warning fires when under 10% of the last 8 s (80 × 100 ms ticks) holds +speech. It used to clear when that same 8 s window reached 25%. At the moment +it fires the window is near-empty, so clearing needed 20 speech ticks — at +least 2 s of unbroken speech, about 3 s with normal pauses. The rule dated from +February, but it only became noticeable once the overlay (2026-07-25) put the +"Silent" label in front of the user while its level bars, gated on a 250 ms +VAD hold, visibly reacted at once. + +Now: clear when 5 of the last 10 ticks hold speech (0.5 s minimum). A lone VAD +false positive is 1 of 10 and does not clear. On a clear the 8 s warn window +restarts — otherwise the few speech ticks that cleared it leave the 8 s ratio +under 10% and the warning re-fires (with its error beep) on the next tick. + +## A stalled stream fails the session; it never blocks capture or drops chunks (2026-10-20) + +`streamSession.Feed` runs on the capture callback, under the recording lock. +It used to hand chunks to the sender with a blocking channel send. On a network +stall the websocket write blocked (the stream context has no deadline), the +128-chunk queue filled, and Feed blocked for good: `isRecording` stayed true and +the hotkey was dead until restart. + +Now each write has its own timeout, Feed never blocks, and a full queue fails the +stream. Close aborts a failed stream at once and gives a healthy one a bounded +time to flush. Rejected: dropping chunks on a full queue and carrying on. That +would paste a transcript with silent holes in it. A failed stream instead takes +the existing error path, and the full PCM (kept regardless of stream health) is +auto-saved to `samples/` for a retry. + +## A local session waits for the model in Close, not NewSession (2026-10-20) + +`localProvider.NewSession` used to wait out the model load (read + warm-up) +before it returned. `handleRecording` calls it before `rec.Start()`, so while a +model loaded at startup or after a switch, the start beep and overlay said +"Recording" but no audio was captured, and the speech was lost. The comment +saying "recording overlaps the load" was wrong. + +Now NewSession returns at once and the session resolves the engine in Close. +Only a model file missing from disk still fails in NewSession, because no +recording could be transcribed with it. Rejected: starting a `go p.load()` from +NewSession to also overlap loads that no background warm started (after a +failed load, or after `Close`). That goroutine can outlive the session and get +`loadMu` after a provider switch has called `Close`. It would then load the old +provider's model again, and nothing would ever free it. Those rare cases load +in Close instead. diff --git a/hotkey/hotkey.go b/hotkey/hotkey.go index 3bd4914..ee80c1e 100644 --- a/hotkey/hotkey.go +++ b/hotkey/hotkey.go @@ -47,7 +47,7 @@ func AwaitRecord(hk Hotkey, longPress, timeout time.Duration) (stop <-chan struc } stopCh := make(chan struct{}) go func() { - WaitStop(hk, longPress, nil) + WaitStop(hk, longPress, nil, nil) close(stopCh) }() return stopCh, true @@ -60,11 +60,16 @@ func AwaitRecord(hk Hotkey, longPress, timeout time.Duration) (stop <-chan struc // (the app uses it to arm silence auto-close). Shared by the main record loop // and `zee doctor`, so both behave identically. // +// ended (optional; nil never fires) closes when the recording stops some other +// way — silence auto-close, the tray, an error. Toggle mode then returns at +// once instead of waiting for a tap: otherwise the user's next press, meant to +// start a new recording, would be eaten as the stop tap of the finished one. +// // It returns how long keydown→keyup took as observed (downToUp) and whether it // resolved to toggle mode — surfaced for diagnostics: a downToUp far above // longPress on a quick tap means the keyup event arrived late (a stalled run // loop), which is what makes a tap misfire as a hold. -func WaitStop(hk Hotkey, longPress time.Duration, onToggle func()) (downToUp time.Duration, toggled bool) { +func WaitStop(hk Hotkey, longPress time.Duration, ended <-chan struct{}, onToggle func()) (downToUp time.Duration, toggled bool) { start := time.Now() timer := time.NewTimer(longPress) defer timer.Stop() @@ -77,8 +82,11 @@ func WaitStop(hk Hotkey, longPress time.Duration, onToggle func()) (downToUp tim if onToggle != nil { onToggle() } - <-hk.Keydown() - <-hk.Keyup() + select { + case <-hk.Keydown(): + <-hk.Keyup() + case <-ended: + } return downToUp, true } } diff --git a/hotkey/hotkey_test.go b/hotkey/hotkey_test.go index bb5281d..7a39b70 100644 --- a/hotkey/hotkey_test.go +++ b/hotkey/hotkey_test.go @@ -3,6 +3,7 @@ package hotkey import ( "errors" "testing" + "time" ) func TestDefaultComboValid(t *testing.T) { @@ -66,3 +67,48 @@ func TestComboEqual(t *testing.T) { t.Fatal("different mods should not be equal") } } + +// TestWaitStopTogglePress: in toggle mode the next tap stops the recording. +func TestWaitStopTogglePress(t *testing.T) { + hk := NewFake() + done := make(chan bool, 1) + go func() { + _, toggled := WaitStop(hk, time.Hour, nil, nil) + done <- toggled + }() + hk.SimKeyup() // short tap → toggle mode + hk.SimKeydown() + hk.SimKeyup() + select { + case toggled := <-done: + if !toggled { + t.Fatal("a short tap should resolve to toggle mode") + } + case <-time.After(time.Second): + t.Fatal("WaitStop did not return on the stop tap") + } +} + +// TestWaitStopToggleEndedElsewhere: a toggle recording that ends another way +// (silence auto-close, tray stop, error) must release WaitStop without +// consuming a key event — the next press belongs to a new recording. +func TestWaitStopToggleEndedElsewhere(t *testing.T) { + hk := NewFake() + ended := make(chan struct{}) + done := make(chan struct{}) + go func() { + WaitStop(hk, time.Hour, ended, nil) + close(done) + }() + hk.SimKeyup() // short tap → toggle mode + close(ended) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("WaitStop still waiting for a stop tap after the recording ended") + } + hk.SimKeydown() + if len(hk.Keydown()) != 1 { + t.Fatal("the next press was consumed by the finished recording") + } +} diff --git a/main.go b/main.go index 9fc8d1f..c1b92a3 100644 --- a/main.go +++ b/main.go @@ -96,7 +96,8 @@ func modelSupportsStream(tr transcriber.Transcriber) bool { type recSession struct { Stop <-chan struct{} SilenceClose *atomic.Bool - PressedAt time.Time // when the press was accepted; drives the reflex-latency metric + PressedAt time.Time // when the press was accepted; drives the reflex-latency metric + Ended chan struct{} // closed by recordSessions when the recording phase ends, however it ends } type recordingConfig struct { @@ -464,22 +465,23 @@ func run() { // then). preferredDevice := *deviceFlag tray.SetBTCheck(audio.IsBluetooth) - if devices, err := ctx.Devices(); err == nil && len(devices) > 0 { - tray.SetDevices(deviceNames(devices), preferredDevice, func(name string) { - if guardBusy("Can't change the microphone while recording or transcribing.") { - return - } - captureMu.Lock() - preferredDevice = name - captureMu.Unlock() - config.Update(func(s *config.Settings) { s.Device = name }) - if name == "" { - applyDeviceSwitch(ctx, captureConfig, &captureDevice, &selectedDevice, nil) - } else { - switchDeviceByName(ctx, captureConfig, &captureDevice, &selectedDevice, name) - } - }) - } + // Registered even with no mic attached yet: the device watcher fills the + // menu in when one appears, and its items need this callback to work. + attached, _ := ctx.Devices() + tray.SetDevices(deviceNames(attached), preferredDevice, func(name string) { + if guardBusy("Can't change the microphone while recording or transcribing.") { + return + } + captureMu.Lock() + preferredDevice = name + captureMu.Unlock() + config.Update(func(s *config.Settings) { s.Device = name }) + if name == "" { + applyDeviceSwitch(ctx, captureConfig, &captureDevice, &selectedDevice, nil) + } else { + switchDeviceByName(ctx, captureConfig, &captureDevice, &selectedDevice, name) + } + }) tray.SetAutoPaste(autoPaste) var trayModels []tray.Model @@ -694,6 +696,12 @@ func run() { if slices.Equal(last, names) { continue } + // A switch closes the live capture device, so it waits for the + // cycle to end, like a tray switch. last is left stale on purpose: + // the first tick after the cycle sees the change and applies it. + if isRecording.Load() { + continue + } last = names // Snapshot the device state under captureMu, then release before the // switch calls (which re-lock it). preferredDevice is also written by @@ -961,10 +969,10 @@ func denyBusy(warning string) { // tryStartSession enqueues a fresh recording session unless a cycle is already // active (recording OR still transcribing) — in which case guardBusy denies it. -// Returns the SilenceClose handle when it started a session, nil when it denied. +// Returns the session it started, nil when it denied. // The hotkey and the tray "Start Recording" button both funnel through here, so // neither can queue an unattended recording that fires the instant inference ends. -func tryStartSession(sessions chan<- recSession) *atomic.Bool { +func tryStartSession(sessions chan<- recSession) *recSession { // Claiming the cycle IS the guard: a plain check-then-send is not atomic // (isRecording only went true once recordSessions picked the session up), so // a hotkey press and a tray click landing together could both pass and both @@ -984,10 +992,10 @@ func tryStartSession(sessions chan<- recSession) *atomic.Bool { if t, ok := tr.(transcriber.Toucher); ok { go t.Touch() } - sc := &atomic.Bool{} audio.PlayStart() // reflexive: sound the press now, not after the record loop spins up (playOne is non-blocking) - sessions <- recSession{Stop: resetStop(), SilenceClose: sc, PressedAt: time.Now()} - return sc + sess := recSession{Stop: resetStop(), SilenceClose: &atomic.Bool{}, PressedAt: time.Now(), Ended: make(chan struct{})} + sessions <- sess + return &sess } // recordSessions is the core record→transcribe loop, shared by the live app and @@ -1009,6 +1017,7 @@ func recordSessions(getCapture func() audio.CaptureDevice, sessions <-chan recSe overlay.Show() // every path — hotkey, toggle, tray — funnels through here done, err := handleRecording(capture, sess) + close(sess.Ended) if err != nil { log.Errorf("recording error: %v", err) tray.SetError(err.Error()) @@ -1041,21 +1050,27 @@ func listenHotkey(hk hotkey.Hotkey, longPress time.Duration, sessions chan<- rec } continue } - sc := tryStartSession(sessions) - if sc == nil { + sess := tryStartSession(sessions) + if sess == nil { log.HotkeyPress(0, "denied") <-hk.Keyup() // denied (a cycle began between the guard above and here) continue } // Shared press semantics (hold vs toggle) — see hotkey.WaitStop. Toggle // mode arms silence auto-close the moment it is entered. - downToUp, toggled := hotkey.WaitStop(hk, longPress, func() { sc.Store(true) }) + downToUp, toggled := hotkey.WaitStop(hk, longPress, sess.Ended, func() { sess.SilenceClose.Store(true) }) mode := "hold" if toggled { mode = "toggle" } log.HotkeyPress(float64(downToUp.Milliseconds()), mode) - requestStop() + // A recording that already ended needs no stop — and stopCh may belong + // to a newer session by now. + select { + case <-sess.Ended: + default: + requestStop() + } } } diff --git a/main_test.go b/main_test.go index 574fba0..bb5c0fc 100644 --- a/main_test.go +++ b/main_test.go @@ -49,7 +49,7 @@ func TestRecordSessionsBlocksDuringInference(t *testing.T) { // Start a recording, let it capture briefly, then stop it (as a keyup would) // so the 800ms "inference" begins. - sessions <- recSession{Stop: resetStop(), SilenceClose: &atomic.Bool{}} + sessions <- recSession{Stop: resetStop(), SilenceClose: &atomic.Bool{}, Ended: make(chan struct{})} time.Sleep(150 * time.Millisecond) requestStop() @@ -120,7 +120,7 @@ func TestRecordSessionsPicksUpDeviceSwitch(t *testing.T) { runSession := func() { want := atomic.LoadInt32(&cycles) + 1 - sessions <- recSession{Stop: resetStop(), SilenceClose: &atomic.Bool{}} + sessions <- recSession{Stop: resetStop(), SilenceClose: &atomic.Bool{}, Ended: make(chan struct{})} time.Sleep(120 * time.Millisecond) requestStop() deadline := time.Now().Add(3 * time.Second) @@ -151,6 +151,10 @@ func TestRecordSessionsPicksUpDeviceSwitch(t *testing.T) { } } +// TestListenHotkey_TrayStopNoStaleSignal: a toggle recording ended from the +// tray must leave the hotkey ready — the very next tap starts a new session +// (it used to be eaten as the stop tap of the finished one), and that session +// is not stopped by a stale signal from the old one. func TestListenHotkey_TrayStopNoStaleSignal(t *testing.T) { hk := hotkey.NewFake() sessions := make(chan recSession, 3) @@ -162,28 +166,29 @@ func TestListenHotkey_TrayStopNoStaleSignal(t *testing.T) { isRecording.Store(false) hk.SimKeydown() sess1 := <-sessions - isRecording.Store(true) time.Sleep(10 * time.Millisecond) hk.SimKeyup() - // 2. Tray stop ends the recording externally + // 2. Tray stop ends the recording externally; recordSessions closes Ended + // and releases the cycle. requestStop() - isRecording.Store(false) select { case <-sess1.Stop: case <-time.After(time.Second): t.Fatal("tray stop did not end session") } + close(sess1.Ended) + isRecording.Store(false) + time.Sleep(20 * time.Millisecond) // let listenHotkey leave WaitStop - // 3. Still in toggleRecording — this tap transitions back to idle - hk.SimKeydown() - hk.SimKeyup() - time.Sleep(20 * time.Millisecond) // let state machine settle - - // 4. New tap should start a session that stays alive + // 3. The next tap starts a new session at once hk.SimKeydown() - sess2 := <-sessions - isRecording.Store(true) + var sess2 recSession + select { + case sess2 = <-sessions: + case <-time.After(time.Second): + t.Fatal("first tap after a tray stop was swallowed instead of starting a recording") + } time.Sleep(10 * time.Millisecond) hk.SimKeyup() @@ -191,8 +196,9 @@ func TestListenHotkey_TrayStopNoStaleSignal(t *testing.T) { case <-sess2.Stop: t.Fatal("new session immediately stopped by stale stopCh signal") case <-time.After(300 * time.Millisecond): - // session stayed alive — fix works + // session stayed alive } + isRecording.Store(false) } func TestListenHotkey_StopsTrayRecording(t *testing.T) { diff --git a/silence.go b/silence.go index 56d02ea..3815880 100644 --- a/silence.go +++ b/silence.go @@ -10,7 +10,13 @@ const ( silenceWarnEvery = 8 * time.Second silenceAutoCloseDur = 30 * time.Second speechMinRatio = 0.10 - speechClearRatio = 0.25 // higher threshold to clear warning (hysteresis) + + // The warning clears on a short window, not the 8s one that raised it: + // that window is nearly empty when the warning fires, so clearing on it + // kept the warning up for seconds after speech resumed. Half of the last + // second still ignores a lone VAD false positive. + silenceClearWindow = 1 * time.Second + speechClearRatio = 0.5 ) type SilenceEvent int @@ -25,6 +31,7 @@ const ( type silenceMonitor struct { warnAt int + clearAt int windowSz int silenceClose *atomic.Bool @@ -34,6 +41,7 @@ type silenceMonitor struct { speechCount int warned bool lastBeep int + armedAt int // tick the warn window starts from: session start or last clear } func newSilenceMonitor(silenceClose *atomic.Bool) *silenceMonitor { @@ -41,6 +49,7 @@ func newSilenceMonitor(silenceClose *atomic.Bool) *silenceMonitor { windowSz := int(silenceAutoCloseDur / tickInterval) return &silenceMonitor{ warnAt: warnAt, + clearAt: int(silenceClearWindow / tickInterval), windowSz: windowSz, silenceClose: silenceClose, window: make([]bool, windowSz), @@ -74,17 +83,18 @@ func (m *silenceMonitor) Tick(hasSpeech bool) SilenceEvent { } m.ticks++ - r := m.ratio(m.warnAt) - - // Warn: 8s window below threshold - if m.ticks >= m.warnAt && r < speechMinRatio && !m.warned { + // Warn: 8s window below threshold. The window restarts on a clear — + // the speech that cleared it is too little to keep a full 8s window above + // threshold, and the warning would fire again on the next tick. + if !m.warned && m.ticks-m.armedAt >= m.warnAt && m.ratio(m.warnAt) < speechMinRatio { m.warned = true m.lastBeep = m.ticks return SilenceWarn } - // Clear: speech ratio above clear threshold - if m.warned && r >= speechClearRatio { + // Clear: speech in the last second + if m.warned && m.ratio(m.clearAt) >= speechClearRatio { m.warned = false + m.armedAt = m.ticks return SilenceWarnClear } diff --git a/silence_test.go b/silence_test.go index 5f22a8a..2f5bf16 100644 --- a/silence_test.go +++ b/silence_test.go @@ -41,14 +41,35 @@ func TestSilenceWarnClearsOnSpeech(t *testing.T) { m := pttMonitor() feedN(m, false, 80) // triggers warn - // Sustained speech clears warning (need 25% of 80-tick window) - for i := 0; i < 80; i++ { - ev := m.Tick(true) - if ev == SilenceWarnClear { + // Half a second of speech clears it (5 of the last 10 ticks) + for i := 1; i <= 5; i++ { + if ev := m.Tick(true); ev == SilenceWarnClear { + if i != 5 { + t.Fatalf("cleared after %d speech ticks, want 5", i) + } + return + } + } + t.Fatal("expected SilenceWarnClear after 5 speech ticks") +} + +func TestNoRewarnRightAfterClear(t *testing.T) { + m := pttMonitor() + feedN(m, false, 80) // warn + feedN(m, true, 5) // clear + + // The 8s window still holds mostly silence; it must not warn again until + // a full window of silence has passed since the clear. + for i := 1; i <= 80; i++ { + ev := m.Tick(false) + if ev == SilenceWarn { + if i != 80 { + t.Fatalf("re-warned %d ticks after clear, want 80", i) + } return } } - t.Fatal("expected SilenceWarnClear after speech") + t.Fatal("expected SilenceWarn after 8s of silence following a clear") } func TestNoWarnDuringSpeech(t *testing.T) { diff --git a/transcriber/deepgram_stream.go b/transcriber/deepgram_stream.go index 3da7847..9018520 100644 --- a/transcriber/deepgram_stream.go +++ b/transcriber/deepgram_stream.go @@ -117,12 +117,20 @@ func (d *Deepgram) startStream(ctx context.Context, cfg streamSessionConfig) (ra } func (s *deepgramStreamSession) Send(pcm []byte) error { - return s.conn.Write(s.ctx, websocket.MessageBinary, pcm) + return s.write(websocket.MessageBinary, pcm) } func (s *deepgramStreamSession) CloseSend() error { - msg := []byte(`{"type":"Finalize"}`) - return s.conn.Write(s.ctx, websocket.MessageText, msg) + return s.write(websocket.MessageText, []byte(`{"type":"Finalize"}`)) +} + +// write bounds each write on its own: the stream ctx has no deadline (it lives +// as long as the session), so a stalled network would block a bare write +// forever. A timed-out write closes the connection, failing the stream. +func (s *deepgramStreamSession) write(typ websocket.MessageType, p []byte) error { + ctx, cancel := context.WithTimeout(s.ctx, streamSendTimeout) + defer cancel() + return s.conn.Write(ctx, typ, p) } func (s *deepgramStreamSession) Recv() (streamUpdate, error) { diff --git a/transcriber/local.go b/transcriber/local.go index 0b9820b..f12bd2c 100644 --- a/transcriber/local.go +++ b/transcriber/local.go @@ -50,8 +50,9 @@ type localProvider struct { // newLocalProvider builds a provider around one engine and warms its default // model in the background so construction never blocks startup. The load holds // only loadMu (not mu) while it reads the model, so metadata accessors stay -// responsive; the first NewSession waits for it via load()'s idempotent fast -// path. A missing or failed model surfaces at NewSession as an error. +// responsive; the first session waits for it in Close via load()'s idempotent +// fast path. A model that is not on disk surfaces at NewSession as an error, a +// failed load at the session's Close. func newLocalProvider(name, defaultID, defaultLang string, hints bool, open func(localmodel.Model) (localEngine, error), langsFor func(localmodel.Model) []Language) *localProvider { @@ -135,7 +136,7 @@ func modelInfos(models []localmodel.Model, langsFor func(localmodel.Model) []Lan // load brings the engine in line with the desired modelID, freeing the previous // one. It's serialized by loadMu and idempotent: if the desired model is already -// loaded it returns at once, so NewSession can call it to *wait out* an in-flight +// loaded it returns at once, so a session can call it to *wait out* an in-flight // background load without triggering a reload. The slow model read runs with mu // released, so GetModel/SetLanguage/etc. never block on it. func (p *localProvider) load() { @@ -156,30 +157,15 @@ func (p *localProvider) load() { old.Close() } - var ( - eng localEngine - err error - ) - // A model ID belonging to another engine is not an error, it is the normal - // state right after a provider switch: the ID persisted in config.json is - // whatever the PREVIOUS provider had selected, and it arrives here via - // SetModel before the tray ever offers this engine's own list. Fall back to - // this engine's default instead of failing every recording until the user - // happens to open the model menu. Falling back is safe because the default - // is by construction the right engine's file — what must never happen is - // loading another engine's weights, which is a hang, not an error. - if m, ok := localmodel.ByID(want); !ok || m.Engine != p.name { - want = p.defaultID + m, err := p.resolve(want) + if m.ID != "" && m.ID != want { // fell back to the default, see resolve + want = m.ID p.mu.Lock() p.modelID = want p.mu.Unlock() } - - if m, ok := localmodel.ByID(want); !ok || m.Engine != p.name { - err = fmt.Errorf("unknown %s model %q", p.name, want) - } else if !localmodel.Present(m) { - err = fmt.Errorf("model %q not downloaded", m.Label) - } else { + var eng localEngine + if err == nil { eng, err = p.open(m) // slow; mu released } @@ -188,6 +174,30 @@ func (p *localProvider) load() { p.mu.Unlock() } +// resolve maps a desired model ID to the file this engine loads, and fails if +// that file is not on disk. +// +// A model ID belonging to another engine is not an error, it is the normal +// state right after a provider switch: the ID persisted in config.json is +// whatever the PREVIOUS provider had selected, and it arrives here via +// SetModel before the tray ever offers this engine's own list. Fall back to +// this engine's default instead of failing every recording until the user +// happens to open the model menu. Falling back is safe because the default +// is by construction the right engine's file — what must never happen is +// loading another engine's weights, which is a hang, not an error. +func (p *localProvider) resolve(want string) (localmodel.Model, error) { + m, ok := localmodel.ByID(want) + if !ok || m.Engine != p.name { + if m, ok = localmodel.ByID(p.defaultID); !ok || m.Engine != p.name { + return localmodel.Model{}, fmt.Errorf("unknown %s model %q", p.name, p.defaultID) + } + } + if !localmodel.Present(m) { + return m, fmt.Errorf("model %q not downloaded", m.Label) + } + return m, nil +} + // IsLocal reports whether tr is an on-device provider. Local decode has no // streaming and no audio encoding, so the UI greys those out. Hints are a // per-engine capability, not a local/cloud one — ask SupportsHints. @@ -264,9 +274,9 @@ func (p *localProvider) GetModel() string { p.mu.Lock(); defer p.mu.Unlock(); re // SetModel records the desired model and warms it in the background, so a tray // switch returns immediately instead of blocking the UI on the model load. The -// next NewSession waits for the load (the record→inference guard keeps switches -// out of an active cycle, and recording overlaps the load, hiding its latency); -// a load failure surfaces there. +// next session waits for the load only in Close, so recording overlaps it and +// hides its latency (the record→inference guard keeps switches out of an +// active cycle); a load failure surfaces there. func (p *localProvider) SetModel(id string) { p.mu.Lock() same := id == p.modelID @@ -307,19 +317,32 @@ func (p *localProvider) Transcribe(audioData []byte, format, lang, hints string) return res, nil } +// NewSession returns at once, even while the model is still loading: capture +// must start on the keypress, not after the load. The session waits for the +// engine in Close. Only a model file that is missing fails here, since no +// recording could ever be transcribed with it. func (p *localProvider) NewSession(_ context.Context, cfg SessionConfig) (Session, error) { if cfg.Stream { return nil, fmt.Errorf("%s does not support streaming", p.name) } - p.mu.Lock() - ready := p.engine != nil && p.loadedID == p.modelID - p.mu.Unlock() - if !ready { - p.load() // waits out an in-flight background load, or loads now + if _, err := p.resolve(p.GetModel()); err != nil { + return nil, err } + // An explicit per-session language (the -transcribe path) wins over the + // provider's current setting; the live hotkey path leaves it empty. + lang := cfg.Language + if lang == "" { + lang = p.GetLanguage() + } + return &localSession{engine: p.readyEngine, lang: lang, hints: cfg.Hints, updates: make(chan string)}, nil +} + +// readyEngine returns the engine for the desired model, waiting out an +// in-flight load (or loading now, after a Close or a failed load). +func (p *localProvider) readyEngine() (localEngine, error) { + p.load() p.mu.Lock() eng, err := p.engine, p.loadErr - lang := p.lang p.mu.Unlock() if err != nil { return nil, err @@ -330,17 +353,12 @@ func (p *localProvider) NewSession(_ context.Context, cfg SessionConfig) (Sessio if eng == nil { return nil, fmt.Errorf("%s: model %q is reloading, try again", p.name, p.GetModel()) } - // An explicit per-session language (the -transcribe path) wins over the - // provider's current setting; the live hotkey path leaves it empty. - if cfg.Language != "" { - lang = cfg.Language - } - return &localSession{engine: eng, lang: lang, hints: cfg.Hints, updates: make(chan string)}, nil + return eng, nil } // Close frees the loaded model. It waits out any in-flight background load // (loadMu) so it can't free an engine a loader is about to publish. The provider -// is reusable afterwards (the next NewSession reloads lazily). +// is reusable afterwards (the next session reloads lazily, in its Close). func (p *localProvider) Close() { p.loadMu.Lock() defer p.loadMu.Unlock() diff --git a/transcriber/local_session.go b/transcriber/local_session.go index 93b9a0c..18a144d 100644 --- a/transcriber/local_session.go +++ b/transcriber/local_session.go @@ -41,9 +41,10 @@ func logDetectedLanguage(e localEngine) { // localSession buffers raw S16LE PCM during recording, then runs one batch // inference on Close. Same Session interface as the cloud batch path, so the -// live hotkey and -transcribe share it — no encoder, no network. +// live hotkey and -transcribe share it — no encoder, no network. engine is +// resolved in Close, so a model still loading never delays the recording. type localSession struct { - engine localEngine + engine func() (localEngine, error) lang string hints string mu sync.Mutex @@ -76,13 +77,22 @@ func (s *localSession) Close() (SessionResult, error) { audioData := audio.PCMToWAV(raw) convertMs := float64(time.Since(convStart).Microseconds()) / 1000 + waitStart := time.Now() + eng, err := s.engine() + if err != nil { + return SessionResult{AudioData: audioData, AudioFormat: "wav"}, err + } + if wait := time.Since(waitStart); wait >= time.Millisecond { + log.Info(fmt.Sprintf("model_load_wait ms=%.0f", float64(wait.Microseconds())/1000)) + } + start := time.Now() - text, err := s.engine.Transcribe(f32, s.lang, s.hints) + text, err := eng.Transcribe(f32, s.lang, s.hints) if err != nil { return SessionResult{AudioData: audioData, AudioFormat: "wav"}, err } inferenceMs := float64(time.Since(start).Microseconds()) / 1000 - logDetectedLanguage(s.engine) + logDetectedLanguage(eng) text = strings.TrimSpace(text) noSpeech := text == "" diff --git a/transcriber/parakeet_async_test.go b/transcriber/parakeet_async_test.go index 71d29e8..079803f 100644 --- a/transcriber/parakeet_async_test.go +++ b/transcriber/parakeet_async_test.go @@ -2,8 +2,12 @@ package transcriber import ( "context" + "fmt" + "os" + "path/filepath" "sync" "testing" + "time" "zee/localmodel" ) @@ -47,3 +51,84 @@ func TestLocalProviderConcurrentAccess(t *testing.T) { } wg.Wait() } + +// countEngine "transcribes" by reporting how many samples it was given. +type countEngine struct{} + +func (countEngine) Transcribe(pcm []float32, _, _ string) (string, error) { + return fmt.Sprintf("%d samples", len(pcm)), nil +} + +func (countEngine) Close() {} + +// TestLocalSessionStartsBeforeLoad: capture starts on the keypress, so +// NewSession must not wait for a model that is still loading — speech spoken +// meanwhile used to be lost while the start beep already said "recording". +// The audio buffered during the load must be transcribed once it finishes. +func TestLocalSessionStartsBeforeLoad(t *testing.T) { + m, _ := localmodel.ByID(localmodel.ID110mEN) + dir := t.TempDir() + t.Setenv("ZEE_MODELS_DIR", dir) + f, err := os.Create(filepath.Join(dir, m.Filename)) + if err != nil { + t.Fatal(err) + } + if err := f.Truncate(m.SizeBytes); err != nil { // sparse: Present only checks the size + t.Fatal(err) + } + f.Close() + + release := make(chan struct{}) + slowOpen := func(localmodel.Model) (localEngine, error) { + <-release + return countEngine{}, nil + } + p := newLocalProvider(localmodel.EngineParakeet, m.ID, "en", false, slowOpen, parakeetLanguages) + defer p.Close() + + sessCh := make(chan Session, 1) + go func() { + s, err := p.NewSession(context.Background(), SessionConfig{}) + if err != nil { + t.Errorf("NewSession: %v", err) + } + sessCh <- s + }() + var sess Session + select { + case sess = <-sessCh: + case <-time.After(2 * time.Second): + close(release) + t.Fatal("NewSession blocked on the model load") + } + if sess == nil { + close(release) + return + } + + pcm := make([]byte, 3200) // 1600 samples + sess.Feed(pcm) + + resCh := make(chan SessionResult, 1) + go func() { + res, err := sess.Close() + if err != nil { + t.Errorf("Close: %v", err) + } + resCh <- res + }() + select { + case <-resCh: + t.Fatal("Close returned before the model finished loading") + case <-time.After(50 * time.Millisecond): + } + close(release) + select { + case res := <-resCh: + if res.Text != "1600 samples" { + t.Fatalf("Text = %q, want the buffered audio transcribed (%q)", res.Text, "1600 samples") + } + case <-time.After(2 * time.Second): + t.Fatal("Close did not return after the load finished") + } +} diff --git a/transcriber/stream_session.go b/transcriber/stream_session.go index 5aec416..1abc982 100644 --- a/transcriber/stream_session.go +++ b/transcriber/stream_session.go @@ -1,6 +1,7 @@ package transcriber import ( + "errors" "fmt" "strings" "sync" @@ -15,8 +16,24 @@ const ( streamChunkBytes = encoder.SampleRate * encoder.Channels * (encoder.BitsPerSample / 8) * streamChunkMs / 1000 streamFinalizeIdle = 200 * time.Millisecond streamFinalizeMax = 2000 * time.Millisecond + // streamSendTimeout bounds one websocket write. A stalled network makes a + // write block instead of fail, and a sender stuck there never drains audioCh. + streamSendTimeout = 5 * time.Second + // streamQueueChunks is how much audio may wait for the sender before Feed + // gives up on the stream. + streamQueueChunks = 128 ) +// streamDrainMax bounds how long Close waits for the sender to flush the +// queued audio before it aborts the stream. A var so tests can shorten it. +var streamDrainMax = 5 * time.Second + +// errStreamStalled is recorded when the sender falls a full queue behind: +// Feed runs on the capture callback and must never block on the network. +var errStreamStalled = errors.New("stream stalled: network not keeping up, audio kept for saving") + +// rawStreamSession is one provider websocket. Close must unblock a Send or Recv +// in progress; streamSession relies on it to abort a stalled stream. type rawStreamSession interface { Send(pcm []byte) error CloseSend() error @@ -51,7 +68,6 @@ type streamSession struct { mu sync.Mutex err error - errOnce sync.Once closing bool stats streamStats } @@ -74,7 +90,7 @@ func (s streamStats) audioDuration() float64 { func newStreamSession(dial func() (rawStreamSession, error)) *streamSession { ss := &streamSession{ - audioCh: make(chan []byte, 128), + audioCh: make(chan []byte, streamQueueChunks), updates: make(chan string, 16), startedAt: time.Now(), sendDone: make(chan struct{}), @@ -91,11 +107,7 @@ func newStreamSession(dial func() (rawStreamSession, error)) *streamSession { ss.mu.Unlock() if err != nil { - ss.errOnce.Do(func() { - ss.mu.Lock() - ss.err = err - ss.mu.Unlock() - }) + ss.recordErr(err) close(ss.sendDone) close(ss.recvDone) close(ss.connected) @@ -138,7 +150,22 @@ func (s *streamSession) Feed(pcm []byte) { s.feedMu.Unlock() for _, chunk := range chunks { - s.audioCh <- chunk + if !s.enqueue(chunk) { + return + } + } +} + +// enqueue hands a chunk to the sender without blocking. A full queue means the +// sender is stuck on the network: the stream is failed rather than letting the +// capture callback (and the recording lock it holds) wait on it. +func (s *streamSession) enqueue(chunk []byte) bool { + select { + case s.audioCh <- chunk: + return true + default: + s.recordErr(errStreamStalled) + return false } } @@ -149,47 +176,52 @@ func (s *streamSession) Updates() <-chan string { func (s *streamSession) Close() (SessionResult, error) { <-s.connected - // If connection failed, drain and return error - s.mu.Lock() - if s.err != nil { - connErr := s.err - s.mu.Unlock() - go func() { // drain audioCh so any blocked Feed() unblocks - for range s.audioCh { - } - }() - s.feedMu.Lock() - s.feedBuf = nil - s.feedMu.Unlock() - close(s.audioCh) - <-s.sendDone - <-s.recvDone + // The dial failed: sender and receiver never ran. + if s.ws == nil { close(s.updates) sr := SessionResult{NoSpeech: true} s.attachAudio(&sr) - return sr, connErr + return sr, s.sessionErr() } - s.mu.Unlock() // Flush remaining buffered PCM s.feedMu.Lock() - if len(s.feedBuf) > 0 { + if len(s.feedBuf) > 0 && s.sessionErr() == nil { tail := make([]byte, len(s.feedBuf)) copy(tail, s.feedBuf) - s.feedBuf = nil - s.audioCh <- tail + s.enqueue(tail) } + s.feedBuf = nil s.feedMu.Unlock() close(s.audioCh) finalizeStart := time.Now() - <-s.sendDone - - // Wait for server finalize acknowledgment, then brief quiet period + // A failed stream has nothing left worth sending: abort it now, which also + // unblocks a write stuck on the network. A healthy one gets streamDrainMax + // to flush its queue. + if s.sessionErr() != nil { + s.ws.Close() + } select { - case <-s.finalized: - time.Sleep(streamFinalizeIdle) - case <-time.After(streamFinalizeMax): + case <-s.sendDone: + case <-time.After(streamDrainMax): + s.recordErr(fmt.Errorf("stream send did not finish in %s", streamDrainMax)) + s.ws.Close() + select { + case <-s.sendDone: + case <-time.After(2 * time.Second): + log.Warn("stream sender drain timeout") + } + } + + // Wait for server finalize acknowledgment, then brief quiet period. A + // failed stream never gets one. + if s.sessionErr() == nil { + select { + case <-s.finalized: + time.Sleep(streamFinalizeIdle) + case <-time.After(streamFinalizeMax): + } } s.mu.Lock() @@ -265,6 +297,9 @@ func (s *streamSession) attachAudio(sr *SessionResult) { func (s *streamSession) runSender() { defer close(s.sendDone) for chunk := range s.audioCh { + if s.sessionErr() != nil { + return // the stream already failed; the queued audio is moot + } if err := s.ws.Send(chunk); err != nil { s.setErr(err) return @@ -335,18 +370,29 @@ func (s *streamSession) runReceiver() { } } +// setErr records a sender/receiver failure and closes the websocket so the +// other goroutine unblocks. Only those goroutines call it, so ws is set. func (s *streamSession) setErr(err error) { - if err == nil { - return + if err != nil && s.recordErr(err) { + s.ws.Close() } - s.errOnce.Do(func() { - s.mu.Lock() - s.err = err - s.mu.Unlock() - if s.ws != nil { - s.ws.Close() - } - }) +} + +// recordErr keeps the first error and reports whether this call set it. +func (s *streamSession) recordErr(err error) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.err != nil { + return false + } + s.err = err + return true +} + +func (s *streamSession) sessionErr() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.err } func (s *streamSession) formatMetrics(stats streamStats) []string { diff --git a/transcriber/stream_session_test.go b/transcriber/stream_session_test.go index 14f5932..56c6b75 100644 --- a/transcriber/stream_session_test.go +++ b/transcriber/stream_session_test.go @@ -5,6 +5,7 @@ import ( "errors" "sync" "testing" + "time" "zee/audio" ) @@ -128,3 +129,90 @@ func TestStreamSessionRetainsAudioOnConnectError(t *testing.T) { t.Fatalf("retained PCM differs from fed PCM (got %d bytes, want %d)", len(got), len(pcm)) } } + +// stallRawStream models a network stall: every Send blocks until Close, the +// way a websocket write does when the peer stops reading. +type stallRawStream struct { + closed chan struct{} + closeOnce sync.Once +} + +func newStallRawStream() *stallRawStream { return &stallRawStream{closed: make(chan struct{})} } + +func (f *stallRawStream) Send([]byte) error { + <-f.closed + return errors.New("connection closed") +} + +func (f *stallRawStream) CloseSend() error { return nil } + +func (f *stallRawStream) Recv() (streamUpdate, error) { + <-f.closed + return streamUpdate{}, errors.New("connection closed") +} + +func (f *stallRawStream) Close() error { + f.closeOnce.Do(func() { close(f.closed) }) + return nil +} + +// within fails the test if fn does not return in d. +func within(t *testing.T, d time.Duration, what string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { fn(); close(done) }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("%s blocked for over %s", what, d) + } +} + +// TestStreamSessionStallNeverBlocksFeed reproduces the hotkey-dead hang: the +// network stalls mid-dictation, the send queue fills, and Feed — called from +// the capture callback under the recording lock — must not block on it. Close +// must then return the error with every byte of audio kept for saving. +func TestStreamSessionStallNeverBlocksFeed(t *testing.T) { + ss := newStreamSession(func() (rawStreamSession, error) { return newStallRawStream(), nil }) + <-ss.connected + + chunk := make([]byte, streamChunkBytes) + var fed []byte + within(t, 2*time.Second, "Feed", func() { + for range streamQueueChunks + 10 { + ss.Feed(chunk) + fed = append(fed, chunk...) + } + }) + + var res SessionResult + var err error + within(t, 2*time.Second, "Close", func() { res, err = ss.Close() }) + if !errors.Is(err, errStreamStalled) { + t.Fatalf("Close error = %v, want %v", err, errStreamStalled) + } + got, wavErr := audio.WAVToPCM(res.AudioData) + if wavErr != nil { + t.Fatalf("AudioData after a stall is not a valid WAV: %v", wavErr) + } + if !bytes.Equal(got, fed) { + t.Fatalf("retained PCM differs from fed PCM (got %d bytes, want %d)", len(got), len(fed)) + } +} + +// TestStreamSessionCloseBoundsStalledSend: a stall short of a full queue +// leaves the sender stuck in one write when recording ends. Close must give up +// after streamDrainMax instead of waiting on the network forever. +func TestStreamSessionCloseBoundsStalledSend(t *testing.T) { + defer func(d time.Duration) { streamDrainMax = d }(streamDrainMax) + streamDrainMax = 100 * time.Millisecond + + ss := newStreamSession(func() (rawStreamSession, error) { return newStallRawStream(), nil }) + ss.Feed(testPCM()) + + var err error + within(t, 2*time.Second, "Close", func() { _, err = ss.Close() }) + if err == nil { + t.Fatal("Close on a stalled stream returned no error") + } +} diff --git a/tray/tray_darwin.go b/tray/tray_darwin.go index 716e7d7..e9b7abc 100644 --- a/tray/tray_darwin.go +++ b/tray/tray_darwin.go @@ -3,6 +3,9 @@ package tray import ( + "slices" + "sync" + "github.com/energye/systray" "golang.design/x/hotkey/mainthread" @@ -15,8 +18,12 @@ var ( mCopy *systray.MenuItem mDevices *systray.MenuItem mDefaultDevice *systray.MenuItem - deviceItems []*systray.MenuItem + deviceItems []*systray.MenuItem // guarded by trayMu deviceReady chan struct{} + // deviceMenuMu serializes RefreshDevices (the device watcher and Reload + // Config both call it), so two renders can't append the same new items. + // Never taken on the main thread. + deviceMenuMu sync.Mutex mSettings *systray.MenuItem mAutoPaste *systray.MenuItem @@ -38,13 +45,15 @@ var ( // models, so a model-state change can re-derive the group's enabled state // and "(no API key)" suffix (a key added via Reload Config re-enables its // provider without rebuilding the menu). - modelGroups []struct { - item *systray.MenuItem - label string - start, end int - } + modelGroups []modelGroup ) +type modelGroup struct { + item *systray.MenuItem + label string + start, end int +} + func Init() <-chan struct{} { deviceReady = make(chan struct{}) start, _ := systray.RunWithExternalLoop(onReady, onExit) @@ -137,32 +146,50 @@ func addDeviceItem(parent *systray.MenuItem, idx int, name string, checked bool) if cb != nil && currentName != "" { cb(currentName) } - trayMu.Lock() - if mDefaultDevice != nil { - mDefaultDevice.Uncheck() - } - for _, it := range deviceItems { - it.Uncheck() - } - if idx < len(deviceItems) { - deviceItems[idx].Check() - } - trayMu.Unlock() + checkDevice(idx) }) return item } +// checkDevice moves the checkmark to deviceItems[idx], or to System Default +// when idx is -1. +func checkDevice(idx int) { + trayMu.Lock() + items := deviceItems + trayMu.Unlock() + if idx < 0 { + mDefaultDevice.Check() + } else { + mDefaultDevice.Uncheck() + } + for i, it := range items { + if i == idx { + it.Check() + } else { + it.Uncheck() + } + } +} + +// RefreshDevices re-renders the device submenu. It runs off the main thread, +// and every systray call waits for the main thread to apply it — where menu +// click handlers take trayMu — so state is swapped under trayMu and the menu is +// drawn after releasing it. Holding trayMu across the draw was a lock cycle +// that froze the app. func RefreshDevices(names []string, selected string) { if deviceReady == nil { return } <-deviceReady - trayMu.Lock() - defer trayMu.Unlock() + deviceMenuMu.Lock() + defer deviceMenuMu.Unlock() + trayMu.Lock() deviceNames = names deviceSel = selected + items := deviceItems + trayMu.Unlock() if mDefaultDevice != nil { if selected == "" { @@ -172,7 +199,7 @@ func RefreshDevices(names []string, selected string) { } } - for i, item := range deviceItems { + for i, item := range items { if i < len(names) { label := deviceDisplayName(names[i]) item.SetTitle(label) @@ -189,9 +216,14 @@ func RefreshDevices(names []string, selected string) { } } - for i := len(deviceItems); i < len(names); i++ { - item := addDeviceItem(mDevices, i, names[i], names[i] == selected) - deviceItems = append(deviceItems, item) + var added []*systray.MenuItem + for i := len(items); i < len(names); i++ { + added = append(added, addDeviceItem(mDevices, i, names[i], names[i] == selected)) + } + if len(added) > 0 { + trayMu.Lock() + deviceItems = append(deviceItems, added...) + trayMu.Unlock() } } @@ -327,8 +359,12 @@ func onReady() { mDevices = mSettings.AddSubMenuItem("Microphone", "Select input device") + // onReady runs off the main thread too: the menu is built from snapshots, + // never with trayMu held across a systray call. trayMu.Lock() - mDefaultDevice = mDevices.AddSubMenuItemCheckbox("System Default", "Use system default device", deviceSel == "") + devSel, devNames := deviceSel, deviceNames + trayMu.Unlock() + mDefaultDevice = mDevices.AddSubMenuItemCheckbox("System Default", "Use system default device", devSel == "") mDefaultDevice.Click(func() { trayMu.Lock() cb := deviceCb @@ -336,34 +372,33 @@ func onReady() { if cb != nil { cb("") } - trayMu.Lock() - for _, it := range deviceItems { - it.Uncheck() - } - mDefaultDevice.Check() - trayMu.Unlock() + checkDevice(-1) }) - deviceItems = make([]*systray.MenuItem, 0, len(deviceNames)) - for i, name := range deviceNames { - item := addDeviceItem(mDevices, i, name, name == deviceSel) - deviceItems = append(deviceItems, item) + devItems := make([]*systray.MenuItem, 0, len(devNames)) + for i, name := range devNames { + devItems = append(devItems, addDeviceItem(mDevices, i, name, name == devSel)) } + trayMu.Lock() + deviceItems = devItems trayMu.Unlock() // Divide the input device from the transcription pair (Model + Language). addSubmenuSeparator(mSettings) trayMu.Lock() - if len(models) > 0 { + snap := slices.Clone(models) + trayMu.Unlock() + if len(snap) > 0 { mBackend = mSettings.AddSubMenuItem("Model", "Select transcription model") - modelItems = make([]*systray.MenuItem, len(models)) - // models are grouped by Group (contiguous); one submenu per group. Both + items := make([]*systray.MenuItem, len(snap)) + var groups []modelGroup + // Models are grouped by Group (contiguous); one submenu per group. Both // local engines share the "Local" group, each cloud provider is its own. - for i := 0; i < len(models); { - group := models[i].Group + for i := 0; i < len(snap); { + group := snap[i].Group j, anyUsable := i, false - for j < len(models) && models[j].Group == group { - if models[j].State != ModelUnavailable { + for j < len(snap) && snap[j].Group == group { + if snap[j].State != ModelUnavailable { anyUsable = true } j++ @@ -376,14 +411,10 @@ func onReady() { if !anyUsable { provMenu.Disable() } - modelGroups = append(modelGroups, struct { - item *systray.MenuItem - label string - start, end int - }{provMenu, group, i, j}) + groups = append(groups, modelGroup{provMenu, group, i, j}) for k := i; k < j; k++ { idx := k - m := models[k] + m := snap[k] item := provMenu.AddSubMenuItemCheckbox(modelTitle(m), m.Label, m.Active && m.State == ModelReady) if m.State == ModelUnavailable || m.State == ModelDownloading { item.Disable() @@ -400,16 +431,18 @@ func onReady() { } cb(mm.Provider, mm.ModelID) }) - modelItems[idx] = item + items[idx] = item } // Divide Local from the cloud providers. - if group == "Local" && j < len(models) { + if group == "Local" && j < len(snap) { addSubmenuSeparator(mBackend) } i = j } + trayMu.Lock() + modelItems, modelGroups = items, groups + trayMu.Unlock() } - trayMu.Unlock() // Build a fixed item per known language (systray can't add items after // CreateMenu). refreshLanguageMenu then shows only the active model's set.