Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ Worker vars: `ELEVENLABS_VOICE_ID`
| File | Lines | Purpose |
|------|-------|---------|
| `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. |
| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. |
| `CompanionManager.swift` | ~1260 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. |
| `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. |
| `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. |
| `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. |
Expand All @@ -70,6 +70,7 @@ Worker vars: `ELEVENLABS_VOICE_ID`
| `OpenAIAPI.swift` | ~142 | OpenAI GPT vision API client. |
| `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. |
| `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. |
| `StepByStepGuide.swift` | ~220 | Step-by-step guidance model and parser. `GuidanceStep`, `StepByStepGuide`, `StepAdvanceCommand`, and `StepByStepGuideParser` for multi-step visual guidance "show me how" feature. Claude returns `[GUIDE:N]` blocks which are parsed into sequential steps with per-step pointing coordinates. |
| `DesignSystem.swift` | ~880 | Design system tokens — colors, corner radii, shared styles. All UI references `DS.Colors`, `DS.CornerRadius`, etc. |
| `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. |
| `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. |
Expand Down
25 changes: 20 additions & 5 deletions leanring-buddy/BuddyDictationManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,10 @@ final class BuddyDictationManager: NSObject, ObservableObject {
private var contextualKeyterms: [String] = []
private var lastRecordedAudioPowerSampleDate = Date.distantPast
private var activePermissionRequestTask: Task<Bool, Never>?
/// Incremented each time a new recognition session starts. Callbacks from
/// a cancelled/preempted session can detect staleness by comparing against
/// this value and ignoring their result.
private var sessionGeneration = 0
/// Timestamp of the last completed permission request, used to debounce
/// rapid follow-up requests that arrive before macOS updates its cache.
private var lastPermissionRequestCompletedAt: Date?
Expand Down Expand Up @@ -515,18 +519,22 @@ final class BuddyDictationManager: NSObject, ObservableObject {
activeTranscriptionSession?.cancel()
activeTranscriptionSession = nil

sessionGeneration += 1
let currentGeneration = sessionGeneration

print("🎙️ BuddyDictationManager: opening transcription provider \(transcriptionProvider.displayName)")

let activeTranscriptionSession = try await transcriptionProvider.startStreamingSession(
keyterms: buildTranscriptionKeyterms(),
onTranscriptUpdate: { [weak self] transcriptText in
Task { @MainActor in
self?.latestRecognizedText = transcriptText
guard let self, self.sessionGeneration == currentGeneration else { return }
self.latestRecognizedText = transcriptText
}
},
onFinalTranscriptReady: { [weak self] transcriptText in
Task { @MainActor in
guard let self else { return }
guard let self, self.sessionGeneration == currentGeneration else { return }
self.latestRecognizedText = transcriptText

if self.isFinalizingTranscript {
Expand All @@ -538,7 +546,8 @@ final class BuddyDictationManager: NSObject, ObservableObject {
},
onError: { [weak self] error in
Task { @MainActor in
self?.handleRecognitionError(error)
guard let self, self.sessionGeneration == currentGeneration else { return }
self.handleRecognitionError(error)
}
}
)
Expand All @@ -551,8 +560,13 @@ final class BuddyDictationManager: NSObject, ObservableObject {

inputNode.removeTap(onBus: 0)
inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in
self?.activeTranscriptionSession?.appendAudioBuffer(buffer)
self?.updateAudioPowerLevel(from: buffer)
// Dispatch from the background audio thread to @MainActor properties
// to avoid data races on activeTranscriptionSession and audio power level.
Task { @MainActor [weak self] in
guard let self else { return }
self.activeTranscriptionSession?.appendAudioBuffer(buffer)
self.updateAudioPowerLevel(from: buffer)
}
}

audioEngine.prepare()
Expand Down Expand Up @@ -647,6 +661,7 @@ final class BuddyDictationManager: NSObject, ObservableObject {
)
microphoneButtonRecordingStartedAt = nil
lastRecordedAudioPowerSampleDate = .distantPast
sessionGeneration += 1
}

private func buildTranscriptionKeyterms() -> [String] {
Expand Down
23 changes: 15 additions & 8 deletions leanring-buddy/ClaudeAPI.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@ class ClaudeAPI {
var model: String
private let session: URLSession

init(proxyURL: String, model: String = "claude-sonnet-4-6") {
self.apiURL = URL(string: proxyURL)!
init?(proxyURL: String, model: String = "claude-sonnet-4-6") {
guard let url = URL(string: proxyURL) else { return nil }
self.apiURL = url
self.model = model

// Use .default instead of .ephemeral so TLS session tickets are cached.
Expand All @@ -36,6 +37,10 @@ class ClaudeAPI {
warmUpTLSConnectionIfNeeded()
}

deinit {
session.invalidateAndCancel()
}

private func makeAPIRequest() -> URLRequest {
var request = URLRequest(url: apiURL)
request.httpMethod = "POST"
Expand Down Expand Up @@ -65,11 +70,11 @@ class ClaudeAPI {
/// Failures are silently ignored — this is purely an optimization.
private func warmUpTLSConnectionIfNeeded() {
Self.tlsWarmupLock.lock()
defer { Self.tlsWarmupLock.unlock() }
let shouldStartTLSWarmup = !Self.hasStartedTLSWarmup
if shouldStartTLSWarmup {
Self.hasStartedTLSWarmup = true
}
Self.tlsWarmupLock.unlock()

guard shouldStartTLSWarmup else { return }

Expand Down Expand Up @@ -188,9 +193,13 @@ class ClaudeAPI {
// End of stream marker
guard jsonString != "[DONE]" else { break }

guard let jsonData = jsonString.data(using: .utf8),
let eventPayload = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any],
guard let jsonData = jsonString.data(using: .utf8) else {
print("⚠️ Claude SSE: non-UTF8 data: \(jsonString.prefix(80))")
continue
}
guard let eventPayload = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any],
let eventType = eventPayload["type"] as? String else {
print("⚠️ Claude SSE: malformed event payload: \(jsonString.prefix(80))")
continue
}

Expand All @@ -201,9 +210,7 @@ class ClaudeAPI {
deltaType == "text_delta",
let textChunk = delta["text"] as? String {
accumulatedResponseText += textChunk
// Send the accumulated text so far to the UI for progressive rendering
let currentAccumulatedText = accumulatedResponseText
await onTextChunk(currentAccumulatedText)
await onTextChunk(textChunk)
}
}

Expand Down
Loading