diff --git a/AGENTS.md b/AGENTS.md index 6946d4419..a4b88dc91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Worker vars: `ELEVENLABS_VOICE_ID` | File | Lines | Purpose | |------|-------|---------| | `leanring_buddyApp.swift` | ~89 | Menu bar app entry point. Uses `@NSApplicationDelegateAdaptor` with `CompanionAppDelegate` which creates `MenuBarPanelManager` and starts `CompanionManager`. No main window — the app lives entirely in the status bar. | -| `CompanionManager.swift` | ~1026 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | +| `CompanionManager.swift` | ~1260 | Central state machine. Owns dictation, shortcut monitoring, screen capture, Claude API, ElevenLabs TTS, and overlay management. Tracks voice state (idle/listening/processing/responding), conversation history, model selection, and cursor visibility. Coordinates the full push-to-talk → screenshot → Claude → TTS → pointing pipeline. | | `MenuBarPanelManager.swift` | ~243 | NSStatusItem + custom NSPanel lifecycle. Creates the menu bar icon, manages the floating companion panel (show/hide/position), installs click-outside-to-dismiss monitor. | | `CompanionPanelView.swift` | ~761 | SwiftUI panel content for the menu bar dropdown. Shows companion status, push-to-talk instructions, model picker (Sonnet/Opus), permissions UI, DM feedback button, and quit button. Dark aesthetic using `DS` design system. | | `OverlayWindow.swift` | ~881 | Full-screen transparent overlay hosting the blue cursor, response text, waveform, and spinner. Handles cursor animation, element pointing with bezier arcs, multi-monitor coordinate mapping, and fade-out transitions. | @@ -70,6 +70,7 @@ Worker vars: `ELEVENLABS_VOICE_ID` | `OpenAIAPI.swift` | ~142 | OpenAI GPT vision API client. | | `ElevenLabsTTSClient.swift` | ~81 | ElevenLabs TTS client. Sends text to the Worker proxy, plays back audio via `AVAudioPlayer`. Exposes `isPlaying` for transient cursor scheduling. | | `ElementLocationDetector.swift` | ~335 | Detects UI element locations in screenshots for cursor pointing. | +| `StepByStepGuide.swift` | ~220 | Step-by-step guidance model and parser. `GuidanceStep`, `StepByStepGuide`, `StepAdvanceCommand`, and `StepByStepGuideParser` for multi-step visual guidance "show me how" feature. Claude returns `[GUIDE:N]` blocks which are parsed into sequential steps with per-step pointing coordinates. | | `DesignSystem.swift` | ~880 | Design system tokens — colors, corner radii, shared styles. All UI references `DS.Colors`, `DS.CornerRadius`, etc. | | `ClickyAnalytics.swift` | ~121 | PostHog analytics integration for usage tracking. | | `WindowPositionManager.swift` | ~262 | Window placement logic, Screen Recording permission flow, and accessibility permission helpers. | diff --git a/leanring-buddy/BuddyDictationManager.swift b/leanring-buddy/BuddyDictationManager.swift index 5bca26779..61e760569 100644 --- a/leanring-buddy/BuddyDictationManager.swift +++ b/leanring-buddy/BuddyDictationManager.swift @@ -276,6 +276,10 @@ final class BuddyDictationManager: NSObject, ObservableObject { private var contextualKeyterms: [String] = [] private var lastRecordedAudioPowerSampleDate = Date.distantPast private var activePermissionRequestTask: Task? + /// Incremented each time a new recognition session starts. Callbacks from + /// a cancelled/preempted session can detect staleness by comparing against + /// this value and ignoring their result. + private var sessionGeneration = 0 /// Timestamp of the last completed permission request, used to debounce /// rapid follow-up requests that arrive before macOS updates its cache. private var lastPermissionRequestCompletedAt: Date? @@ -515,18 +519,22 @@ final class BuddyDictationManager: NSObject, ObservableObject { activeTranscriptionSession?.cancel() activeTranscriptionSession = nil + sessionGeneration += 1 + let currentGeneration = sessionGeneration + print("🎙️ BuddyDictationManager: opening transcription provider \(transcriptionProvider.displayName)") let activeTranscriptionSession = try await transcriptionProvider.startStreamingSession( keyterms: buildTranscriptionKeyterms(), onTranscriptUpdate: { [weak self] transcriptText in Task { @MainActor in - self?.latestRecognizedText = transcriptText + guard let self, self.sessionGeneration == currentGeneration else { return } + self.latestRecognizedText = transcriptText } }, onFinalTranscriptReady: { [weak self] transcriptText in Task { @MainActor in - guard let self else { return } + guard let self, self.sessionGeneration == currentGeneration else { return } self.latestRecognizedText = transcriptText if self.isFinalizingTranscript { @@ -538,7 +546,8 @@ final class BuddyDictationManager: NSObject, ObservableObject { }, onError: { [weak self] error in Task { @MainActor in - self?.handleRecognitionError(error) + guard let self, self.sessionGeneration == currentGeneration else { return } + self.handleRecognitionError(error) } } ) @@ -551,8 +560,13 @@ final class BuddyDictationManager: NSObject, ObservableObject { inputNode.removeTap(onBus: 0) inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in - self?.activeTranscriptionSession?.appendAudioBuffer(buffer) - self?.updateAudioPowerLevel(from: buffer) + // Dispatch from the background audio thread to @MainActor properties + // to avoid data races on activeTranscriptionSession and audio power level. + Task { @MainActor [weak self] in + guard let self else { return } + self.activeTranscriptionSession?.appendAudioBuffer(buffer) + self.updateAudioPowerLevel(from: buffer) + } } audioEngine.prepare() @@ -647,6 +661,7 @@ final class BuddyDictationManager: NSObject, ObservableObject { ) microphoneButtonRecordingStartedAt = nil lastRecordedAudioPowerSampleDate = .distantPast + sessionGeneration += 1 } private func buildTranscriptionKeyterms() -> [String] { diff --git a/leanring-buddy/ClaudeAPI.swift b/leanring-buddy/ClaudeAPI.swift index 0c7070b56..56cea99a3 100644 --- a/leanring-buddy/ClaudeAPI.swift +++ b/leanring-buddy/ClaudeAPI.swift @@ -14,8 +14,9 @@ class ClaudeAPI { var model: String private let session: URLSession - init(proxyURL: String, model: String = "claude-sonnet-4-6") { - self.apiURL = URL(string: proxyURL)! + init?(proxyURL: String, model: String = "claude-sonnet-4-6") { + guard let url = URL(string: proxyURL) else { return nil } + self.apiURL = url self.model = model // Use .default instead of .ephemeral so TLS session tickets are cached. @@ -36,6 +37,10 @@ class ClaudeAPI { warmUpTLSConnectionIfNeeded() } + deinit { + session.invalidateAndCancel() + } + private func makeAPIRequest() -> URLRequest { var request = URLRequest(url: apiURL) request.httpMethod = "POST" @@ -65,11 +70,11 @@ class ClaudeAPI { /// Failures are silently ignored — this is purely an optimization. private func warmUpTLSConnectionIfNeeded() { Self.tlsWarmupLock.lock() + defer { Self.tlsWarmupLock.unlock() } let shouldStartTLSWarmup = !Self.hasStartedTLSWarmup if shouldStartTLSWarmup { Self.hasStartedTLSWarmup = true } - Self.tlsWarmupLock.unlock() guard shouldStartTLSWarmup else { return } @@ -188,9 +193,13 @@ class ClaudeAPI { // End of stream marker guard jsonString != "[DONE]" else { break } - guard let jsonData = jsonString.data(using: .utf8), - let eventPayload = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any], + guard let jsonData = jsonString.data(using: .utf8) else { + print("⚠️ Claude SSE: non-UTF8 data: \(jsonString.prefix(80))") + continue + } + guard let eventPayload = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any], let eventType = eventPayload["type"] as? String else { + print("⚠️ Claude SSE: malformed event payload: \(jsonString.prefix(80))") continue } @@ -201,9 +210,7 @@ class ClaudeAPI { deltaType == "text_delta", let textChunk = delta["text"] as? String { accumulatedResponseText += textChunk - // Send the accumulated text so far to the UI for progressive rendering - let currentAccumulatedText = accumulatedResponseText - await onTextChunk(currentAccumulatedText) + await onTextChunk(textChunk) } } diff --git a/leanring-buddy/CompanionManager.swift b/leanring-buddy/CompanionManager.swift index 0234cf19f..d4c16a899 100644 --- a/leanring-buddy/CompanionManager.swift +++ b/leanring-buddy/CompanionManager.swift @@ -42,6 +42,26 @@ final class CompanionManager: ObservableObject { /// BlueCursorView uses this instead of a random pointer phrase. @Published var detectedElementBubbleText: String? + // MARK: - Step-by-Step Guidance State + + /// True when the AI is walking the user through a multi-step visual guide. + @Published var isStepByStepActive = false + + /// The full set of steps for the current guide. + @Published var guideSteps: [GuidanceStep] = [] + + /// Index of the currently visible step (0-based). + @Published var currentStepIndex: Int = 0 + + /// Total number of steps in the current guide. + @Published var totalSteps: Int = 0 + + /// The instruction text for the current step. + @Published var currentStepInstruction: String = "" + + /// Progress through the guide from 0.0 to 1.0. + @Published var stepProgress: Double = 0.0 + // MARK: - Onboarding Video State (shared across all screen overlays) @Published var onboardingVideoPlayer: AVPlayer? @@ -73,11 +93,11 @@ final class CompanionManager: ObservableObject { private static let workerBaseURL = "https://your-worker-name.your-subdomain.workers.dev" private lazy var claudeAPI: ClaudeAPI = { - return ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel) + ClaudeAPI(proxyURL: "\(Self.workerBaseURL)/chat", model: selectedModel)! }() private lazy var elevenLabsTTSClient: ElevenLabsTTSClient = { - return ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts") + ElevenLabsTTSClient(proxyURL: "\(Self.workerBaseURL)/tts")! }() /// Conversation history so Claude remembers prior exchanges within a session. @@ -96,6 +116,12 @@ final class CompanionManager: ObservableObject { /// Scheduled hide for transient cursor mode — cancelled if the user /// speaks again before the delay elapses. private var transientHideTask: Task? + /// Tracks the current step's coordinate mapping task so it can be cancelled + /// when the user advances to the next step before mapping completes. + private var stepCoordinateTask: Task? + /// Incremented each time a new step executes. Stale TTS completion handlers + /// check this to avoid updating state for a superseded step. + private var stepGeneration: UInt = 0 /// True when all three required permissions (accessibility, screen recording, /// microphone) are granted. Used by the panel to show a single "all good" state. @@ -473,6 +499,12 @@ final class CompanionManager: ObservableObject { private func handleShortcutTransition(_ transition: BuddyPushToTalkShortcut.ShortcutTransition) { switch transition { case .pressed: + // If a step-by-step guide is active, cancel it — the user is + // starting a fresh interaction + if isStepByStepActive { + cancelStepByStepGuide() + } + guard !buddyDictationManager.isDictationInProgress else { return } // Don't register push-to-talk while the onboarding video is playing guard !showOnboardingVideo else { return } @@ -574,6 +606,23 @@ final class CompanionManager: ObservableObject { - user asks what html is: "html stands for hypertext markup language, it's basically the skeleton of every web page. curious how it connects to the css you're looking at? [POINT:none]" - user asks how to commit in xcode: "see that source control menu up top? click that and hit commit, or you can use command option c as a shortcut. [POINT:285,11:source control]" - element is on screen 2 (not where cursor is): "that's over on your other monitor — see the terminal window? [POINT:400,300:terminal:screen2]" + + step by step guidance: + when the user asks "how do i..." or "show me how to..." or any question about walking through a multi-step task on their computer, respond with a step by step visual guide instead of a regular answer. here's how to format it: + + start with a short acknowledgment: "let me walk you through it." + then output a [GUIDE:N] tag where N is the number of steps, then each step on its own line separated by " ### ", then end with [END_GUIDE]. each step should have a [POINT:x,y:label] tag pointing at the relevant ui element. keep each step instruction to 8 words or fewer. speak in natural language, not bullet points. + + here is the exact format: + [GUIDE:3] + go to the file menu [POINT:50,10:file menu] + ### + click new project [POINT:200,150:new project] + ### + choose ios app [POINT:400,300:ios template] + [END_GUIDE] + + the text before [GUIDE will be spoken. the steps inside [GUIDE will be shown one at a time as the user advances through them. the text after [END_GUIDE will be ignored. keep each step instruction short and concrete. always include a [POINT] tag per step pointing at the relevant element. if a step doesn't have a specific on-screen element to point at, leave the tag out. """ // MARK: - AI Response Pipeline @@ -582,8 +631,21 @@ final class CompanionManager: ObservableObject { /// and plays the response aloud via ElevenLabs TTS. The cursor stays in /// the spinner/processing state until TTS audio begins playing. /// Claude's response may include a [POINT:x,y:label] tag which triggers - /// the buddy to fly to that element on screen. + /// the buddy to fly to that element on screen. When the response contains + /// a [GUIDE:N] block, the app enters step-by-step guidance mode instead. private func sendTranscriptToClaudeWithScreenshot(transcript: String) { + // If step-by-step mode is active, check if the user said an advance + // command ("next", "ok", "done", etc.) instead of a full question. + if isStepByStepActive { + if StepAdvanceCommand.matches(transcript) { + advanceToNextStep() + return + } + // The user asked something else — cancel the guide and + // send the query to Claude normally + cancelStepByStepGuide() + } + currentResponseTask?.cancel() elevenLabsTTSClient.stopPlayback() @@ -622,6 +684,42 @@ final class CompanionManager: ObservableObject { guard !Task.isCancelled else { return } + // Check if Claude returned a multi-step guide + if let (guide, preamble) = StepByStepGuideParser.parse(from: fullResponseText) { + // Cancel any existing guide and start the new one + if isStepByStepActive { cancelStepByStepGuide() } + + // Speak the preamble acknowledgment + if !preamble.isEmpty { + do { + try await elevenLabsTTSClient.speakText(preamble) + voiceState = .responding + // Wait for preamble TTS to finish before starting + // the first step with a 30-second safety timeout. + var preambleWaitSeconds = 0 + while elevenLabsTTSClient.isPlaying && preambleWaitSeconds < 30 { + try await Task.sleep(nanoseconds: 100_000_000) + preambleWaitSeconds += 1 + } + } catch { + print("⚠️ Guide preamble TTS error: \(error)") + } + } + + // Save a brief reference to conversation history so Claude + // has context if the user asks a follow-up + conversationHistory.append(( + userTranscript: transcript, + assistantResponse: "[provided a step-by-step guide with \(guide.totalSteps) steps]" + )) + if conversationHistory.count > 10 { + conversationHistory.removeFirst(conversationHistory.count - 10) + } + + startStepByStepGuide(guide) + return + } + // Parse the [POINT:...] tag from Claude's response let parseResult = Self.parsePointingCoordinates(from: fullResponseText) let spokenText = parseResult.spokenText @@ -719,7 +817,11 @@ final class CompanionManager: ObservableObject { } if !Task.isCancelled { - voiceState = .idle + // Don't reset to idle if step-by-step mode is active — + // the step sequencer manages voice state directly + if !isStepByStepActive { + voiceState = .idle + } scheduleTransientHideIfNeeded() } } @@ -755,6 +857,154 @@ final class CompanionManager: ObservableObject { } } + // MARK: - Step-by-Step Guidance Sequencing + + /// Activates step-by-step mode and executes the first step. + private func startStepByStepGuide(_ guide: StepByStepGuide) { + isStepByStepActive = true + guideSteps = guide.steps + totalSteps = guide.totalSteps + currentStepIndex = 0 + stepProgress = 0.0 + + executeStep(guide.steps[0]) + } + + /// Advances the guide to the next step, or completes if all steps are done. + func advanceToNextStep() { + guard isStepByStepActive else { return } + guard currentStepIndex < totalSteps - 1 else { + finishStepByStepGuide() + return + } + + currentStepIndex += 1 + stepProgress = Double(currentStepIndex) / Double(totalSteps) + let step = guideSteps[currentStepIndex] + executeStep(step) + } + + /// Executes a single guidance step: plays the instruction via TTS and + /// points the cursor at the relevant element. + private func executeStep(_ step: GuidanceStep) { + currentStepInstruction = step.instruction + // Don't override the navigation bubble text — the existing random + // pointer phrases ("right here!", "this one!") work better than + // element labels like "insert tab" + detectedElementBubbleText = nil + + let hasPointCoordinate = step.rawPointCoordinate != nil + if hasPointCoordinate { + voiceState = .idle + } + + // Cancel any in-flight coordinate mapping from a previous step + stepCoordinateTask?.cancel() + + // Map coordinates from step to screen coordinates (same logic as + // the point tag parsing in sendTranscriptToClaudeWithScreenshot) + if let pointCoordinate = step.rawPointCoordinate { + stepCoordinateTask = Task { + do { + let screenCaptures = try await CompanionScreenCaptureUtility.captureAllScreensAsJPEG() + + let targetScreenCapture: CompanionScreenCapture? = { + if let screenNumber = step.screenNumber, + screenNumber >= 1 && screenNumber <= screenCaptures.count { + return screenCaptures[screenNumber - 1] + } + return screenCaptures.first(where: { $0.isCursorScreen }) + }() + + guard let targetScreenCapture else { return } + + let screenshotWidth = CGFloat(targetScreenCapture.screenshotWidthInPixels) + let screenshotHeight = CGFloat(targetScreenCapture.screenshotHeightInPixels) + let displayWidth = CGFloat(targetScreenCapture.displayWidthInPoints) + let displayHeight = CGFloat(targetScreenCapture.displayHeightInPoints) + let displayFrame = targetScreenCapture.displayFrame + + let clampedX = max(0, min(pointCoordinate.x, screenshotWidth)) + let clampedY = max(0, min(pointCoordinate.y, screenshotHeight)) + + let displayLocalX = clampedX * (displayWidth / screenshotWidth) + let displayLocalY = clampedY * (displayHeight / screenshotHeight) + + let appKitY = displayHeight - displayLocalY + + let globalLocation = CGPoint( + x: displayLocalX + displayFrame.origin.x, + y: appKitY + displayFrame.origin.y + ) + + detectedElementScreenLocation = globalLocation + detectedElementDisplayFrame = displayFrame + } catch { + print("🎯 Step-by-step: coordinate mapping failed: \(error)") + } + } + stepCoordinateTask = nil + } else { + clearDetectedElementLocation() + } + + // Speak the step instruction (stops any previous TTS first) + if !step.instruction.isEmpty { + elevenLabsTTSClient.stopPlayback() + voiceState = .processing + stepGeneration += 1 + let capturedGeneration = stepGeneration + Task { + do { + try await elevenLabsTTSClient.speakText(step.instruction) + guard capturedGeneration == stepGeneration else { return } + voiceState = .responding + } catch { + guard capturedGeneration == stepGeneration else { return } + print("⚠️ Step-by-step TTS error: \(error)") + } + + if !Task.isCancelled && isStepByStepActive { + voiceState = .idle + } + } + } + } + + /// Ends the step-by-step guide and returns to normal mode. + func finishStepByStepGuide() { + cancelStepByStepGuide() + voiceState = .idle + let doneText = "that's it, you're all set. anything else?" + Task { + do { + try await elevenLabsTTSClient.speakText(doneText) + voiceState = .responding + try? await Task.sleep(nanoseconds: 2_000_000_000) + voiceState = .idle + } catch { + print("⚠️ Step-by-step finish TTS error: \(error)") + } + } + } + + /// Immediately cancels the current guide and clears all step state. + /// Stops any in-progress TTS from the guide's current step instruction. + func cancelStepByStepGuide() { + isStepByStepActive = false + guideSteps = [] + currentStepIndex = 0 + voiceState = .idle + totalSteps = 0 + currentStepInstruction = "" + stepProgress = 0.0 + stepCoordinateTask?.cancel() + stepCoordinateTask = nil + stepGeneration += 1 // invalidates any stale TTS completion handlers + elevenLabsTTSClient.stopPlayback() + clearDetectedElementLocation() + } + /// Speaks a hardcoded error message using macOS system TTS when API /// credits run out. Uses NSSpeechSynthesizer so it works even when /// ElevenLabs is down. diff --git a/leanring-buddy/CompanionResponseOverlay.swift b/leanring-buddy/CompanionResponseOverlay.swift index a11c6240f..3bc351249 100644 --- a/leanring-buddy/CompanionResponseOverlay.swift +++ b/leanring-buddy/CompanionResponseOverlay.swift @@ -104,11 +104,10 @@ final class CompanionResponseOverlayManager { } private func startCursorTracking() { - // 60fps cursor tracking so the panel stays glued to the mouse + // 60fps cursor tracking so the panel stays glued to the mouse. + // Timer already fires on the main runloop; no need for a Task hop. cursorTrackingTimer = Timer.scheduledTimer(withTimeInterval: 1.0 / 60.0, repeats: true) { [weak self] _ in - Task { @MainActor [weak self] in - self?.repositionPanelNearCursor() - } + self?.repositionPanelNearCursor() } } diff --git a/leanring-buddy/CompanionScreenCaptureUtility.swift b/leanring-buddy/CompanionScreenCaptureUtility.swift index 797841783..e9bc2f3e5 100644 --- a/leanring-buddy/CompanionScreenCaptureUtility.swift +++ b/leanring-buddy/CompanionScreenCaptureUtility.swift @@ -73,9 +73,11 @@ enum CompanionScreenCaptureUtility { // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so // displayFrame is in the same coordinate system as NSEvent.mouseLocation // and the overlay window's screenFrame in BlueCursorView. - let displayFrame = nsScreenByDisplayID[display.displayID]?.frame - ?? CGRect(x: display.frame.origin.x, y: display.frame.origin.y, - width: CGFloat(display.width), height: CGFloat(display.height)) + guard let nsScreen = nsScreenByDisplayID[display.displayID] else { + print("⚠️ No NSScreen found for display \(display.displayID) — cannot determine cursor screen, skipping") + continue + } + let displayFrame = nsScreen.frame let isCursorScreen = displayFrame.contains(mouseLocation) let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) diff --git a/leanring-buddy/ElevenLabsTTSClient.swift b/leanring-buddy/ElevenLabsTTSClient.swift index 35545c9d4..220116244 100644 --- a/leanring-buddy/ElevenLabsTTSClient.swift +++ b/leanring-buddy/ElevenLabsTTSClient.swift @@ -19,8 +19,9 @@ final class ElevenLabsTTSClient { /// audio finishes playing even if the caller doesn't hold a reference. private var audioPlayer: AVAudioPlayer? - init(proxyURL: String) { - self.proxyURL = URL(string: proxyURL)! + init?(proxyURL: String) { + guard let url = URL(string: proxyURL) else { return nil } + self.proxyURL = url let configuration = URLSessionConfiguration.default configuration.timeoutIntervalForRequest = 30 @@ -28,6 +29,10 @@ final class ElevenLabsTTSClient { self.session = URLSession(configuration: configuration) } + deinit { + session.invalidateAndCancel() + } + /// Sends `text` to ElevenLabs TTS and plays the resulting audio. /// Throws on network or decoding errors. Cancellation-safe. func speakText(_ text: String) async throws { diff --git a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift index 8020269b4..554b22a54 100644 --- a/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift +++ b/leanring-buddy/GlobalPushToTalkShortcutMonitor.swift @@ -17,6 +17,9 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { private var globalEventTap: CFMachPort? private var globalEventTapRunLoopSource: CFRunLoopSource? + /// Set to false in stop() and checked in the CGEvent tap callback. + /// Prevents use-after-free when deinit races with a tap callback. + private var isRunning = false /// Mutated exclusively from the CGEvent tap callback, which runs on /// `CFRunLoopGetMain()` and therefore always executes on the main thread. /// Published so the overlay can hide immediately on key release without @@ -48,6 +51,12 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { .fromOpaque(userInfo) .takeUnretainedValue() + // Skip processing if the monitor has been stopped; this prevents + // use-after-free when deinit races with a tap callback. + guard globalPushToTalkShortcutMonitor.isRunning else { + return Unmanaged.passUnretained(event) + } + return globalPushToTalkShortcutMonitor.handleGlobalEventTap( eventType: eventType, event: event @@ -78,12 +87,14 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { self.globalEventTap = globalEventTap self.globalEventTapRunLoopSource = globalEventTapRunLoopSource + isRunning = true CFRunLoopAddSource(CFRunLoopGetMain(), globalEventTapRunLoopSource, .commonModes) CGEvent.tapEnable(tap: globalEventTap, enable: true) } func stop() { + isRunning = false isShortcutCurrentlyPressed = false if let globalEventTapRunLoopSource { diff --git a/leanring-buddy/MenuBarPanelManager.swift b/leanring-buddy/MenuBarPanelManager.swift index e5eb98de7..2beaa1362 100644 --- a/leanring-buddy/MenuBarPanelManager.swift +++ b/leanring-buddy/MenuBarPanelManager.swift @@ -30,6 +30,10 @@ final class MenuBarPanelManager: NSObject { private var panel: NSPanel? private var clickOutsideMonitor: Any? private var dismissPanelObserver: NSObjectProtocol? + /// Incremented each time the panel is shown. The asyncAfter dismissal block + /// captures this value and skips hiding if the panel has been re-shown since + /// the click that triggered the dismissal, preventing stale dismissals. + private var panelShowGeneration = 0 private let companionManager: CompanionManager private let panelWidth: CGFloat = 320 @@ -133,6 +137,7 @@ final class MenuBarPanelManager: NSObject { positionPanelBelowStatusItem() + panelShowGeneration += 1 panel?.makeKeyAndOrderFront(nil) panel?.orderFrontRegardless() installClickOutsideMonitor() @@ -220,8 +225,11 @@ final class MenuBarPanelManager: NSObject { // Delay dismissal slightly to avoid closing the panel when // a system permission dialog appears (e.g. microphone access). + let generationWhenClicked = self.panelShowGeneration DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { guard panel.isVisible else { return } + // If the panel was re-shown since this click, don't dismiss it. + guard self.panelShowGeneration == generationWhenClicked else { return } // If permissions aren't all granted yet, a system dialog // may have focus — don't dismiss during onboarding. diff --git a/leanring-buddy/OverlayWindow.swift b/leanring-buddy/OverlayWindow.swift index 884ebcbfb..bf25d11cf 100644 --- a/leanring-buddy/OverlayWindow.swift +++ b/leanring-buddy/OverlayWindow.swift @@ -130,6 +130,11 @@ struct BlueCursorView: View { @State private var bubbleOpacity: Double = 1.0 @State private var cursorOpacity: Double = 0.0 + /// Tracks whether the view is still on screen. All async callbacks check this + /// before touching @State to prevent use-after-free when the overlay disappears + /// while animations (DispatchQueue.asyncAfter, Timer) are still pending. + @State private var isViewActive = true + // MARK: - Buddy Navigation State /// The buddy's current behavioral mode (following cursor, navigating, or pointing). @@ -152,6 +157,10 @@ struct BlueCursorView: View { /// Invalidated when the flight completes, is canceled, or the view disappears. @State private var navigationAnimationTimer: Timer? + /// Timer for the welcome character-streaming animation. Invalidated in + /// onDisappear to prevent writing to deallocated @State storage. + @State private var welcomeAnimationTimer: Timer? + /// Scale factor applied to the buddy triangle during flight. Grows to ~1.3x /// at the midpoint of the arc and shrinks back to 1.0x on landing, creating /// an energetic "swooping" feel. @@ -294,6 +303,63 @@ struct BlueCursorView: View { } } + // Step-by-step guidance panel — shown when a multi-step guide is active. + if isCursorOnThisScreen && companionManager.isStepByStepActive && !companionManager.currentStepInstruction.isEmpty { + VStack(alignment: .leading, spacing: 4) { + // Step header with count + Text("Step \(companionManager.currentStepIndex + 1) of \(companionManager.totalSteps)") + .font(.system(size: 9, weight: .bold, design: .rounded)) + .foregroundColor(DS.Colors.overlayCursorBlue.opacity(0.8)) + + // Instruction text + Text(companionManager.currentStepInstruction) + .font(.system(size: 12, weight: .semibold)) + .foregroundColor(.white) + + // Progress bar + GeometryReader { geo in + ZStack(alignment: .leading) { + RoundedRectangle(cornerRadius: 2, style: .continuous) + .fill(Color.white.opacity(0.15)) + .frame(height: 3) + + RoundedRectangle(cornerRadius: 2, style: .continuous) + .fill(DS.Colors.overlayCursorBlue) + .frame( + width: geo.size.width * CGFloat(companionManager.totalSteps > 0 + ? Double(companionManager.currentStepIndex + 1) / Double(companionManager.totalSteps) + : 0), + height: 3 + ) + .animation(.easeOut(duration: 0.3), value: companionManager.currentStepIndex) + } + } + .frame(height: 3) + + // Advance hint + Text("press hotkey for next step") + .font(.system(size: 9, weight: .medium)) + .foregroundColor(DS.Colors.overlayCursorBlue.opacity(0.6)) + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .background( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .fill(Color.black.opacity(0.75)) + .overlay( + RoundedRectangle(cornerRadius: 8, style: .continuous) + .stroke(DS.Colors.overlayCursorBlue.opacity(0.3), lineWidth: 0.8) + ) + ) + .fixedSize() + .position( + x: cursorPosition.x + 50, + y: cursorPosition.y - 28 + ) + .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .transition(.opacity.combined(with: .scale(scale: 0.9))) + } + // Blue triangle cursor — shown when idle or while TTS is playing (responding). // All three states (triangle, waveform, spinner) stay in the view tree // permanently and cross-fade via opacity so SwiftUI doesn't remove/re-insert @@ -355,7 +421,8 @@ struct BlueCursorView: View { withAnimation(.easeIn(duration: 2.0)) { self.cursorOpacity = 1.0 } - DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in + guard let self, self.isViewActive else { return } self.bubbleOpacity = 0.0 startWelcomeAnimation() } @@ -364,7 +431,9 @@ struct BlueCursorView: View { } } .onDisappear { + isViewActive = false timer?.invalidate() + welcomeAnimationTimer?.invalidate() navigationAnimationTimer?.invalidate() companionManager.tearDownOnboardingVideo() } @@ -409,7 +478,8 @@ struct BlueCursorView: View { // MARK: - Cursor Tracking private func startTrackingCursor() { - timer = Timer.scheduledTimer(withTimeInterval: 0.016, repeats: true) { _ in + timer = Timer.scheduledTimer(withTimeInterval: 0.016, repeats: true) { [weak self] _ in + guard let self, self.isViewActive else { return } let mouseLocation = NSEvent.mouseLocation self.isCursorOnThisScreen = self.screenFrame.contains(mouseLocation) @@ -521,7 +591,8 @@ struct BlueCursorView: View { let arcHeight = min(distance * 0.2, 80.0) let controlPoint = CGPoint(x: midPoint.x, y: midPoint.y - arcHeight) - navigationAnimationTimer = Timer.scheduledTimer(withTimeInterval: frameInterval, repeats: true) { _ in + navigationAnimationTimer = Timer.scheduledTimer(withTimeInterval: frameInterval, repeats: true) { [weak self] _ in + guard let self, self.isViewActive else { return } currentFrame += 1 if currentFrame > totalFrames { @@ -588,12 +659,13 @@ struct BlueCursorView: View { ?? "right here!" streamNavigationBubbleCharacter(phrase: pointerPhrase, characterIndex: 0) { + guard self.isViewActive else { return } // All characters streamed — hold for 3 seconds, then fly back - DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { - guard self.buddyNavigationMode == .pointingAtTarget else { return } + DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { [weak self] in + guard let self, self.isViewActive, self.buddyNavigationMode == .pointingAtTarget else { return } self.navigationBubbleOpacity = 0.0 - DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { - guard self.buddyNavigationMode == .pointingAtTarget else { return } + DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in + guard let self, self.isViewActive, self.buddyNavigationMode == .pointingAtTarget else { return } self.startFlyingBackToCursor() } } @@ -622,7 +694,8 @@ struct BlueCursorView: View { } let characterDelay = Double.random(in: 0.03...0.06) - DispatchQueue.main.asyncAfter(deadline: .now() + characterDelay) { + DispatchQueue.main.asyncAfter(deadline: .now() + characterDelay) { [weak self] in + guard let self, self.isViewActive else { return } self.streamNavigationBubbleCharacter( phrase: phrase, characterIndex: characterIndex + 1, @@ -680,16 +753,20 @@ struct BlueCursorView: View { } var currentIndex = 0 - Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { timer in + welcomeAnimationTimer = Timer.scheduledTimer(withTimeInterval: 0.03, repeats: true) { [weak self] timer in + guard let self, self.isViewActive else { + timer.invalidate() + return + } guard currentIndex < self.fullWelcomeMessage.count else { timer.invalidate() - // Hold the text for 2 seconds, then fade it out - DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { [weak self] in + guard let self, self.isViewActive else { return } self.bubbleOpacity = 0.0 } - DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { + DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { [weak self] in + guard let self, self.isViewActive else { return } self.showWelcome = false - // Start the onboarding video right after the welcome text disappears self.companionManager.setupOnboardingVideo() } return @@ -816,7 +893,9 @@ class OverlayWindowManager { } /// Fades out overlay windows over `duration` seconds, then removes them. + /// Re-entrant: subsequent calls while a fade is in progress are no-ops. func fadeOutAndHideOverlay(duration: TimeInterval = 0.4) { + guard !overlayWindows.isEmpty else { return } let windowsToFade = overlayWindows overlayWindows.removeAll() diff --git a/leanring-buddy/StepByStepGuide.swift b/leanring-buddy/StepByStepGuide.swift new file mode 100644 index 000000000..18e80399d --- /dev/null +++ b/leanring-buddy/StepByStepGuide.swift @@ -0,0 +1,166 @@ +import Foundation + +/// A single step in a step-by-step visual guidance sequence. +struct GuidanceStep: Equatable { + let index: Int + let total: Int + let instruction: String + let rawPointCoordinate: CGPoint? + let elementLabel: String? + let screenNumber: Int? +} + +/// A multi-step guide parsed from Claude's response when the user asks +/// "how do I..." or "show me how to..." questions. +struct StepByStepGuide: Equatable { + let totalSteps: Int + let steps: [GuidanceStep] + + var currentStepIndex: Int = 0 + + var currentStep: GuidanceStep? { + guard currentStepIndex >= 0, currentStepIndex < steps.count else { return nil } + return steps[currentStepIndex] + } + + var isComplete: Bool { + currentStepIndex >= totalSteps + } + + var progress: Double { + guard totalSteps > 0 else { return 1.0 } + return Double(currentStepIndex) / Double(totalSteps) + } +} + +/// Set of phrases the user can say to advance to the next step. +enum StepAdvanceCommand: String, CaseIterable { + case next = "next" + case goOn = "go on" + case okay = "okay" + case ok = "ok" + case done = "done" + case `continue` = "continue" + case ready = "ready" + case gotIt = "got it" + + static func matches(_ transcript: String) -> Bool { + let trimmed = transcript.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + return allCases.contains { $0.rawValue == trimmed } + } +} + +/// Parsing result for a [POINT:...] tag within a single step. +private struct TagParseResult { + let spokenText: String + let coordinate: CGPoint? + let elementLabel: String? + let screenNumber: Int? +} + +enum StepByStepGuideParser { + + /// Parses a multi-step guide from Claude's response. + /// Expected format: + /// ``` + /// [GUIDE:3] + /// Click the Insert tab [POINT:500,30:insert tab] + /// ### + /// Click Chart in the ribbon [POINT:600,60:chart button] + /// ### + /// Select your chart type [POINT:700,200:chart type] + /// [END_GUIDE] + /// ``` + /// Returns nil if the response doesn't contain a guide. + static func parse(from responseText: String) -> (guide: StepByStepGuide, spokenText: String)? { + let pattern = #"\[GUIDE:(\d+)\]\s*(.*?)\[END_GUIDE\]"# + guard let regex = try? NSRegularExpression(pattern: pattern, options: [.dotMatchesLineSeparators]), + let match = regex.firstMatch(in: responseText, range: NSRange(responseText.startIndex..., in: responseText)) else { + return nil + } + + let fullMatchRange = Range(match.range, in: responseText)! + let spokenText = String(responseText[..= 3, + let totalRange = Range(match.range(at: 1), in: responseText), + let bodyRange = Range(match.range(at: 2), in: responseText), + let totalSteps = Int(responseText[totalRange]) else { + return nil + } + + let body = String(responseText[bodyRange]).trimmingCharacters(in: .whitespacesAndNewlines) + + let rawSteps = body.components(separatedBy: "###") + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + + guard !rawSteps.isEmpty else { return nil } + + var parsedSteps: [GuidanceStep] = [] + + for (index, rawStep) in rawSteps.enumerated() { + let tagResult = parsePointingTag(from: rawStep) + parsedSteps.append(GuidanceStep( + index: index, + total: totalSteps, + instruction: tagResult.spokenText, + rawPointCoordinate: tagResult.coordinate, + elementLabel: tagResult.elementLabel, + screenNumber: tagResult.screenNumber + )) + } + + if parsedSteps.count > totalSteps { + parsedSteps = Array(parsedSteps.prefix(totalSteps)) + } + + guard !parsedSteps.isEmpty else { return nil } + + let guide = StepByStepGuide( + totalSteps: totalSteps, + steps: parsedSteps + ) + + return (guide: guide, spokenText: spokenText) + } + + /// Parses a [POINT:x,y:label:screenN] or [POINT:none] tag from the end of a step's text. + /// Returns the text with the tag removed and the optional coordinate + label + screen number. + private static func parsePointingTag(from text: String) -> TagParseResult { + let pattern = #"\[POINT:(?:none|(\d+)\s*,\s*(\d+)(?::([^\]:\s][^\]:]*?))?(?::screen(\d+))?)\]"# + + guard let regex = try? NSRegularExpression(pattern: pattern, options: []), + let match = regex.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)) else { + return TagParseResult(spokenText: text, coordinate: nil, elementLabel: nil, screenNumber: nil) + } + + let tagRange = Range(match.range, in: text)! + let spokenText = String(text[..= 3, + let xRange = Range(match.range(at: 1), in: text), + let yRange = Range(match.range(at: 2), in: text), + let x = Double(text[xRange]), + let y = Double(text[yRange]) else { + return TagParseResult(spokenText: spokenText, coordinate: nil, elementLabel: "none", screenNumber: nil) + } + + var elementLabel: String? = nil + if match.numberOfRanges >= 4, let labelRange = Range(match.range(at: 3), in: text) { + elementLabel = String(text[labelRange]).trimmingCharacters(in: .whitespaces) + } + + var screenNumber: Int? = nil + if match.numberOfRanges >= 5, let screenRange = Range(match.range(at: 4), in: text) { + screenNumber = Int(text[screenRange]) + } + + return TagParseResult( + spokenText: spokenText, + coordinate: CGPoint(x: x, y: y), + elementLabel: elementLabel, + screenNumber: screenNumber + ) + } +} diff --git a/leanring-buddyTests/leanring_buddyTests.swift b/leanring-buddyTests/leanring_buddyTests.swift index 188fe7ae0..72fcdaa6e 100644 --- a/leanring-buddyTests/leanring_buddyTests.swift +++ b/leanring-buddyTests/leanring_buddyTests.swift @@ -1,15 +1,10 @@ -// -// leanring_buddyTests.swift -// leanring-buddyTests -// -// Created by thorfinn on 3/2/26. -// - import Testing @testable import leanring_buddy struct leanring_buddyTests { + // MARK: - Permission Request Tests (Existing) + @Test func firstPermissionRequestUsesSystemPromptOnly() async throws { let presentationDestination = WindowPositionManager.permissionRequestPresentationDestination( hasPermissionNow: false, @@ -37,4 +32,402 @@ struct leanring_buddyTests { #expect(shouldTreatPermissionAsGranted) } + // MARK: - StepByStepGuideParser Tests + + @Test func parseValidGuideReturnsCorrectSteps() async throws { + let response = """ + let me walk you through it + [GUIDE:3] + go to the file menu [POINT:50,10:file menu] + ### + click new project [POINT:200,150:new project] + ### + choose ios app [POINT:400,300:ios template] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.totalSteps == 3) + #expect(result?.guide.steps.count == 3) + #expect(result?.spokenText == "let me walk you through it") + } + + @Test func parseValidGuideFirstStepHasCorrectData() async throws { + let response = """ + [GUIDE:2] + click the insert tab [POINT:500,30:insert tab] + ### + choose chart type [POINT:700,200:chart type:screen2] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + let firstStep = result?.guide.steps[0] + #expect(firstStep?.index == 0) + #expect(firstStep?.total == 2) + #expect(firstStep?.instruction == "click the insert tab") + #expect(firstStep?.rawPointCoordinate != nil) + #expect(firstStep?.rawPointCoordinate?.x == 500) + #expect(firstStep?.rawPointCoordinate?.y == 30) + #expect(firstStep?.elementLabel == "insert tab") + #expect(firstStep?.screenNumber == nil) + } + + @Test func parseValidGuideSecondStepHasScreenNumber() async throws { + let response = """ + [GUIDE:2] + click the insert tab [POINT:500,30:insert tab] + ### + choose chart type [POINT:700,200:chart type:screen2] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + let secondStep = result?.guide.steps[1] + #expect(secondStep?.instruction == "choose chart type") + #expect(secondStep?.rawPointCoordinate?.x == 700) + #expect(secondStep?.rawPointCoordinate?.y == 200) + #expect(secondStep?.elementLabel == "chart type") + #expect(secondStep?.screenNumber == 2) + } + + @Test func parseGuideWithNoPointTag() async throws { + let response = """ + [GUIDE:2] + just think about what you want to create + ### + then open the app [POINT:100,100:app icon] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.steps.count == 2) + + let firstStep = result?.guide.steps[0] + #expect(firstStep?.instruction == "just think about what you want to create") + #expect(firstStep?.rawPointCoordinate == nil) + + let secondStep = result?.guide.steps[1] + #expect(secondStep?.instruction == "then open the app") + #expect(secondStep?.rawPointCoordinate != nil) + } + + @Test func parseGuideWithNonePoint() async throws { + let response = """ + [GUIDE:1] + go to settings [POINT:none] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.steps.count == 1) + + let step = result?.guide.steps[0] + #expect(step?.instruction == "go to settings") + #expect(step?.rawPointCoordinate == nil) + #expect(step?.elementLabel == "none") + } + + @Test func parseResponseWithoutGuideReturnsNil() async throws { + let response = "you should click the file menu up top [POINT:50,10:file menu]" + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result == nil) + } + + @Test func parseGuideWithExtraNewlinesAndSpaces() async throws { + let response = """ + okay! + + [GUIDE:2] + + step one here [POINT:100,100:step one] + + ### + + step two there [POINT:200,200:step two] + + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.steps.count == 2) + #expect(result?.spokenText == "okay!") + } + + @Test func parseGuideWithNoPreamble() async throws { + let response = """ + [GUIDE:1] + do the thing [POINT:300,300:the thing] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.steps.count == 1) + #expect(result?.spokenText == "") + } + + @Test func parseGuideWithMoreStepsThanDeclaredTruncates() async throws { + let response = """ + [GUIDE:2] + step one [POINT:100,100:one] + ### + step two [POINT:200,200:two] + ### + step three [POINT:300,300:three] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + #expect(result?.guide.steps.count == 2) + #expect(result?.guide.totalSteps == 2) + } + + @Test func parseEmptyGuideBodyReturnsNil() async throws { + let response = """ + [GUIDE:0] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result == nil) + } + + @Test func parseMalformedGuideMissingTotalReturnsNil() async throws { + let response = """ + [GUIDE:] + some step [POINT:100,100:test] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result == nil) + } + + @Test func parseGuideWithPointCoordinatesAcrossMultipleDigits() async throws { + let response = """ + [GUIDE:1] + find the button [POINT:1234,567:big button] + [END_GUIDE] + """ + + let result = StepByStepGuideParser.parse(from: response) + + #expect(result != nil) + let step = result?.guide.steps[0] + #expect(step?.rawPointCoordinate?.x == 1234) + #expect(step?.rawPointCoordinate?.y == 567) + #expect(step?.elementLabel == "big button") + } + + // MARK: - StepAdvanceCommand Tests + + @Test func advanceCommandMatchesBasicNext() { + #expect(StepAdvanceCommand.matches("next") == true) + } + + @Test func advanceCommandMatchesWithWhitespace() { + #expect(StepAdvanceCommand.matches(" next ") == true) + } + + @Test func advanceCommandMatchesCaseInsensitive() { + #expect(StepAdvanceCommand.matches("NEXT") == true) + #expect(StepAdvanceCommand.matches("Next") == true) + } + + @Test func advanceCommandMatchesAllVariants() { + #expect(StepAdvanceCommand.matches("next") == true) + #expect(StepAdvanceCommand.matches("go on") == true) + #expect(StepAdvanceCommand.matches("ok") == true) + #expect(StepAdvanceCommand.matches("okay") == true) + #expect(StepAdvanceCommand.matches("done") == true) + #expect(StepAdvanceCommand.matches("continue") == true) + #expect(StepAdvanceCommand.matches("ready") == true) + #expect(StepAdvanceCommand.matches("got it") == true) + } + + @Test func advanceCommandDoesNotMatchRandomPhrase() { + #expect(StepAdvanceCommand.matches("what's next") == false) + #expect(StepAdvanceCommand.matches("next step") == false) + #expect(StepAdvanceCommand.matches("show me how") == false) + #expect(StepAdvanceCommand.matches("") == false) + #expect(StepAdvanceCommand.matches(" ") == false) + } + + @Test func advanceCommandDoesNotMatchPartialWords() { + #expect(StepAdvanceCommand.matches("nexting") == false) + #expect(StepAdvanceCommand.matches("doned") == false) + #expect(StepAdvanceCommand.matches("continues") == false) + } + + // MARK: - GuidanceStep Model Tests + + @Test func guidanceStepInitializesCorrectly() { + let step = GuidanceStep( + index: 0, + total: 3, + instruction: "click the button", + rawPointCoordinate: CGPoint(x: 100, y: 200), + elementLabel: "button", + screenNumber: 1 + ) + + #expect(step.index == 0) + #expect(step.total == 3) + #expect(step.instruction == "click the button") + #expect(step.rawPointCoordinate?.x == 100) + #expect(step.rawPointCoordinate?.y == 200) + #expect(step.elementLabel == "button") + #expect(step.screenNumber == 1) + } + + @Test func guidanceStepWithoutOptionalFields() { + let step = GuidanceStep( + index: 2, + total: 5, + instruction: "just think about it", + rawPointCoordinate: nil, + elementLabel: nil, + screenNumber: nil + ) + + #expect(step.index == 2) + #expect(step.total == 5) + #expect(step.rawPointCoordinate == nil) + #expect(step.elementLabel == nil) + #expect(step.screenNumber == nil) + } + + // MARK: - StepByStepGuide Model Tests + + @Test func guideCurrentStepReturnsCorrectStep() { + let steps = [ + GuidanceStep(index: 0, total: 2, instruction: "first", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil), + GuidanceStep(index: 1, total: 2, instruction: "second", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil) + ] + var guide = StepByStepGuide(totalSteps: 2, steps: steps) + + #expect(guide.currentStep?.instruction == "first") + + guide.currentStepIndex = 1 + #expect(guide.currentStep?.instruction == "second") + } + + @Test func guideIsCompleteWhenPastLastStep() { + let steps = [ + GuidanceStep(index: 0, total: 1, instruction: "only step", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil) + ] + var guide = StepByStepGuide(totalSteps: 1, steps: steps) + + #expect(guide.isComplete == false) + + guide.currentStepIndex = 1 + #expect(guide.isComplete == true) + } + + @Test func guideProgressStartsAtZero() { + let steps = [ + GuidanceStep(index: 0, total: 3, instruction: "a", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil), + GuidanceStep(index: 1, total: 3, instruction: "b", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil), + GuidanceStep(index: 2, total: 3, instruction: "c", rawPointCoordinate: nil, elementLabel: nil, screenNumber: nil) + ] + let guide = StepByStepGuide(totalSteps: 3, steps: steps) + + #expect(guide.progress == 0.0) + } + + @Test func guideCurrentStepIsNilForEmptySteps() { + let guide = StepByStepGuide(totalSteps: 0, steps: []) + + #expect(guide.currentStep == nil) + #expect(guide.isComplete == false) + } + + @Test func guideProgressPreventsDivisionByZero() { + let guide = StepByStepGuide(totalSteps: 0, steps: []) + + #expect(guide.progress == 1.0) + } + + // MARK: - CompanionManager Point Tag Parsing Tests + + @Test func parsePointingCoordinatesBasicPoint() { + let response = "click the button up top [POINT:500,30:the button]" + + let result = CompanionManager.parsePointingCoordinates(from: response) + + #expect(result.spokenText == "click the button up top") + #expect(result.coordinate?.x == 500) + #expect(result.coordinate?.y == 30) + #expect(result.elementLabel == "the button") + #expect(result.screenNumber == nil) + } + + @Test func parsePointingCoordinatesNoneTag() { + let response = "that's a general question [POINT:none]" + + let result = CompanionManager.parsePointingCoordinates(from: response) + + #expect(result.spokenText == "that's a general question") + #expect(result.coordinate == nil) + #expect(result.elementLabel == "none") + } + + @Test func parsePointingCoordinatesWithScreenNumber() { + let response = "over on your other screen [POINT:400,300:terminal:screen2]" + + let result = CompanionManager.parsePointingCoordinates(from: response) + + #expect(result.spokenText == "over on your other screen") + #expect(result.coordinate?.x == 400) + #expect(result.coordinate?.y == 300) + #expect(result.elementLabel == "terminal") + #expect(result.screenNumber == 2) + } + + @Test func parsePointingCoordinatesNoTagReturnsFullText() { + let response = "just a regular response with no point tag" + + let result = CompanionManager.parsePointingCoordinates(from: response) + + #expect(result.spokenText == response) + #expect(result.coordinate == nil) + #expect(result.elementLabel == nil) + #expect(result.screenNumber == nil) + } + + @Test func parsePointingCoordinatesEmptyString() { + let result = CompanionManager.parsePointingCoordinates(from: "") + + #expect(result.spokenText == "") + #expect(result.coordinate == nil) + #expect(result.elementLabel == nil) + } + + @Test func stepByStepGuideEquality() { + let step1 = GuidanceStep(index: 0, total: 1, instruction: "do it", rawPointCoordinate: CGPoint(x: 10, y: 20), elementLabel: "test", screenNumber: nil) + let step2 = GuidanceStep(index: 0, total: 1, instruction: "do it", rawPointCoordinate: CGPoint(x: 10, y: 20), elementLabel: "test", screenNumber: nil) + let step3 = GuidanceStep(index: 0, total: 1, instruction: "do something else", rawPointCoordinate: CGPoint(x: 10, y: 20), elementLabel: "test", screenNumber: nil) + + #expect(step1 == step2) + #expect(step1 != step3) + } }