diff --git a/.gitignore b/.gitignore index bf1327903..de4a1702c 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,4 @@ apps/*/tooling/node-deps/node_modules/ # Vitest / Browser testing .vitest-attachments/ **/__screenshots__/ +.amp/plugins/emdash-hook.ts diff --git a/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.test.ts b/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.test.ts index 4c2677fae..dca556416 100644 --- a/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.test.ts +++ b/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.test.ts @@ -1,8 +1,14 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Pty, PtyExitInfo } from '@main/core/pty/pty'; +import { events } from '@main/lib/events'; +import { conversationInitialPromptInjectionFailedChannel } from '@shared/core/conversations/conversationEvents'; import type { Conversation } from '@shared/core/conversations/conversations'; import { scheduleInitialPromptInjection } from './keystroke-injection'; +vi.mock('@main/lib/events', () => ({ + events: { emit: vi.fn() }, +})); + function makeConversation(providerId: Conversation['providerId']): Conversation { return { id: 'conv-1', @@ -47,13 +53,14 @@ function makePty(): { describe('scheduleInitialPromptInjection', () => { beforeEach(() => { vi.useFakeTimers(); + vi.mocked(events.emit).mockClear(); }); afterEach(() => { vi.useRealTimers(); }); - it('injects after PTY output goes quiet', () => { + it('injects after provider output goes quiet', () => { const { pty, write, emitData } = makePty(); scheduleInitialPromptInjection({ pty, @@ -62,7 +69,7 @@ describe('scheduleInitialPromptInjection', () => { isResuming: false, }); - emitData('booting...'); + emitData('Hermes booting...'); vi.advanceTimersByTime(200); emitData('still booting...'); expect(write).not.toHaveBeenCalled(); @@ -71,7 +78,7 @@ describe('scheduleInitialPromptInjection', () => { expect(write).toHaveBeenCalledExactlyOnceWith('Fix the bug\r'); }); - it('falls back to a max wait when no output ever arrives', () => { + it('does not inject when no provider output ever arrives', () => { const { pty, write } = makePty(); scheduleInitialPromptInjection({ pty, @@ -81,7 +88,7 @@ describe('scheduleInitialPromptInjection', () => { }); vi.advanceTimersByTime(15_000); - expect(write).toHaveBeenCalledExactlyOnceWith('Fix the bug\r'); + expect(write).not.toHaveBeenCalled(); }); it('wraps multi-line prompts in bracketed paste sequences', () => { @@ -93,11 +100,55 @@ describe('scheduleInitialPromptInjection', () => { isResuming: false, }); - emitData('ready'); + emitData('Hermes ready'); vi.advanceTimersByTime(900); expect(write).toHaveBeenCalledExactlyOnceWith('\x1b[200~line one\nline two\x1b[201~\r'); }); + it('does not inject into generic shell output', () => { + const { pty, write, emitData } = makePty(); + scheduleInitialPromptInjection({ + pty, + conversation: makeConversation('hermes'), + initialPrompt: 'Fix the bug', + isResuming: false, + }); + + emitData('Last login: Fri Jun 12\n% '); + vi.advanceTimersByTime(20_000); + expect(write).not.toHaveBeenCalled(); + }); + + it('cancels injection when shell failures appear before provider output', () => { + const { pty, write, emitData } = makePty(); + scheduleInitialPromptInjection({ + pty, + conversation: makeConversation('hermes'), + initialPrompt: 'Fix the bug', + isResuming: false, + }); + + emitData('zsh:1: command not found: hermes\n% '); + emitData('Hermes ready'); + vi.advanceTimersByTime(900); + expect(write).not.toHaveBeenCalled(); + }); + + it('cancels injection when shell failure output is split across chunks', () => { + const { pty, write, emitData } = makePty(); + scheduleInitialPromptInjection({ + pty, + conversation: makeConversation('hermes'), + initialPrompt: 'Fix the bug', + isResuming: false, + }); + + emitData('zsh: '); + emitData('command not found: hermes\n% '); + vi.advanceTimersByTime(900); + expect(write).not.toHaveBeenCalled(); + }); + it('does nothing for OpenCode because its initial prompt is passed with --prompt', () => { const { pty, write, emitData } = makePty(); scheduleInitialPromptInjection({ @@ -154,7 +205,7 @@ describe('scheduleInitialPromptInjection', () => { expect(write).not.toHaveBeenCalled(); }); - it('cancels injection when the PTY exits before idle', () => { + it('emits a user-visible failure event when the PTY exits before ready output', () => { const { pty, write, emitData, emitExit } = makePty(); scheduleInitialPromptInjection({ pty, @@ -167,5 +218,34 @@ describe('scheduleInitialPromptInjection', () => { emitExit(); vi.advanceTimersByTime(20_000); expect(write).not.toHaveBeenCalled(); + expect(events.emit).toHaveBeenCalledWith(conversationInitialPromptInjectionFailedChannel, { + conversationId: 'conv-1', + taskId: 'task-1', + projectId: 'proj-1', + providerId: 'hermes', + }); + }); + + it('emits a user-visible failure event when the PTY exits after ready output but before injection', () => { + const { pty, write, emitData, emitExit } = makePty(); + scheduleInitialPromptInjection({ + pty, + conversation: makeConversation('hermes'), + initialPrompt: 'Fix the bug', + isResuming: false, + }); + + emitData('Hermes ready'); + vi.advanceTimersByTime(200); + emitExit(); + vi.advanceTimersByTime(20_000); + + expect(write).not.toHaveBeenCalled(); + expect(events.emit).toHaveBeenCalledWith(conversationInitialPromptInjectionFailedChannel, { + conversationId: 'conv-1', + taskId: 'task-1', + projectId: 'proj-1', + providerId: 'hermes', + }); }); }); diff --git a/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.ts b/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.ts index 187833d80..51b106f5e 100644 --- a/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.ts +++ b/apps/emdash-desktop/src/main/core/conversations/impl/keystroke-injection.ts @@ -1,13 +1,26 @@ import type { Pty } from '@main/core/pty/pty'; +import { events } from '@main/lib/events'; import { log } from '@main/lib/logger'; import { getProvider } from '@shared/core/agents/agent-provider-registry'; +import { conversationInitialPromptInjectionFailedChannel } from '@shared/core/conversations/conversationEvents'; import type { Conversation } from '@shared/core/conversations/conversations'; import { buildPromptInjectionPayload } from '@shared/prompt-injection'; // Inject only after the TUI has produced output and stayed idle for a beat; // fixed delays race the agent's startup (auth, sync, model load). const QUIET_PERIOD_MS = 800; -const MAX_WAIT_MS = 15_000; + +const KEYSTROKE_READY_OUTPUT: Partial> = { + grok: /\b(grok|xai|x\.ai)\b/i, + hermes: /\bhermes\b/i, + kimi: /\bkimi\b/i, + jules: /\bjules\b/i, + letta: /\bletta\b/i, +}; + +const SHELL_FAILURE_OUTPUT = + /(?:^|\n)(?:zsh|bash|sh|fish)(?::\d+)?(?::\s+|\s+)(?:command not found|parse error|no such file or directory)\b/i; +const OUTPUT_BUFFER_MAX = 4096; export function scheduleInitialPromptInjection(args: { pty: Pty; @@ -21,6 +34,9 @@ export function scheduleInitialPromptInjection(args: { const provider = getProvider(args.conversation.providerId); if (!provider?.useKeystrokeInjection) return; + const readyOutput = KEYSTROKE_READY_OUTPUT[args.conversation.providerId]; + if (!readyOutput) return; + const payload = buildPromptInjectionPayload({ providerId: args.conversation.providerId, text: args.initialPrompt, @@ -28,13 +44,14 @@ export function scheduleInitialPromptInjection(args: { let injected = false; let sawAnyOutput = false; + let sawReadyOutput = false; + let outputBuffer = ''; let quietTimer: ReturnType | null = null; const inject = () => { if (injected) return; injected = true; if (quietTimer) clearTimeout(quietTimer); - clearTimeout(maxWaitTimer); try { const submitSequence = provider.keystrokeSubmitSequence ?? '\r'; const submitDelayMs = provider.keystrokeSubmitDelayMs; @@ -53,11 +70,21 @@ export function scheduleInitialPromptInjection(args: { } }; - const maxWaitTimer = setTimeout(inject, MAX_WAIT_MS); - - args.pty.onData(() => { + args.pty.onData((data) => { if (injected) return; sawAnyOutput = true; + outputBuffer = `${outputBuffer}${data}`.slice(-OUTPUT_BUFFER_MAX); + if (SHELL_FAILURE_OUTPUT.test(outputBuffer)) { + injected = true; + if (quietTimer) clearTimeout(quietTimer); + log.warn('ConversationProvider: shell output detected before initial prompt injection', { + providerId: args.conversation.providerId, + conversationId: args.conversation.id, + }); + return; + } + sawReadyOutput = sawReadyOutput || readyOutput.test(outputBuffer); + if (!sawReadyOutput) return; if (quietTimer) clearTimeout(quietTimer); quietTimer = setTimeout(inject, QUIET_PERIOD_MS); }); @@ -66,12 +93,18 @@ export function scheduleInitialPromptInjection(args: { const promptWasInjected = injected; injected = true; if (quietTimer) clearTimeout(quietTimer); - clearTimeout(maxWaitTimer); if (!promptWasInjected) { log.warn('ConversationProvider: PTY exited before initial prompt could be injected', { providerId: args.conversation.providerId, conversationId: args.conversation.id, sawAnyOutput, + sawReadyOutput, + }); + events.emit(conversationInitialPromptInjectionFailedChannel, { + conversationId: args.conversation.id, + taskId: args.conversation.taskId, + projectId: args.conversation.projectId, + providerId: args.conversation.providerId, }); } }); diff --git a/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-bar.tsx b/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-bar.tsx new file mode 100644 index 000000000..3aa4f2ec1 --- /dev/null +++ b/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-bar.tsx @@ -0,0 +1,363 @@ +import { ChevronDown, Send, Trash2 } from 'lucide-react'; +import { observer } from 'mobx-react-lite'; +import { useState } from 'react'; +import { getProjectSshConnectionId } from '@renderer/features/projects/stores/project-selectors'; +import { + formatConversationTitleForDisplay, + nextDefaultConversationTitle, +} from '@renderer/features/tasks/conversations/conversation-title-utils'; +import { useEffectiveProvider } from '@renderer/features/tasks/conversations/use-effective-provider'; +import { useAgentAutoApproveDefaults } from '@renderer/features/tasks/hooks/useAgentAutoApproveDefaults'; +import { + useConversations, + useTaskViewContext, + useWorkspaceViewModel, +} from '@renderer/features/tasks/task-view-context'; +import AgentLogo from '@renderer/lib/components/agent-logo'; +import { toast } from '@renderer/lib/hooks/use-toast'; +import { rpc } from '@renderer/lib/ipc'; +import { pastePromptInjection } from '@renderer/lib/pty/prompt-injection'; +import { appState } from '@renderer/lib/stores/app-state'; +import { Button } from '@renderer/lib/ui/button'; +import { + Combobox, + ComboboxCollection, + ComboboxContent, + ComboboxEmpty, + ComboboxGroup, + ComboboxInput, + ComboboxItem, + ComboboxLabel, + ComboboxList, + ComboboxTrigger, +} from '@renderer/lib/ui/combobox'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from '@renderer/lib/ui/dropdown-menu'; +import { agentConfig } from '@renderer/utils/agentConfig'; +import { + AGENT_PROVIDER_IDS, + type AgentProviderId, +} from '@shared/core/agents/agent-provider-registry'; +import { buildAnnotationPrompt } from './browser-annotation-prompt'; +import type { BrowserAnnotationState } from './browser-annotation-store'; + +type AnnotationSendTarget = + | { kind: 'existing'; conversationId: string } + | { kind: 'new'; providerId: AgentProviderId }; + +type TargetOption = { + value: string; + label: string; + providerId: AgentProviderId; + target: AnnotationSendTarget; +}; + +type TargetGroup = { value: string; label: string; items: TargetOption[] }; + +function targetValue(target: AnnotationSendTarget): string { + return target.kind === 'existing' ? `conv:${target.conversationId}` : `new:${target.providerId}`; +} + +export const BrowserAnnotationBar = observer(function BrowserAnnotationBar({ + state, + onSent, + onClearAll, + onRemoveAnnotation, +}: { + state: BrowserAnnotationState; + onSent: () => void; + onClearAll: () => void; + onRemoveAnnotation: (token: number, epoch: number) => void; +}) { + const { projectId, taskId } = useTaskViewContext(); + const conversations = useConversations(); + const taskView = useWorkspaceViewModel(); + const connectionId = getProjectSshConnectionId(projectId); + const { providerId: defaultNewProviderId, createDisabled } = useEffectiveProvider(connectionId); + const autoApproveDefaults = useAgentAutoApproveDefaults(); + const [target, setTarget] = useState(null); + const [targetPickerOpen, setTargetPickerOpen] = useState(false); + const [isSending, setIsSending] = useState(false); + + // Match the tabbar: only conversations with an open tab are editable targets here. + const openConversationOptions = new Map< + string, + { + id: string; + title: string; + providerId: AgentProviderId; + } + >(); + for (const group of taskView.tabGroupManager.groups) { + for (const tab of group.tabManager.resolvedTabs) { + if (tab.kind !== 'conversation' || openConversationOptions.has(tab.conversationId)) continue; + openConversationOptions.set(tab.conversationId, { + id: tab.conversationId, + title: tab.store.data.title, + providerId: tab.store.data.providerId, + }); + } + } + const options = Array.from(openConversationOptions.values()); + + const dependencyResource = connectionId + ? appState.dependencies.getRemote(connectionId) + : appState.dependencies.local; + const installedProviders = AGENT_PROVIDER_IDS.filter( + (id) => dependencyResource.data?.[id]?.status === 'available' + ); + + const resolvedTarget: AnnotationSendTarget | null = (() => { + if (target?.kind === 'existing' && options.some((o) => o.id === target.conversationId)) { + return target; + } + if (target?.kind === 'new' && installedProviders.includes(target.providerId)) { + return target; + } + if (options[0]) return { kind: 'existing', conversationId: options[0].id }; + if (defaultNewProviderId && !createDisabled) { + return { kind: 'new', providerId: defaultNewProviderId }; + } + return null; + })(); + + const conversationOptions: TargetOption[] = options.map((option) => ({ + value: `conv:${option.id}`, + label: formatConversationTitleForDisplay(option.providerId, option.title), + providerId: option.providerId, + target: { kind: 'existing', conversationId: option.id }, + })); + const providerOptions: TargetOption[] = installedProviders.map((providerId) => ({ + value: `new:${providerId}`, + label: agentConfig[providerId]?.name ?? providerId, + providerId, + target: { kind: 'new', providerId }, + })); + const targetGroups: TargetGroup[] = [ + ...(conversationOptions.length + ? [{ value: 'conversations', label: 'Agents', items: conversationOptions }] + : []), + ...(providerOptions.length + ? [{ value: 'providers', label: 'New agent', items: providerOptions }] + : []), + ]; + const selectedTargetOption = resolvedTarget + ? [...conversationOptions, ...providerOptions].find( + (option) => option.value === targetValue(resolvedTarget) + ) + : undefined; + + const count = state.annotations.length; + const canSend = count > 0 && !isSending && resolvedTarget !== null; + + const sendToExisting = async (conversationId: string) => { + const session = conversations.sessions.get(conversationId); + const conversation = conversations.conversations.get(conversationId); + if (!session || !conversation) throw new Error('Conversation unavailable'); + + // The target may be dehydrated (tab closed → PTY killed). Hydrating is + // idempotent: the session supervisor ignores it when the PTY is running. + await conversations.hydrateConversation(conversationId); + + const text = buildAnnotationPrompt(state.annotations.slice()); + let delivered = true; + const sendInput = async (data: string) => { + const result = await rpc.pty.sendInput(session.sessionId, data); + if (!result.success) delivered = false; + return result; + }; + await pastePromptInjection({ + providerId: conversation.data.providerId, + text, + forceBracketedPaste: true, + sendInput, + }); + if (delivered) { + const submit = await rpc.pty.sendInput(session.sessionId, '\r'); + delivered = submit.success; + } + if (!delivered) throw new Error('Agent session is not running'); + conversation.setWorking(); + taskView.tabGroupManager.openConversation(conversationId); + taskView.setFocusedRegion('main'); + }; + + const sendToNewConversation = async (providerId: AgentProviderId) => { + const text = buildAnnotationPrompt(state.annotations.slice(), { mode: 'initial' }); + const title = nextDefaultConversationTitle( + providerId, + Array.from(conversations.conversations.values(), (store) => store.data) + ); + const conversationId = crypto.randomUUID(); + await conversations.createConversation({ + id: conversationId, + projectId, + taskId, + provider: providerId, + title, + autoApprove: autoApproveDefaults.getDefault(providerId), + initialPrompt: text, + }); + taskView.tabGroupManager.openConversation(conversationId); + taskView.setFocusedRegion('main'); + }; + + const send = async () => { + if (count === 0 || isSending || !resolvedTarget) return; + setIsSending(true); + try { + if (resolvedTarget.kind === 'existing') { + await sendToExisting(resolvedTarget.conversationId); + } else { + await sendToNewConversation(resolvedTarget.providerId); + } + onSent(); + } catch (error) { + toast({ + title: 'Failed to send annotations to agent', + description: error instanceof Error ? error.message : String(error), + variant: 'destructive', + }); + } finally { + setIsSending(false); + } + }; + + if (count === 0) return null; + + return ( +
+ + + } + > + {count} {count === 1 ? 'annotation' : 'annotations'} + + + + {state.annotations.map((annotation, index) => ( + onRemoveAnnotation(annotation.token, annotation.epoch)} + > + + {index + 1} + + {annotation.comment} + + + ))} + + Clear all annotations + + +
+ { + if (item) setTarget(item.target); + setTargetPickerOpen(false); + }} + open={targetPickerOpen} + onOpenChange={setTargetPickerOpen} + isItemEqualToValue={(a: TargetOption, b: TargetOption) => a.value === b.value} + filter={(item: TargetOption, query) => + item.label.toLowerCase().includes(query.toLowerCase()) + } + autoHighlight + > + + {resolvedTarget ? ( + + ) : ( + No agent available + )} + + + + + + {(group: TargetGroup) => ( + + {group.label} + + {(item: TargetOption) => ( + + + + )} + + + )} + + No open agents or installed agents + + + +
+ ); +}); + +function SendTargetLabel({ + target, + conversations, +}: { + target: AnnotationSendTarget; + conversations: Array<{ id: string; title: string; providerId: AgentProviderId }>; +}) { + if (target.kind === 'existing') { + const conversation = conversations.find((option) => option.id === target.conversationId); + if (!conversation) return null; + return ( + + ); + } + return ( + + ); +} + +function ProviderLabel({ providerId, label }: { providerId: AgentProviderId; label: string }) { + const config = agentConfig[providerId]; + return ( + + {config && ( + + )} + {label} + + ); +} diff --git a/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-overlay.tsx b/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-overlay.tsx new file mode 100644 index 000000000..36444031c --- /dev/null +++ b/apps/emdash-desktop/src/renderer/features/browser/browser-annotation-overlay.tsx @@ -0,0 +1,177 @@ +import { observer } from 'mobx-react-lite'; +import { useEffect, useRef, useState } from 'react'; +import { Button } from '@renderer/lib/ui/button'; +import { Shortcut } from '@renderer/lib/ui/shortcut'; +import { Textarea } from '@renderer/lib/ui/textarea'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@renderer/lib/ui/tooltip'; +import type { BrowserAnnotationState } from './browser-annotation-store'; + +const DRAFT_CARD_WIDTH = 320; +const DRAFT_CARD_ESTIMATED_HEIGHT = 170; + +export const BrowserAnnotationOverlay = observer(function BrowserAnnotationOverlay({ + state, + zoomFactor, + onCommitDraft, + onCancelDraft, + onRemoveAnnotation, +}: { + state: BrowserAnnotationState; + /** Page zoom — picker rects are in page CSS pixels, the overlay in embedder pixels. */ + zoomFactor: number; + onCommitDraft: (comment: string) => void; + onCancelDraft: () => void; + onRemoveAnnotation: (token: number, epoch: number) => void; +}) { + const containerRef = useRef(null); + const [containerSize, setContainerSize] = useState({ width: 0, height: 0 }); + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + const sync = () => { + setContainerSize({ width: container.clientWidth, height: container.clientHeight }); + }; + sync(); + const resizeObserver = new ResizeObserver(sync); + resizeObserver.observe(container); + return () => resizeObserver.disconnect(); + }, []); + + const draft = state.draft; + const draftRect = draft ? scaleRect(draft.element.rect, zoomFactor) : null; + + return ( +
+ {state.markers.map((marker) => { + const rect = scaleRect(marker.rect, zoomFactor); + return ( + + onRemoveAnnotation(marker.token, state.navigationEpoch)} + > + {marker.ordinal} + + } + /> + + {marker.comment} + Click to remove + + + ); + })} + {draft && draftRect && ( + <> +
+ + + )} +
+ ); +}); + +function scaleRect( + rect: { x: number; y: number; width: number; height: number }, + zoomFactor: number +): { x: number; y: number; width: number; height: number } { + if (zoomFactor === 1) return rect; + return { + x: rect.x * zoomFactor, + y: rect.y * zoomFactor, + width: rect.width * zoomFactor, + height: rect.height * zoomFactor, + }; +} + +function draftCardPosition( + rect: { x: number; y: number; width: number; height: number }, + container: { width: number; height: number } +): { left: number; top: number } { + const left = Math.max(8, Math.min(rect.x, container.width - DRAFT_CARD_WIDTH - 8)); + const below = rect.y + rect.height + 8; + const top = + below + DRAFT_CARD_ESTIMATED_HEIGHT > container.height + ? Math.max(8, rect.y - DRAFT_CARD_ESTIMATED_HEIGHT - 8) + : below; + return { left, top }; +} + +function DraftCommentCard({ + position, + onCommit, + onCancel, +}: { + position: { left: number; top: number }; + onCommit: (comment: string) => void; + onCancel: () => void; +}) { + const [comment, setComment] = useState(''); + const canCommit = comment.trim().length > 0; + + const commit = () => { + if (canCommit) onCommit(comment); + }; + + return ( +
+