From 568ca1b41285e94a57ff8d755493bcfb4e50da4a Mon Sep 17 00:00:00 2001 From: Shahaf Antwarg Date: Sat, 22 Aug 2026 00:47:53 +0300 Subject: [PATCH] feat(desktop): Restore Telegram image sending and add inbound media Image sending was dropped from main in 1224a4c2 as part of a discovery refactor cleanup. Restore it and extend the bridge to handle media in both directions: - /image generates via the best openai-images seller; drafts a Generating status and delivers the result as a photo (document fallback for oversized or non-inline formats) - Assistant replies with image file blocks now send those images instead of collapsing to Done (no text reply) - Incoming photos and documents are downloaded, stored as conversation attachments, and passed through the same multimodal pipeline as the app's paper-clip attachments; captions become the message text - Voice/video/audio get an explicit unsupported notice instead of the generic text-only error - Image-only sellers are excluded from the bot's text-model picker and default-route fallback; /stop also cancels image generations --- CHANGELOG.md | 2 + apps/desktop/src/main/chat/engine.ts | 31 +- apps/desktop/src/main/telegram/bot-api.ts | 133 ++++++++- apps/desktop/src/main/telegram/bridge.ts | 284 ++++++++++++++++++- apps/desktop/src/main/telegram/media.test.ts | 114 ++++++++ apps/desktop/src/main/telegram/media.ts | 141 +++++++++ 6 files changed, 677 insertions(+), 28 deletions(-) create mode 100644 apps/desktop/src/main/telegram/media.test.ts create mode 100644 apps/desktop/src/main/telegram/media.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a486f38..6ce5ee753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project uses selective package publishing. Each release entry lists the pub ### Changed +- The desktop Telegram bot now handles images and documents in both directions. Generated images (via the new `/image ` command, which routes to the best available `openai-images` seller) are delivered to the owner as Telegram photos — with an automatic fallback to file attachment for oversized or exotic formats — instead of being dropped with a "no text reply" placeholder. Photos and documents sent to the bot are downloaded, stored as conversation attachments, and fed to the agent through the same multimodal pipeline as the app's paper-clip attachments (captions become the message text; unsupported media such as voice notes get a clear explanation instead of a generic error). Image-only sellers no longer appear in the bot's text-model picker or default-route fallback, and `/stop` now also cancels in-flight image generations. + - Desktop Help now explains the Virtual Private Router as a VPN for AI and adds practical guidance for built-in chat, connected apps, per-conversation model and seller selection, floating-window controls, routing, credits, rewards, the local API, and troubleshooting. Each subject links to a new comprehensive VPR guide or relevant supporting source. - Desktop model prices and the **Free** badge now reflect only sellers the routing trust gate can actually select, so a low-trust seller's $0 or teaser offer no longer advertises a price a send would never be billed at (picking such a "Free" model previously failed with a 402 for unfunded users). Savings percentages in the model dropdown, model page, and model lists now consistently compare that eligible live price with the retail reference baseline. The Recommended list's free ride-along models are trust-gated the same way, and the first-run default model is chosen from a free-model priority list (DeepSeek Flash, MiniMax, Haiku, Qwen, Nemotron, Gemma, Mistral Large), falling back to the free offer from the highest-trust seller. diff --git a/apps/desktop/src/main/chat/engine.ts b/apps/desktop/src/main/chat/engine.ts index 29ea1798c..360142839 100644 --- a/apps/desktop/src/main/chat/engine.ts +++ b/apps/desktop/src/main/chat/engine.ts @@ -69,7 +69,7 @@ import { projectPersistedConversationRoute, } from './conversation-store.js'; import { createStreamingRunner } from './streaming-run.js'; -import { generateChatImage } from './image-generation.js'; +import { generateChatImage, type GenerateChatImageResult } from './image-generation.js'; import type { ActiveRun, ChatStreamErrorPayload, @@ -93,7 +93,13 @@ export type PiChatEngine = { sendMessageStream( conversationId: string, userMessage: string, - options?: { service?: string; peerId?: string; permissionMode?: ChatPermissionMode }, + options?: { + service?: string; + peerId?: string; + permissionMode?: ChatPermissionMode; + /** Prepared attachments (images, documents) — same pipeline as the app's paper-clip flow. */ + attachments?: PreparedChatAttachment[]; + }, ): Promise<{ ok: boolean; error?: string; stopReason?: ChatStreamStopReason }>; abort(conversationId?: string): Promise; /** Resolves a pending tool approval; returns false when the id is unknown (already decided). */ @@ -114,6 +120,18 @@ export type PiChatEngine = { getModelPicker(): ModelPickerSnapshot | null; /** Rebinds a conversation's model/peer — same semantics as the chat UI's model dropdown. */ selectPeer(request: ChatPeerSelectionRequest): Promise<{ ok: boolean; error?: string }>; + /** + * Runs a one-shot image generation against an `openai-images` service and + * persists the result into the conversation — the same flow as the chat + * UI's image button. The generated image lands in the attachment store; + * the returned assistant message carries its file block. + */ + generateImage(request: { + conversationId: string; + prompt: string; + peerId: string; + service: string; + }): Promise; /** * Sets the buyer default route (the sticky "current model" new conversations * inherit) and tells the renderer so the UI selection follows suit. @@ -721,7 +739,7 @@ export function registerPiChatHandlers({ } }); - ipcMain.handle('chat:generate-image', async (_event, payload: unknown) => { + const generateImageRequest = async (payload: unknown): Promise => { const request = payload && typeof payload === 'object' ? payload as { conversationId?: unknown; prompt?: unknown; peerId?: unknown; moderation?: unknown; service?: unknown; sourceImageAttachmentId?: unknown } : {}; @@ -764,7 +782,9 @@ export function registerPiChatHandlers({ activeImageRunsByConversation.delete(conversationId); } } - }); + }; + + ipcMain.handle('chat:generate-image', async (_event, payload: unknown) => generateImageRequest(payload)); ipcMain.handle( 'chat:ai-send-stream', @@ -878,7 +898,7 @@ export function registerPiChatHandlers({ conversationId, userMessage, options?.service, - undefined, + options?.attachments, options?.peerId, options?.permissionMode, ), @@ -888,6 +908,7 @@ export function registerPiChatHandlers({ discoverServiceCatalog: discoverPolicyAllowedCatalog, getModelPicker: () => modelPickerSnapshot, selectPeer: applyPeerSelection, + generateImage: (request) => generateImageRequest(request), setDefaultRoute: async (peerId, service, provider) => { const result = await setBuyerDefaultRoute(peerId, service); sendToRenderer('chat:default-route-changed', { peerId, service, provider: provider ?? null }); diff --git a/apps/desktop/src/main/telegram/bot-api.ts b/apps/desktop/src/main/telegram/bot-api.ts index 45555e39a..0f18045b5 100644 --- a/apps/desktop/src/main/telegram/bot-api.ts +++ b/apps/desktop/src/main/telegram/bot-api.ts @@ -24,12 +24,39 @@ export type TgChat = { title?: string; }; +export type TgPhotoSize = { + file_id: string; + width: number; + height: number; + file_size?: number; +}; + +export type TgDocument = { + file_id: string; + file_name?: string; + mime_type?: string; + file_size?: number; +}; + +export type TgVoice = { + file_id: string; + duration: number; + mime_type?: string; + file_size?: number; +}; + export type TgMessage = { message_id: number; from?: TgUser; chat: TgChat; date: number; text?: string; + /** Caption on media messages (photos, documents, voice notes). */ + caption?: string; + /** Present on photo messages; ordered smallest → largest. */ + photo?: TgPhotoSize[]; + document?: TgDocument; + voice?: TgVoice; }; export type TgCallbackQuery = { @@ -60,6 +87,12 @@ export class TelegramApiError extends Error { } } +export type TgFile = { + file_id: string; + /** Relative path under https://api.telegram.org/file/bot/ — absent for files >20 MB. */ + file_path?: string; +}; + type TgResponse = { ok: boolean; result?: T; @@ -68,6 +101,24 @@ type TgResponse = { parameters?: { retry_after?: number }; }; +async function parseTgResponse(method: string, response: Response): Promise { + let body: TgResponse; + try { + body = await response.json() as TgResponse; + } catch { + throw new TelegramApiError(method, response.status, 'Invalid response from Telegram'); + } + if (!body.ok || body.result === undefined) { + throw new TelegramApiError( + method, + body.error_code ?? response.status, + body.description ?? 'Unknown error', + body.parameters?.retry_after ?? null, + ); + } + return body.result; +} + async function tgCall( token: string, method: string, @@ -88,21 +139,41 @@ async function tgCall( clearTimeout(timer); } - let body: TgResponse; + return parseTgResponse(method, response); +} + +/** File payload for the multipart upload methods (sendPhoto / sendDocument). */ +export type TgUploadFile = { + bytes: Uint8Array; + fileName: string; + contentType: string; +}; + +async function tgUpload( + token: string, + method: string, + fileField: string, + file: TgUploadFile, + params: Record, + timeoutMs = 120_000, +): Promise { + const form = new FormData(); + for (const [key, value] of Object.entries(params)) form.append(key, value); + form.append(fileField, new Blob([file.bytes], { type: file.contentType }), file.fileName); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + let response: Response; try { - body = await response.json() as TgResponse; - } catch { - throw new TelegramApiError(method, response.status, 'Invalid response from Telegram'); - } - if (!body.ok || body.result === undefined) { - throw new TelegramApiError( - method, - body.error_code ?? response.status, - body.description ?? 'Unknown error', - body.parameters?.retry_after ?? null, - ); + // No explicit content-type: fetch derives the multipart boundary. + response = await fetch(`${TELEGRAM_API_BASE}/bot${token}/${method}`, { + method: 'POST', + body: form, + signal: controller.signal, + }); + } finally { + clearTimeout(timer); } - return body.result; + return parseTgResponse(method, response); } export type TelegramBotClient = { @@ -125,6 +196,14 @@ export type TelegramBotClient = { * cosmetic and must never fail a turn. */ sendMessageDraft(chatId: number, draftId: number, text: string): Promise; + /** Uploads an image as a photo (Telegram caps photos at ~10 MB). */ + sendPhoto(chatId: number, file: TgUploadFile, options?: { caption?: string }): Promise; + /** Uploads a file as a document — the fallback for oversized or exotic images. */ + sendDocument(chatId: number, file: TgUploadFile, options?: { caption?: string }): Promise; + /** Resolves a media file_id to a downloadable path. */ + getFile(fileId: string): Promise; + /** Downloads media bytes for a path returned by getFile (bots: ≤20 MB). */ + downloadFile(filePath: string): Promise; editMessageText(chatId: number, messageId: number, text: string): Promise; answerCallbackQuery(callbackQueryId: string, text?: string): Promise; setMyCommands(commands: Array<{ command: string; description: string }>): Promise; @@ -185,6 +264,34 @@ export function createTelegramBotClient(token: string): TelegramBotClient { }, 10_000); }, + sendPhoto: (chatId, file, options) => tgUpload(token, 'sendPhoto', 'photo', file, { + chat_id: String(chatId), + ...(options?.caption ? { caption: options.caption } : {}), + }), + + sendDocument: (chatId, file, options) => tgUpload(token, 'sendDocument', 'document', file, { + chat_id: String(chatId), + ...(options?.caption ? { caption: options.caption } : {}), + }), + + getFile: (fileId) => tgCall(token, 'getFile', { file_id: fileId }), + + downloadFile: async (filePath) => { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 60_000); + try { + const response = await fetch(`${TELEGRAM_API_BASE}/file/bot${token}/${filePath}`, { + signal: controller.signal, + }); + if (!response.ok) { + throw new TelegramApiError('downloadFile', response.status, `HTTP ${String(response.status)}`); + } + return Buffer.from(await response.arrayBuffer()); + } finally { + clearTimeout(timer); + } + }, + editMessageText: async (chatId, messageId, text) => { await tgCall(token, 'editMessageText', { chat_id: chatId, diff --git a/apps/desktop/src/main/telegram/bridge.ts b/apps/desktop/src/main/telegram/bridge.ts index 06d59dead..09e2b2692 100644 --- a/apps/desktop/src/main/telegram/bridge.ts +++ b/apps/desktop/src/main/telegram/bridge.ts @@ -6,8 +6,15 @@ // keyboards; whichever surface (desktop dialog or Telegram) decides first // wins via PiChatEngine.resolveToolApproval. -import { randomBytes } from 'node:crypto'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { readFile } from 'node:fs/promises'; import { onChatEvents } from '../chat/event-bus.js'; +import { resolveAttachmentPath, saveAttachment } from '../chat/attachments/store.js'; +import { + prepareChatAttachments, + type PreparedChatAttachment, + type RawChatAttachment, +} from '../chat/attachments/prepare.js'; import type { ChatStreamStopReason } from '../chat/stream-stop.js'; import type { ToolApprovalRequest } from '../chat/permissions.js'; import type { PiChatEngine } from '../chat/engine.js'; @@ -19,7 +26,15 @@ import { type TgCallbackQuery, type TgMessage, type TgReplyMarkup, + type TgUploadFile, } from './bot-api.js'; +import { + classifyIncomingMedia, + extractImageFileBlocks, + extractImageFileBlocksFromUiMessage, + type DownloadableTgMedia, + type ImageFileBlock, +} from './media.js'; import { markdownToTelegramHtml } from './markdown.js'; import { clearTelegramSettings, @@ -67,6 +82,14 @@ const POLL_BACKOFF_MIN_MS = 1_000; const POLL_BACKOFF_MAX_MS = 30_000; /** Telegram hard message limit; drafts share it. */ const TG_TEXT_LIMIT = 4096; +/** Telegram rejects sendPhoto uploads above ~10 MB; larger images go as documents. */ +const TG_PHOTO_LIMIT_BYTES = 10 * 1024 * 1024; +/** Telegram caps media captions at 1024 chars. */ +const TG_CAPTION_LIMIT = 1024; +/** Mime types Telegram renders inline as photos; anything else goes as a document. */ +const TG_PHOTO_MIME_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp']); +/** Bot API refuses getFile downloads past this size. */ +const TG_DOWNLOAD_LIMIT_BYTES = 20 * 1024 * 1024; const WELCOME_TEXT = [ 'Connected to your VPR. Messages here run on the agent on your computer.', @@ -74,7 +97,10 @@ const WELCOME_TEXT = [ '', '/new — start a fresh conversation', '/model — choose which model answers', + '/image — generate an image', '/stop — cancel the reply in progress', + '', + 'You can also send photos and documents — the agent reads them like the app\'s paper-clip attachments.', ].join('\n'); /** Inline keyboards get unwieldy past this. Sized for the app's dropdown: @@ -84,6 +110,7 @@ const MODEL_PICK_LIMIT = 24; const BOT_COMMANDS = [ { command: 'new', description: 'Start a fresh conversation' }, { command: 'model', description: 'Choose which model answers' }, + { command: 'image', description: 'Generate an image from a prompt' }, { command: 'stop', description: 'Cancel the reply in progress' }, ]; @@ -151,6 +178,10 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel let pollGeneration = 0; let lastError: string | null = null; let activeTurn: ActiveTurn | null = null; + /** Conversation with an /image generation in flight; null when idle. */ + let activeImageRunConversationId: string | null = null; + /** Set synchronously while an inbound attachment downloads, closing the busy-check gap. */ + let mediaInFlight = false; let unsubscribeBus: (() => void) | null = null; let draftCounter = 0; /** Disabled for the session after the first hard sendMessageDraft failure. */ @@ -281,6 +312,30 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel } }; + const sendImageToOwner = async (conversationId: string, block: ImageFileBlock, caption?: string): Promise => { + if (!client || settings?.ownerChatId == null) return false; + const chatId = settings.ownerChatId; + try { + const filePath = await resolveAttachmentPath(conversationId, block.attachmentId); + if (!filePath) { + log(`Image attachment ${block.attachmentId} not found for conversation ${conversationId}.`); + return false; + } + const bytes = await readFile(filePath); + const file = { bytes, fileName: block.fileName, contentType: block.mimeType } satisfies TgUploadFile; + const options = caption ? { caption: caption.slice(0, TG_CAPTION_LIMIT) } : {}; + if (TG_PHOTO_MIME_TYPES.has(block.mimeType) && bytes.length <= TG_PHOTO_LIMIT_BYTES) { + await client.sendPhoto(chatId, file, options); + } else { + await client.sendDocument(chatId, file, options); + } + return true; + } catch (err) { + log(`Image send failed: ${asErrorMessage(err)}`); + return false; + } + }; + // ── Chat engine events ────────────────────────────────────────────── const handleToolApprovalRequested = (payload: ToolApprovalRequest): void => { @@ -353,8 +408,20 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel if (channel === 'chat:ai-done') { turn.finalized = true; clearTurn(turn); - const finalText = extractTextFromUiMessage(payload) || turn.buffer; - void sendToOwner(finalText.trim().length > 0 ? finalText : 'Done (no text reply).'); + const finalText = (extractTextFromUiMessage(payload) || turn.buffer).trim(); + const images = extractImageFileBlocksFromUiMessage(payload); + // Text first, then images, in order. An image-only reply skips the + // "Done (no text reply)." placeholder — the photo is the reply. + void (async () => { + if (finalText.length > 0) { + await sendToOwner(finalText); + } else if (images.length === 0) { + await sendToOwner('Done (no text reply).'); + } + for (const image of images) { + await sendImageToOwner(turn.conversationId, image); + } + })(); return; } @@ -408,7 +475,8 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel // first) when available, otherwise the first discovered catalog entry. const picked = (engine.getModelPicker()?.models ?? []).find((model) => model.routePeerId); if (picked) return { service: picked.serviceId }; - const chosen = entries.find((entry) => entry.peerId); + const textEntries = entries.filter((entry) => entry.protocol !== 'openai-images'); + const chosen = textEntries.find((entry) => entry.peerId); if (chosen) return { service: chosen.id }; } catch { // Discovery unavailable — the engine will surface the buyer error. @@ -444,6 +512,9 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel const seen = new Set(); for (const entry of entries) { if (!entry.peerId) continue; + // Image-only endpoints cannot serve text chat; they are reachable + // via /image instead. + if (entry.protocol === 'openai-images') continue; const key = entry.id.trim().toLowerCase(); if (seen.has(key)) continue; seen.add(key); @@ -516,7 +587,7 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel return conversation.id; }; - const runUserText = async (chatId: number, text: string): Promise => { + const runUserText = async (chatId: number, text: string, attachments?: PreparedChatAttachment[]): Promise => { if (activeTurn) { void sendToOwner('Still working on the previous message — send /stop to cancel it first.'); return; @@ -538,7 +609,7 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel draftTimer: null, draftInFlight: false, finalized: false, - statusText: 'Thinking…', + statusText: attachments ? 'Reading attachment…' : 'Thinking…', keepAliveTimer: null, }; activeTurn = turn; @@ -547,7 +618,7 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel pushDraft(turn); turn.keepAliveTimer = setInterval(() => { pushDraft(turn); }, 4_000); try { - const result = await engine.sendMessageStream(conversationId, text); + const result = await engine.sendMessageStream(conversationId, text, { ...(attachments ? { attachments } : {}) }); // Success and stream errors are reported through bus events; this // fallback only covers early returns that never started a stream // (e.g. buyer proxy unreachable). @@ -565,6 +636,174 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel } }; + // The best-ranked image service within the buyer routing policy. Text + // routes come from /model and the default route; image generation only + // works against `openai-images` services, so it picks its own peer. + const resolveImageRoute = async (): Promise<{ peerId: string; service: string; label: string } | null> => { + try { + const entries = await engine.discoverServiceCatalog(); + const entry = entries.find((candidate) => candidate.protocol === 'openai-images' && candidate.peerId); + if (entry?.peerId) return { peerId: entry.peerId, service: entry.id, label: entry.label || entry.id }; + } catch { + // Discovery unavailable — reported below as "no image models". + } + return null; + }; + + const runImagePrompt = async (chatId: number, prompt: string): Promise => { + if (activeTurn || activeImageRunConversationId || mediaInFlight) { + void sendToOwner('Still working on the previous message — send /stop to cancel it first.'); + return; + } + let conversationId: string; + try { + conversationId = await ensureConversation(); + } catch (err) { + void sendToOwner(`Couldn't start a conversation: ${asErrorMessage(err)}`); + return; + } + const route = await resolveImageRoute(); + if (!route) { + void sendToOwner('No image models discovered on the network yet — try again in a moment.'); + return; + } + activeImageRunConversationId = conversationId; + draftCounter += 1; + const draftId = draftCounter; + // Image generations legitimately take minutes; keep an ephemeral + // "Generating…" draft alive rather than sending a throwaway message. + const statusText = `🎨 Generating an image with ${route.label}…`; + const pushStatusDraft = (): void => { + if (!client || !draftsSupported) return; + client.sendMessageDraft(chatId, draftId, statusText).catch(() => {}); + }; + pushStatusDraft(); + const keepAlive = setInterval(pushStatusDraft, 4_000); + try { + const result = await engine.generateImage({ conversationId, prompt, peerId: route.peerId, service: route.service }); + if (!result.ok) { + const error = result.error ?? 'Unknown error'; + void sendToOwner(error === 'Request aborted' ? 'Stopped.' : `Image generation failed: ${error}`); + return; + } + const images = extractImageFileBlocks(result.assistant?.content); + if (images.length === 0) { + void sendToOwner('The image was generated but its file could not be found.'); + return; + } + let sentAny = false; + for (const image of images) { + if (await sendImageToOwner(conversationId, image, prompt)) sentAny = true; + } + if (!sentAny) void sendToOwner('The image was generated but sending it to Telegram failed.'); + } catch (err) { + void sendToOwner(`Image generation failed: ${asErrorMessage(err)}`); + } finally { + clearInterval(keepAlive); + if (activeImageRunConversationId === conversationId) activeImageRunConversationId = null; + } + }; + + // ── Inbound media ───────────────────────────────────────────────── + + /** Downloads an inbound Telegram attachment's bytes, or a friendly error string. */ + const fetchTelegramFileBytes = async ( + media: DownloadableTgMedia, + ): Promise<{ bytes: Buffer } | { error: string }> => { + if (!client) return { error: 'The bot is not connected.' }; + if ((media.approxBytes ?? 0) > TG_DOWNLOAD_LIMIT_BYTES) { + return { error: 'That file is too large for Telegram bots to download (20 MB max).' }; + } + try { + const file = await client.getFile(media.fileId); + if (!file.file_path) { + return { error: 'Telegram would not let me download that file (it may exceed 20 MB).' }; + } + const bytes = await client.downloadFile(file.file_path); + if (bytes.length === 0) return { error: 'That file arrived empty.' }; + if (bytes.length > TG_DOWNLOAD_LIMIT_BYTES) { + return { error: 'That file is too large for Telegram bots to download (20 MB max).' }; + } + return { bytes }; + } catch (err) { + log(`Media download failed: ${asErrorMessage(err)}`); + return { error: `Could not download that file: ${asErrorMessage(err)}` }; + } + }; + + /** Downloads + prepares one inbound attachment against the conversation's store. */ + const prepareIncomingAttachment = async ( + conversationId: string, + media: DownloadableTgMedia, + ): Promise => { + const downloaded = await fetchTelegramFileBytes(media); + if ('error' in downloaded) { + return [{ + id: media.fileId, + name: media.fileName, + mimeType: media.mimeType, + size: media.approxBytes ?? 0, + kind: 'error', + status: 'error', + error: downloaded.error, + }]; + } + const raw: RawChatAttachment = { + id: randomUUID(), + name: media.fileName, + mimeType: media.mimeType, + size: downloaded.bytes.length, + base64: downloaded.bytes.toString('base64'), + }; + return prepareChatAttachments([raw], { + storage: async (rawMeta, buffer) => { + const attachmentId = randomUUID(); + await saveAttachment(conversationId, attachmentId, rawMeta.name, buffer); + return attachmentId; + }, + }); + }; + + const runUserMedia = async (chatId: number, caption: string, media: DownloadableTgMedia): Promise => { + if (activeTurn || activeImageRunConversationId || mediaInFlight) { + void sendToOwner('Still working on the previous message — send /stop to cancel it first.'); + return; + } + mediaInFlight = true; + try { + await runUserMediaInner(chatId, caption, media); + } finally { + mediaInFlight = false; + } + }; + + const runUserMediaInner = async (chatId: number, caption: string, media: DownloadableTgMedia): Promise => { + let conversationId: string; + try { + conversationId = await ensureConversation(); + } catch (err) { + void sendToOwner(`Couldn't start a conversation: ${asErrorMessage(err)}`); + return; + } + // One-shot cosmetic draft while the download/preparation runs; the real + // "Reading attachment…" turn draft starts once sendMessageStream begins. + draftCounter += 1; + const prepDraftId = draftCounter; + if (client && draftsSupported) { + client.sendMessageDraft(chatId, prepDraftId, `📥 Receiving ${media.fileName}…`).catch(() => {}); + } + const attachments = await prepareIncomingAttachment(conversationId, media); + const failed = attachments.filter((attachment) => attachment.status === 'error'); + if (failed.length === attachments.length) { + void sendToOwner(`Couldn't use ${media.fileName}: ${failed[0]?.error ?? 'unknown error'}`); + return; + } + for (const failure of failed) { + void sendToOwner(`Skipped ${failure.name}: ${failure.error ?? 'unsupported file'}.`); + } + await runUserText(chatId, caption, attachments); + }; + const handlePairingAttempt = async (message: TgMessage): Promise => { if (!settings) return; const text = message.text?.trim() ?? ''; @@ -590,10 +829,20 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel const handleOwnerMessage = async (message: TgMessage): Promise => { const text = message.text?.trim() ?? ''; - if (text.length === 0) { + const media = classifyIncomingMedia(message); + if (text.length === 0 && !media) { void sendToOwner('Only text messages are supported for now.'); return; } + if (media?.kind === 'unsupported') { + void sendToOwner(`${media.label} aren't supported yet — send text, a photo, or a document instead.`); + return; + } + if (media?.kind === 'photo' || media?.kind === 'document') { + // Deliberately not awaited, same as text turns. + void runUserMedia(message.chat.id, message.caption?.trim() ?? '', media); + return; + } if (text.startsWith('/start')) { void sendToOwner(WELCOME_TEXT); return; @@ -611,13 +860,28 @@ export function createTelegramBridge({ engine, appendLog, onStatusChanged }: Tel await showModelPicker(); return; } + if (text === '/image' || text.startsWith('/image ')) { + const prompt = text.slice('/image'.length).trim(); + if (prompt.length === 0) { + void sendToOwner('Usage: /image — e.g. /image a crazy ant carrying a seed'); + return; + } + // Deliberately not awaited, same as text turns: the poll loop stays + // free for /stop while the generation runs. + void runImagePrompt(message.chat.id, prompt); + return; + } if (text === '/stop') { const turn = activeTurn; - if (!turn) { + const imageConversationId = activeImageRunConversationId; + if (!turn && !imageConversationId) { void sendToOwner('Nothing is running.'); return; } - await engine.abort(turn.conversationId).catch(() => {}); + if (turn) await engine.abort(turn.conversationId).catch(() => {}); + // engine.abort also cancels in-flight image generations for the + // conversation (the engine tracks both run kinds). + if (imageConversationId) await engine.abort(imageConversationId).catch(() => {}); return; } // Deliberately not awaited: replies stream in the background while the diff --git a/apps/desktop/src/main/telegram/media.test.ts b/apps/desktop/src/main/telegram/media.test.ts new file mode 100644 index 000000000..d7b56a012 --- /dev/null +++ b/apps/desktop/src/main/telegram/media.test.ts @@ -0,0 +1,114 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + classifyIncomingMedia, + extractImageFileBlocks, + extractImageFileBlocksFromUiMessage, + pickLargestPhoto, +} from './media.js'; + +test('pickLargestPhoto returns the largest sized variant and ignores junk', () => { + assert.equal(pickLargestPhoto(undefined), null); + assert.equal(pickLargestPhoto([]), null); + assert.equal(pickLargestPhoto(['nope']), null); + const best = pickLargestPhoto([ + { file_id: 'small', file_size: 1_000, width: 90, height: 90 }, + { file_id: 'large', file_size: 200_000, width: 1280, height: 720 }, + { file_id: 'mid', file_size: 100_000, width: 320, height: 320 }, + ]); + assert.equal(best?.file_id, 'large'); + // Missing file_size entries lose against known sizes. + const fallback = pickLargestPhoto([ + { file_id: 'unknown' }, + { file_id: 'known', file_size: 5 }, + ]); + assert.equal(fallback?.file_id, 'known'); +}); + +test('extractImageFileBlocks collects only ready image blocks with attachment ids', () => { + const content = [ + { type: 'text', text: 'here you go' }, + { type: 'file', fileName: 'generated-abc.png', mimeType: 'image/png', attachmentId: 'att-1' }, + { type: 'file', fileName: 'notes.pdf', mimeType: 'application/pdf', attachmentId: 'att-2' }, + { type: 'file', fileName: 'no-id.png', mimeType: 'image/png' }, + { type: 'image', source: { type: 'base64', data: '...' } }, + null, + ]; + assert.deepEqual(extractImageFileBlocks(content), [ + { fileName: 'generated-abc.png', mimeType: 'image/png', attachmentId: 'att-1' }, + ]); + assert.deepEqual(extractImageFileBlocks('plain text'), []); + assert.deepEqual( + extractImageFileBlocksFromUiMessage({ message: { content } }), + [{ fileName: 'generated-abc.png', mimeType: 'image/png', attachmentId: 'att-1' }], + ); + assert.deepEqual(extractImageFileBlocksFromUiMessage(null), []); +}); + +test('classifyIncomingMedia picks the largest photo and defaults name/mime', () => { + const media = classifyIncomingMedia({ + date: 1_755_000_000, + photo: [ + { file_id: 'small', file_size: 900 }, + { file_id: 'big', file_size: 250_000 }, + ], + caption: 'what breed is this?', + }); + assert.equal(media?.kind, 'photo'); + if (media?.kind !== 'photo') return; + assert.equal(media.fileId, 'big'); + assert.equal(media.mimeType, 'image/jpeg'); + assert.match(media.fileName, /^photo-\d+\.jpg$/); + assert.equal(media.approxBytes, 250_000); +}); + +test('classifyIncomingMedia maps documents with name and mime fallbacks', () => { + const withMeta = classifyIncomingMedia({ + document: { file_id: 'f1', file_name: ' report.PDF ', mime_type: 'application/pdf;charset=bogus', file_size: 10 }, + }); + assert.deepEqual(withMeta, { + kind: 'document', + fileId: 'f1', + fileName: 'report.PDF', + mimeType: 'application/pdf', + approxBytes: 10, + }); + + const unnamed = classifyIncomingMedia({ document: { file_id: 'f2' } }); + assert.equal(unnamed?.kind, 'document'); + if (unnamed?.kind !== 'document') return; + assert.equal(unnamed.fileName, 'document'); + assert.equal(unnamed.mimeType, 'application/octet-stream'); + + const extFallback = classifyIncomingMedia({ document: { file_id: 'f3', file_name: 'notes.txt' } }); + assert.equal(extFallback?.kind, 'document'); + if (extFallback?.kind !== 'document') return; + assert.equal(extFallback.mimeType, 'text/plain'); +}); + +test('classifyIncomingMedia flags recognised-but-unsupported media', () => { + assert.deepEqual(classifyIncomingMedia({ voice: { file_id: 'v', duration: 3 } }), { + kind: 'unsupported', + label: 'Voice messages', + }); + assert.deepEqual(classifyIncomingMedia({ video_note: { file_id: 'vn', duration: 3 } }), { + kind: 'unsupported', + label: 'Video messages', + }); + assert.deepEqual(classifyIncomingMedia({ audio: { file_id: 'a', duration: 3 } }), { + kind: 'unsupported', + label: 'Audio files', + }); + assert.deepEqual(classifyIncomingMedia({ video: { file_id: 'v2', duration: 3 } }), { + kind: 'unsupported', + label: 'Videos', + }); +}); + +test('classifyIncomingMedia returns null for plain text or unknown payloads', () => { + assert.equal(classifyIncomingMedia({ text: 'hello' }), null); + assert.equal(classifyIncomingMedia({}), null); + // Documents without a usable file_id are not actionable media. + assert.equal(classifyIncomingMedia({ document: { file_name: 'x.pdf' } }), null); +}); diff --git a/apps/desktop/src/main/telegram/media.ts b/apps/desktop/src/main/telegram/media.ts new file mode 100644 index 000000000..20a44a931 --- /dev/null +++ b/apps/desktop/src/main/telegram/media.ts @@ -0,0 +1,141 @@ +// ── Telegram media classification ── +// Pure helpers shared by the bridge: extracting image file blocks from +// assistant payloads (outgoing) and classifying inbound media messages. +// Kept free of electron imports so node:test can exercise them directly. + +export type TgPhotoSizeLike = { + file_id: string; + file_size?: number; + width?: number; + height?: number; +}; + +export type ImageFileBlock = { + fileName: string; + mimeType: string; + attachmentId: string; +}; + +/** Image file blocks (generated images, image attachments) whose bytes live in the attachment store. */ +export function extractImageFileBlocks(content: unknown): ImageFileBlock[] { + if (!Array.isArray(content)) return []; + const blocks: ImageFileBlock[] = []; + for (const raw of content) { + if (!raw || typeof raw !== 'object') continue; + const block = raw as { type?: unknown; fileName?: unknown; mimeType?: unknown; attachmentId?: unknown }; + if (block.type !== 'file') continue; + if (typeof block.attachmentId !== 'string' || block.attachmentId.length === 0) continue; + if (typeof block.mimeType !== 'string' || !block.mimeType.startsWith('image/')) continue; + blocks.push({ + fileName: typeof block.fileName === 'string' && block.fileName.length > 0 ? block.fileName : 'image', + mimeType: block.mimeType, + attachmentId: block.attachmentId, + }); + } + return blocks; +} + +export function extractImageFileBlocksFromUiMessage(payload: unknown): ImageFileBlock[] { + const message = (payload as { message?: { content?: unknown } } | null)?.message; + return extractImageFileBlocks(message?.content); +} + +/** Telegram photo messages carry a size ladder; the largest entry has the full resolution. */ +export function pickLargestPhoto(photo: unknown): TgPhotoSizeLike | null { + if (!Array.isArray(photo)) return null; + let best: TgPhotoSizeLike | null = null; + for (const raw of photo) { + if (!raw || typeof raw !== 'object') continue; + const size = raw as Partial; + if (typeof size.file_id !== 'string' || size.file_id.length === 0) continue; + if (!best || (size.file_size ?? -1) > (best.file_size ?? -1)) best = size as TgPhotoSizeLike; + } + return best; +} + +/** A media message the bridge can download and attach to a conversation. */ +export type DownloadableTgMedia = { + kind: 'photo' | 'document'; + fileId: string; + fileName: string; + mimeType: string; + /** Best-known byte size from the update payload; used to pre-reject oversized files. */ + approxBytes?: number; +}; + +/** Media the bot recognises but cannot process yet. */ +export type UnsupportedTgMedia = { + kind: 'unsupported'; + /** Human-readable media name for the "not supported" notice. */ + label: string; +}; + +export type IncomingTgMedia = DownloadableTgMedia | UnsupportedTgMedia; + +const DOCUMENT_FALLBACK_EXTENSION_MIME = new Map([ + ['.pdf', 'application/pdf'], + ['.txt', 'text/plain'], + ['.md', 'text/markdown'], + ['.csv', 'text/csv'], +]); + +function mimeForDocument(fileName: string, mimeType?: string): string { + const normalized = mimeType?.split(';', 1)[0]?.trim().toLowerCase() ?? ''; + if (normalized) return normalized; + const dot = fileName.lastIndexOf('.'); + const ext = dot >= 0 ? fileName.slice(dot).toLowerCase() : ''; + return DOCUMENT_FALLBACK_EXTENSION_MIME.get(ext) ?? 'application/octet-stream'; +} + +/** + * Maps an inbound message to what the bridge can do with it: + * a downloadable attachment (photo/document), an explicitly unsupported + * medium worth a helpful notice about, or null when the message carries no + * media at all (plain text — or something so exotic the generic + * "text only" fallback covers it). + */ +export function classifyIncomingMedia(message: { + date?: number; + text?: string; + caption?: string; + photo?: unknown; + document?: { file_id?: unknown; file_name?: unknown; mime_type?: unknown; file_size?: unknown } | null; + voice?: unknown; + video_note?: unknown; + video?: unknown; + audio?: unknown; +}): IncomingTgMedia | null { + if (message.voice) return { kind: 'unsupported', label: 'Voice messages' }; + if (message.video_note) return { kind: 'unsupported', label: 'Video messages' }; + if (message.audio) return { kind: 'unsupported', label: 'Audio files' }; + if (message.video) return { kind: 'unsupported', label: 'Videos' }; + + const photo = pickLargestPhoto(message.photo); + if (photo) { + return { + kind: 'photo', + fileId: photo.file_id, + // Telegram photos are always JPEG regardless of how they were uploaded. + fileName: `photo-${message.date ?? 0}.jpg`, + mimeType: 'image/jpeg', + ...(photo.file_size != null ? { approxBytes: photo.file_size } : {}), + }; + } + + const doc = message.document; + if (doc && typeof doc === 'object' && typeof doc.file_id === 'string' && doc.file_id.length > 0) { + const fileName = typeof doc.file_name === 'string' && doc.file_name.trim().length > 0 + ? doc.file_name.trim() + : 'document'; + const fileSize = typeof doc.file_size === 'number' ? doc.file_size : undefined; + return { + kind: 'document', + fileId: doc.file_id, + fileName, + mimeType: mimeForDocument(fileName, typeof doc.mime_type === 'string' ? doc.mime_type : undefined), + ...(fileSize != null ? { approxBytes: fileSize } : {}), + }; + } + + return null; +}