diff --git a/apps/desktop/src/ai/hooks/useLLMConnection.ts b/apps/desktop/src/ai/hooks/useLLMConnection.ts index 9c2914932e..f4b3507301 100644 --- a/apps/desktop/src/ai/hooks/useLLMConnection.ts +++ b/apps/desktop/src/ai/hooks/useLLMConnection.ts @@ -13,6 +13,7 @@ import type { AIProviderStorage } from "@anlg/store"; import { createAppleFoundationModel } from "../apple-foundation-model"; import { createAuthFetch } from "../auth-fetch"; +import { streamOnlyGenerationMiddleware } from "../stream-only-generation"; import { createTracedFetch, tracedFetch } from "../traced-fetch"; import { useAuth } from "~/auth"; @@ -291,7 +292,15 @@ const createLanguageModel = ( baseURL: oauth ? CHATGPT_API_BASE_URL : conn.baseUrl, apiKey: oauth ? "oauth" : conn.apiKey, }); - return wrapWithThinkingMiddleware(provider.responses(conn.modelId)); + const model = provider.responses(conn.modelId); + return wrapWithThinkingMiddleware( + oauth + ? wrapLanguageModel({ + model, + middleware: streamOnlyGenerationMiddleware, + }) + : model, + ); } case "grok": diff --git a/apps/desktop/src/ai/stream-only-generation.test.ts b/apps/desktop/src/ai/stream-only-generation.test.ts new file mode 100644 index 0000000000..05aaedec63 --- /dev/null +++ b/apps/desktop/src/ai/stream-only-generation.test.ts @@ -0,0 +1,118 @@ +import { wrapLanguageModel } from "ai"; +import { describe, expect, test, vi } from "vitest"; + +import { streamOnlyGenerationMiddleware } from "./stream-only-generation"; + +type LanguageModel = Parameters[0]["model"]; +type StreamPart = + Awaited< + ReturnType + >["stream"] extends ReadableStream + ? Part + : never; + +const usage = { + inputTokens: { + total: 4, + noCache: 4, + cacheRead: 0, + cacheWrite: 0, + }, + outputTokens: { total: 2, text: 2, reasoning: 0 }, +}; + +describe("streamOnlyGenerationMiddleware", () => { + test("uses streaming for generate calls and collects the result", async () => { + const doGenerate = vi.fn(async () => { + throw new Error("non-streaming request used"); + }); + const model = createModel(doGenerate, [ + { type: "stream-start", warnings: [] }, + { + type: "response-metadata", + id: "response-1", + modelId: "gpt-test", + timestamp: new Date("2026-08-25T00:00:00.000Z"), + }, + { type: "text-start", id: "message-1" }, + { type: "text-delta", id: "message-1", delta: "Hello" }, + { type: "text-delta", id: "message-1", delta: " world" }, + { type: "text-end", id: "message-1" }, + { + type: "tool-call", + toolCallId: "tool-1", + toolName: "lookup", + input: '{"query":"test"}', + }, + { + type: "finish", + finishReason: { unified: "stop", raw: "completed" }, + usage, + }, + ]); + const wrapped = wrapLanguageModel({ + model, + middleware: streamOnlyGenerationMiddleware, + }); + + const result = await wrapped.doGenerate({ prompt: [] }); + + expect(doGenerate).not.toHaveBeenCalled(); + expect(result.content).toEqual([ + { type: "text", text: "Hello world" }, + { + type: "tool-call", + toolCallId: "tool-1", + toolName: "lookup", + input: '{"query":"test"}', + }, + ]); + expect(result.finishReason).toEqual({ unified: "stop", raw: "completed" }); + expect(result.usage).toEqual(usage); + expect(result.response).toMatchObject({ + headers: { "x-request-id": "request-1" }, + id: "response-1", + modelId: "gpt-test", + }); + }); + + test("rejects provider errors from the response stream", async () => { + const model = createModel(vi.fn(), [ + { type: "stream-start", warnings: [] }, + { type: "error", error: new Error("upstream failed") }, + ]); + const wrapped = wrapLanguageModel({ + model, + middleware: streamOnlyGenerationMiddleware, + }); + + await expect(wrapped.doGenerate({ prompt: [] })).rejects.toThrow( + "upstream failed", + ); + }); +}); + +function createModel( + doGenerate: LanguageModel["doGenerate"], + parts: StreamPart[], +): LanguageModel { + return { + specificationVersion: "v3", + provider: "test", + modelId: "gpt-test", + supportedUrls: {}, + doGenerate, + doStream: async () => ({ + stream: new ReadableStream({ + start(controller) { + for (const part of parts) { + controller.enqueue(part); + } + controller.close(); + }, + }), + request: { body: { stream: true } }, + response: { headers: { "x-request-id": "request-1" } }, + }), + }; +} diff --git a/apps/desktop/src/ai/stream-only-generation.ts b/apps/desktop/src/ai/stream-only-generation.ts new file mode 100644 index 0000000000..0c1fcb998b --- /dev/null +++ b/apps/desktop/src/ai/stream-only-generation.ts @@ -0,0 +1,158 @@ +import type { LanguageModelMiddleware } from "ai"; + +type WrapGenerate = NonNullable; +type GenerateResult = Awaited< + ReturnType[0]["doGenerate"]> +>; +type StreamResult = Awaited< + ReturnType[0]["doStream"]> +>; +type StreamPart = + StreamResult["stream"] extends ReadableStream ? Part : never; +type Content = GenerateResult["content"][number]; +type TextContent = Extract; + +export const streamOnlyGenerationMiddleware: LanguageModelMiddleware = { + specificationVersion: "v3", + wrapGenerate: async ({ doStream }) => collectStream(await doStream()), +}; + +async function collectStream(result: StreamResult): Promise { + const content: Content[] = []; + const openBlocks = new Map(); + let warnings: GenerateResult["warnings"] = []; + let responseMetadata: NonNullable = {}; + let finish: Extract | undefined; + + const reader = result.stream.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + switch (value.type) { + case "text-start": + case "reasoning-start": { + createBlock(value, content, openBlocks); + break; + } + case "text-delta": + case "reasoning-delta": { + const block = createBlock(value, content, openBlocks); + block.text += value.delta; + if (value.providerMetadata) { + block.providerMetadata = value.providerMetadata; + } + break; + } + case "text-end": + case "reasoning-end": { + const block = createBlock(value, content, openBlocks); + if (value.providerMetadata) { + block.providerMetadata = value.providerMetadata; + } + openBlocks.delete(blockKey(value)); + break; + } + case "file": + case "source": + case "tool-approval-request": + case "tool-call": + case "tool-result": + content.push(value); + break; + case "stream-start": + warnings = value.warnings; + break; + case "response-metadata": + responseMetadata = { + ...responseMetadata, + id: value.id, + timestamp: value.timestamp, + modelId: value.modelId, + }; + break; + case "finish": + finish = value; + break; + case "error": + throw value.error; + case "raw": + case "tool-input-start": + case "tool-input-delta": + case "tool-input-end": + break; + } + } + } finally { + reader.releaseLock(); + } + + if (!finish) { + throw new Error("ChatGPT response stream ended without a finish event"); + } + + const hasResponseMetadata = + result.response !== undefined || + responseMetadata.id !== undefined || + responseMetadata.timestamp !== undefined || + responseMetadata.modelId !== undefined; + + return { + content, + finishReason: finish.finishReason, + usage: finish.usage, + providerMetadata: finish.providerMetadata, + request: result.request, + response: hasResponseMetadata + ? { ...responseMetadata, ...result.response } + : undefined, + warnings, + }; +} + +function createBlock( + part: Extract< + StreamPart, + { + type: + | "reasoning-delta" + | "reasoning-end" + | "reasoning-start" + | "text-delta" + | "text-end" + | "text-start"; + } + >, + content: Content[], + openBlocks: Map, +): TextContent { + const key = blockKey(part); + const existing = openBlocks.get(key); + if (existing) { + return existing; + } + + const block: TextContent = { + type: part.type.startsWith("reasoning") ? "reasoning" : "text", + text: "", + providerMetadata: part.providerMetadata, + }; + openBlocks.set(key, block); + content.push(block); + return block; +} + +function blockKey(part: { + id: string; + type: + | "reasoning-delta" + | "reasoning-end" + | "reasoning-start" + | "text-delta" + | "text-end" + | "text-start"; +}): string { + const type = part.type.startsWith("reasoning") ? "reasoning" : "text"; + return `${type}:${part.id}`; +} diff --git a/apps/desktop/src/settings/ai/llm/subscriptions/models.test.ts b/apps/desktop/src/settings/ai/llm/subscriptions/models.test.ts new file mode 100644 index 0000000000..ab3fd69c4e --- /dev/null +++ b/apps/desktop/src/settings/ai/llm/subscriptions/models.test.ts @@ -0,0 +1,74 @@ +import { Effect } from "effect"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + fetchJson: vi.fn(), + resolveSubscriptionAccess: vi.fn(), +})); + +vi.mock("./access", () => ({ + resolveSubscriptionAccess: mocks.resolveSubscriptionAccess, +})); + +vi.mock("~/settings/ai/shared/list-common", async (importOriginal) => ({ + ...(await importOriginal()), + fetchJson: mocks.fetchJson, +})); + +import { listSubscriptionModels } from "./models"; + +describe("ChatGPT subscription models", () => { + beforeEach(() => { + mocks.fetchJson.mockReset(); + mocks.resolveSubscriptionAccess.mockReset(); + mocks.resolveSubscriptionAccess.mockResolvedValue({ + token: "access-token", + credential: { accountId: "account-1" }, + }); + }); + + test("parses the Codex catalog and omits hidden models", async () => { + mocks.fetchJson.mockReturnValue( + Effect.succeed({ + models: [ + { slug: "gpt-5.6-sol", visibility: "list" }, + { slug: "codex-auto-review", visibility: "hide" }, + { + slug: "gpt-5.3-codex-spark", + visibility: "list", + supported_in_api: false, + }, + ], + }), + ); + + await expect( + listSubscriptionModels( + "chatgpt", + "https://api.openai.com/v1", + "stored-credential", + ), + ).resolves.toMatchObject({ + models: ["gpt-5.6-sol", "gpt-5.3-codex-spark"], + }); + expect(mocks.fetchJson).toHaveBeenCalledWith( + "https://chatgpt.com/backend-api/codex/models?client_version=0.145.0", + expect.objectContaining({ + Authorization: "Bearer access-token", + "ChatGPT-Account-ID": "account-1", + }), + ); + }); + + test("does not offer stale fallback models when discovery fails", async () => { + mocks.fetchJson.mockReturnValue(Effect.fail(new Error("unavailable"))); + + await expect( + listSubscriptionModels( + "chatgpt", + "https://api.openai.com/v1", + "stored-credential", + ), + ).resolves.toEqual({ models: [], ignored: [], metadata: {} }); + }); +}); diff --git a/apps/desktop/src/settings/ai/llm/subscriptions/models.ts b/apps/desktop/src/settings/ai/llm/subscriptions/models.ts index 31a025935f..fad0c61f7f 100644 --- a/apps/desktop/src/settings/ai/llm/subscriptions/models.ts +++ b/apps/desktop/src/settings/ai/llm/subscriptions/models.ts @@ -24,20 +24,23 @@ import { const FALLBACK_MODELS: Record = { claude: ["claude-sonnet-4-6", "claude-opus-4-6", "claude-haiku-4-5"], - chatgpt: ["gpt-5.3-codex", "gpt-5.4", "gpt-5.2-codex"], + chatgpt: [], grok: ["grok-4", "grok-4-fast", "grok-3"], github_copilot: ["gpt-4.1", "claude-sonnet-4", "gemini-2.5-pro"], kimi_code: ["kimi-for-coding"], }; const ChatgptModelSchema = Schema.Struct({ - data: Schema.Array( + models: Schema.Array( Schema.Struct({ - id: Schema.String, + slug: Schema.String, + visibility: Schema.optional(Schema.String), }), ), }); +const CHATGPT_CODEX_CLIENT_VERSION = "0.145.0"; + const CopilotModelSchema = Schema.Struct({ data: Schema.Array( Schema.Struct({ @@ -129,16 +132,21 @@ async function listChatgptModels( } return pipe( - fetchJson(`${endpoint}/models`, headers), + fetchJson( + `${endpoint}/models?client_version=${CHATGPT_CODEX_CLIENT_VERSION}`, + headers, + ), Effect.andThen((json) => Schema.decodeUnknown(ChatgptModelSchema)(json)), - Effect.map(({ data }) => { - const models = data.map((model) => model.id); + Effect.map(({ models: catalog }) => { + const models = catalog + .filter((model) => model.visibility !== "hide") + .map((model) => model.slug); return { models, ignored: [], metadata: extractMetadataMap( - data, - (model) => model.id, + catalog, + (model) => model.slug, () => ({ input_modalities: ["text", "image"] as const }), ), };