feat: add LiteLLM gateway support for chat + embeddings

README.md

@@ -42,6 +42,52 @@ You can deploy this template by setting up the following services and adding the
 5. Make a [Together](https://togetherai.link/?utm_source=notesGPT&utm_medium=referral&utm_campaign=example-app) account to get your [API key](https://api.together.xyz/settings/api-keys).
 6. Save your environment variables in Convex [`TOGETHER_API_KEY`](https://dashboard.convex.dev/deployment/settings/environment-variables?var=TOGETHER_API_KEY).
 
+## Using a different LLM provider (LiteLLM / OpenAI-compatible gateway)
+
+notesGPT talks to Together AI over the OpenAI wire format, so you can point the
+chat + embedding calls at any OpenAI-compatible endpoint - including a
+[LiteLLM](https://docs.litellm.ai/docs/simple_proxy) proxy, which lets you route
+to 100+ providers (OpenAI, Anthropic, Azure, Bedrock, Gemini, ...) behind a
+single URL. Together AI stays the default; these variables are all optional:
+
+| Variable              | Default                                   | Description                                                                                                                         |
+| --------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
+| `LLM_BASE_URL`        | `https://api.together.xyz/v1`             | OpenAI-compatible base URL (e.g. your LiteLLM proxy)                                                                                |
+| `LLM_API_KEY`         | falls back to `TOGETHER_API_KEY`          | API key / LiteLLM virtual key for the endpoint                                                                                      |
+| `LLM_CHAT_MODEL`      | `Qwen/Qwen2.5-7B-Instruct-Turbo`          | Model used for title/summary/action-item extraction                                                                                 |
+| `LLM_EMBEDDING_MODEL` | `intfloat/multilingual-e5-large-instruct` | Model used for search embeddings                                                                                                    |
+| `LLM_INSTRUCTOR_MODE` | `JSON_SCHEMA`                             | How Instructor coerces structured JSON. Together supports `JSON_SCHEMA`; for OpenAI/Azure/Anthropic/etc. behind a proxy use `TOOLS` |
+
+> Note: Together AI's native `JSON_SCHEMA` mode sends a `response_format.schema`
+> field that other providers reject. When you point `LLM_BASE_URL` at a LiteLLM
+> proxy fronting OpenAI, Azure, Anthropic, etc., set `LLM_INSTRUCTOR_MODE=TOOLS`
+> so structured extraction goes through cross-provider tool calling.
+
+Example - run a LiteLLM proxy that fronts OpenAI + Anthropic, then set
+`LLM_BASE_URL=http://localhost:4000`, `LLM_API_KEY=sk-litellm-...`, and pick the
+model aliases you configured:
+
+```yaml
+# litellm.config.yaml
+model_list:
+  - model_name: chat
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: embed
+    litellm_params:
+      model: openai/text-embedding-3-small
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+```bash
+pip install "litellm[proxy]"
+litellm --config litellm.config.yaml   # serves an OpenAI-compatible API on :4000
+```
+
+Then set `LLM_CHAT_MODEL=chat` and `LLM_EMBEDDING_MODEL=embed`. Transcription
+still uses Together's Whisper endpoint (`TOGETHER_API_KEY`).
+
 ## Future tasks:
 
 - [ ] Keep recording for future playback and display it on the page somewhere

convex/llm.ts

@@ -0,0 +1,47 @@
+import OpenAI from 'openai';
+
+// Central LLM provider configuration.
+//
+// By default notesGPT talks to Together AI, but every value below can be
+// overridden through environment variables. Because the client speaks the
+// OpenAI wire format, pointing `LLM_BASE_URL` at a LiteLLM proxy (or any other
+// OpenAI-compatible gateway) lets you route the chat + embedding calls through
+// 100+ providers (OpenAI, Anthropic, Azure, Bedrock, Gemini, ...) without
+// touching application code. See https://docs.litellm.ai/docs/simple_proxy.
+//
+// Together AI stays the default, so existing deployments keep working with only
+// `TOGETHER_API_KEY` set.
+
+export const llmBaseURL =
+  process.env.LLM_BASE_URL ?? 'https://api.together.xyz/v1';
+
+export const llmApiKey =
+  process.env.LLM_API_KEY ?? process.env.TOGETHER_API_KEY ?? 'undefined';
+
+export const chatModel =
+  process.env.LLM_CHAT_MODEL ?? 'Qwen/Qwen2.5-7B-Instruct-Turbo';
+
+export const embeddingModel =
+  process.env.LLM_EMBEDDING_MODEL ?? 'intfloat/multilingual-e5-large-instruct';
+
+// Structured-output strategy Instructor uses to coerce JSON out of the model.
+// Together AI supports its native `JSON_SCHEMA` mode (the default), but most
+// other providers reached through a LiteLLM proxy (OpenAI, Azure, Anthropic,
+// ...) reject Together's `response_format.schema` field. For those, set
+// `LLM_INSTRUCTOR_MODE=TOOLS`, which drives structured output through
+// cross-provider tool/function calling. Valid values: FUNCTIONS, TOOLS, JSON,
+// MD_JSON, JSON_SCHEMA.
+export type InstructorMode =
+  'FUNCTIONS' | 'TOOLS' | 'JSON' | 'MD_JSON' | 'JSON_SCHEMA';
+
+export const instructorMode = (process.env.LLM_INSTRUCTOR_MODE ??
+  'JSON_SCHEMA') as InstructorMode;
+
+// OpenAI-compatible client aimed at the configured provider (Together AI by
+// default, a LiteLLM proxy when `LLM_BASE_URL` is set).
+export function createLLMClient() {
+  return new OpenAI({
+    apiKey: llmApiKey,
+    baseURL: llmBaseURL,
+  });
+}

convex/together.ts

@@ -1,4 +1,3 @@
-import OpenAI from 'openai';
 import {
   internalAction,
   internalMutation,
@@ -9,19 +8,24 @@ import { internal } from './_generated/api';
 import { z } from 'zod';
 import { actionWithUser } from './utils';
 import Instructor from '@instructor-ai/instructor';
-
-const togetherApiKey = process.env.TOGETHER_API_KEY ?? 'undefined';
-
-// Together client for LLM extraction
-const togetherai = new OpenAI({
-  apiKey: togetherApiKey,
-  baseURL: 'https://api.together.xyz/v1',
-});
-
-// Instructor for returning structured JSON
+import {
+  createLLMClient,
+  chatModel,
+  embeddingModel,
+  instructorMode,
+} from './llm';
+
+// OpenAI-compatible client for LLM extraction + embeddings. Points at Together
+// AI by default; set LLM_BASE_URL to route through a LiteLLM proxy or any other
+// OpenAI-compatible gateway. See convex/llm.ts.
+const togetherai = createLLMClient();
+
+// Instructor for returning structured JSON. Defaults to Together's JSON_SCHEMA
+// mode; override with LLM_INSTRUCTOR_MODE=TOOLS when routing through a gateway
+// to a provider that doesn't support Together's response_format.schema.
 const client = Instructor({
   client: togetherai,
-  mode: 'JSON_SCHEMA',
+  mode: instructorMode,
 });
 
 const NoteSchema = z.object({
@@ -59,7 +63,7 @@ export const chat = internalAction({
           },
           { role: 'user', content: transcript },
         ],
-        model: 'Qwen/Qwen2.5-7B-Instruct-Turbo',
+        model: chatModel,
         response_model: { schema: NoteSchema, name: 'SummarizeNotes' },
         max_tokens: 1000,
         temperature: 0.6,
@@ -143,7 +147,7 @@ export const similarNotes = actionWithUser({
   handler: async (ctx, args): Promise<SearchResult[]> => {
     const getEmbedding = await togetherai.embeddings.create({
       input: [args.searchQuery.replace('/n', ' ')],
-      model: 'intfloat/multilingual-e5-large-instruct',
+      model: embeddingModel,
     });
     const embedding = getEmbedding.data[0].embedding;
 
@@ -171,7 +175,7 @@ export const embed = internalAction({
   handler: async (ctx, args) => {
     const getEmbedding = await togetherai.embeddings.create({
       input: [args.transcript.replace('/n', ' ')],
-      model: 'intfloat/multilingual-e5-large-instruct',
+      model: embeddingModel,
     });
     const embedding = getEmbedding.data[0].embedding;