From e374eac98108ea6f5f7d2db434e8a2738cce03ff Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 4 Sep 2026 15:30:50 +0100 Subject: [PATCH] Add llmman as a local model provider llmman (https://github.com/llmmanorg/llmman) is a local model runner that serves the Ollama, OpenAI and Anthropic APIs on port 17434. Wire it in as a peer of the Ollama and LM Studio integrations: a `llmman/` prefix in the OpenAI completion provider (LLMMAN_API_BASE, default localhost:17434/v1), `r2r` routing for that prefix, a `llmman.toml` config, and LLMMAN_API_BASE in the Docker env files, swarm compose and the local-models cookbook. --- docker/compose.full.swarm.yaml | 3 ++ docker/env/r2r-full.env | 3 ++ docker/env/r2r.env | 3 ++ docs/cookbooks/local.md | 2 ++ py/core/configs/llmman.toml | 56 ++++++++++++++++++++++++++++++++ py/core/providers/llm/openai.py | 33 ++++++++++++++++++- py/core/providers/llm/r2r_llm.py | 3 +- 7 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 py/core/configs/llmman.toml diff --git a/docker/compose.full.swarm.yaml b/docker/compose.full.swarm.yaml index f8e431cdc3..28820e03f9 100644 --- a/docker/compose.full.swarm.yaml +++ b/docker/compose.full.swarm.yaml @@ -305,6 +305,9 @@ services: # Ollama - OLLAMA_API_BASE=${OLLAMA_API_BASE:-http://host.docker.internal:11434} + # llmman + - LLMMAN_API_BASE=${LLMMAN_API_BASE:-http://host.docker.internal:17434/v1} + # LM Studio - LM_STUDIO_API_BASE=${LM_STUDIO_API_BASE:-http://host.docker.internal:1234} - LM_STUDIO_API_KEY=${LM_STUDIO_API_KEY:-1234} diff --git a/docker/env/r2r-full.env b/docker/env/r2r-full.env index 1bc648cc63..6c35bd942a 100644 --- a/docker/env/r2r-full.env +++ b/docker/env/r2r-full.env @@ -68,6 +68,9 @@ ANYSCALE_API_KEY= # Ollama OLLAMA_API_BASE=http://host.docker.internal:11434 +# llmman (https://github.com/llmmanorg/llmman) +LLMMAN_API_BASE=http://host.docker.internal:17434/v1 + # LM Studio LM_STUDIO_API_BASE=http://host.docker.internal:1234 LM_STUDIO_API_KEY=1234 diff --git a/docker/env/r2r.env b/docker/env/r2r.env index dd14a07e25..e0cb75eb2d 100644 --- a/docker/env/r2r.env +++ b/docker/env/r2r.env @@ -68,6 +68,9 @@ ANYSCALE_API_KEY= # Ollama OLLAMA_API_BASE=http://host.docker.internal:11434 +# llmman (https://github.com/llmmanorg/llmman) +LLMMAN_API_BASE=http://host.docker.internal:17434/v1 + # LM Studio LM_STUDIO_API_BASE=http://host.docker.internal:1234 LM_STUDIO_API_KEY=1234 diff --git a/docs/cookbooks/local.md b/docs/cookbooks/local.md index 3946dab4c1..c01277a2a0 100644 --- a/docs/cookbooks/local.md +++ b/docs/cookbooks/local.md @@ -11,6 +11,8 @@ For this cookbook, we'll serve our local models via Ollama. [You may follow the You can also follow along using LM Studio. To get started with LM Studio, see our [Local LLM documentation](/self-hosting/local-rag). +Or with [llmman](https://github.com/llmmanorg/llmman), a local model runner that serves the Ollama API (alongside OpenAI- and Anthropic-compatible ones) on port 17434. Pull models with `llmman pull gemma4` (or straight from Hugging Face, e.g. `llmman pull hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF`), start it with `llmman serve`, then launch R2R with `R2R_CONFIG_NAME=llmman`. Use the `llmman/` model prefix and set `LLMMAN_API_BASE` (default `http://localhost:17434/v1`) if the server runs elsewhere. + R2R supports [LiteLLM](https://github.com/BerriAI/litellm) for routing embedding and completion requests. This allows for OpenAI-compatible endpoints to be called and seamlessly routed to, if you are serving local models another way. diff --git a/py/core/configs/llmman.toml b/py/core/configs/llmman.toml new file mode 100644 index 0000000000..5511005b14 --- /dev/null +++ b/py/core/configs/llmman.toml @@ -0,0 +1,56 @@ +# llmman (https://github.com/llmmanorg/llmman) serves the Ollama and OpenAI APIs on +# port 17434. Start it with `llmman serve` and pull the models below with +# `llmman pull gemma4` and `llmman pull hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF`. +# Override the server location with `LLMMAN_API_BASE` (default http://localhost:17434/v1). + +[app] +# LLM used for internal operations, like deriving conversation names +fast_llm = "llmman/gemma4" + +# LLM used for user-facing output, like RAG replies +quality_llm = "llmman/gemma4" + +# LLM used for ingesting visual inputs +vlm = "llmman/gemma4" # TODO - Replace with viable candidate + +# LLM used for transcription +audio_lm = "llmman/gemma4" # TODO - Replace with viable candidate + + +# Reasoning model, used for `research` agent +reasoning_llm = "llmman/gemma4" +# Planning model, used for `research` agent +planning_llm = "llmman/gemma4" + +# Embeddings go through llmman's OpenAI `/v1/embeddings` route; no API key is needed. +[embedding] +provider = "litellm" +base_model = "openai/hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF" +base_dimension = 768 +batch_size = 128 +concurrent_request_limit = 2 +api_base = "http://localhost:17434/v1" +api_key = "llmman" + +[completion_embedding] +provider = "litellm" +base_model = "openai/hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF" +base_dimension = 768 +batch_size = 128 +concurrent_request_limit = 2 +api_base = "http://localhost:17434/v1" +api_key = "llmman" + +[agent] +tools = ["search_file_knowledge"] + +# `r2r` routes the `llmman/` prefix to the OpenAI-compatible client at `LLMMAN_API_BASE`. +[completion] +provider = "r2r" +concurrent_request_limit = 1 + + [completion.generation_config] + temperature = 0.1 + top_p = 1 + max_tokens_to_sample = 1_024 + stream = false diff --git a/py/core/providers/llm/openai.py b/py/core/providers/llm/openai.py index e6155d784f..630aa3c35e 100644 --- a/py/core/providers/llm/openai.py +++ b/py/core/providers/llm/openai.py @@ -23,6 +23,8 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None: self.async_deepseek_client = None self.ollama_client = None self.async_ollama_client = None + self.llmman_client = None + self.async_llmman_client = None self.lmstudio_client = None self.async_lmstudio_client = None # NEW: Azure Foundry clients using the Azure Inference API @@ -86,6 +88,22 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None: ) logger.debug("Ollama OpenAI clients initialized successfully") + # Initialize llmman clients (https://github.com/llmmanorg/llmman). + # llmman serves the Ollama and OpenAI APIs on port 17434; no key needed. + llmman_api_base = os.getenv( + "LLMMAN_API_BASE", "http://localhost:17434/v1" + ) + if llmman_api_base: + self.llmman_client = OpenAI( + api_key=os.getenv("LLMMAN_API_KEY", "dummy"), + base_url=llmman_api_base, + ) + self.async_llmman_client = AsyncOpenAI( + api_key=os.getenv("LLMMAN_API_KEY", "dummy"), + base_url=llmman_api_base, + ) + logger.debug("llmman OpenAI clients initialized successfully") + # Initialize LMStudio clients lmstudio_api_base = os.getenv( "LMSTUDIO_API_BASE", "http://localhost:1234/v1" @@ -135,6 +153,7 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None: self.openai_client, self.azure_client, self.ollama_client, + self.llmman_client, self.lmstudio_client, self.azure_foundry_client, ] @@ -142,7 +161,7 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None: raise ValueError( "No valid client credentials found. Please set either OPENAI_API_KEY, " "both AZURE_API_KEY and AZURE_API_BASE environment variables, " - "OLLAMA_API_BASE, LMSTUDIO_API_BASE, or AZURE_FOUNDRY_API_KEY and AZURE_FOUNDRY_API_ENDPOINT." + "OLLAMA_API_BASE, LLMMAN_API_BASE, LMSTUDIO_API_BASE, or AZURE_FOUNDRY_API_KEY and AZURE_FOUNDRY_API_ENDPOINT." ) def _get_client_and_model(self, model: str): @@ -172,6 +191,12 @@ def _get_client_and_model(self, model: str): "Ollama OpenAI credentials not configured but ollama/ model prefix used" ) return self.ollama_client, model[7:] # Strip 'ollama/' prefix + elif model.startswith("llmman/"): + if not self.llmman_client: + raise ValueError( + "llmman credentials not configured but llmman/ model prefix used" + ) + return self.llmman_client, model[7:] # Strip 'llmman/' prefix elif model.startswith("lmstudio/"): if not self.lmstudio_client: raise ValueError( @@ -228,6 +253,12 @@ def _get_async_client_and_model(self, model: str): "Ollama OpenAI credentials not configured but ollama/ model prefix used" ) return self.async_ollama_client, model[7:] + elif model.startswith("llmman/"): + if not self.async_llmman_client: + raise ValueError( + "llmman credentials not configured but llmman/ model prefix used" + ) + return self.async_llmman_client, model[7:] elif model.startswith("lmstudio/"): if not self.async_lmstudio_client: raise ValueError( diff --git a/py/core/providers/llm/r2r_llm.py b/py/core/providers/llm/r2r_llm.py index b95b310a8d..0523604473 100644 --- a/py/core/providers/llm/r2r_llm.py +++ b/py/core/providers/llm/r2r_llm.py @@ -17,7 +17,7 @@ class R2RCompletionProvider(CompletionProvider): - If `generation_config.model` starts with "anthropic/", call AnthropicCompletionProvider. - If it starts with "azure-foundry/", call AzureFoundryCompletionProvider. - - If it starts with one of the other OpenAI-like prefixes ("openai/", "azure/", "deepseek/", "ollama/", "lmstudio/") + - If it starts with one of the other OpenAI-like prefixes ("openai/", "azure/", "deepseek/", "ollama/", "llmman/", "lmstudio/") or has no prefix (e.g. "gpt-4", "gpt-3.5"), call OpenAICompletionProvider. - Otherwise, fallback to LiteLLMCompletionProvider. """ @@ -65,6 +65,7 @@ def _choose_subprovider_by_model( "azure/", "deepseek/", "ollama/", + "llmman/", "lmstudio/", ] if (