Add llmman as a local model provider

docker/compose.full.swarm.yaml

@@ -305,6 +305,9 @@ services:
       # Ollama
       - OLLAMA_API_BASE=${OLLAMA_API_BASE:-http://host.docker.internal:11434}
 
+      # llmman
+      - LLMMAN_API_BASE=${LLMMAN_API_BASE:-http://host.docker.internal:17434/v1}
+
       # LM Studio
       - LM_STUDIO_API_BASE=${LM_STUDIO_API_BASE:-http://host.docker.internal:1234}
       - LM_STUDIO_API_KEY=${LM_STUDIO_API_KEY:-1234}

docker/env/r2r-full.env

@@ -68,6 +68,9 @@ ANYSCALE_API_KEY=
 # Ollama
 OLLAMA_API_BASE=http://host.docker.internal:11434
 
+# llmman (https://github.com/llmmanorg/llmman)
+LLMMAN_API_BASE=http://host.docker.internal:17434/v1
+
 # LM Studio
 LM_STUDIO_API_BASE=http://host.docker.internal:1234
 LM_STUDIO_API_KEY=1234

docker/env/r2r.env

@@ -68,6 +68,9 @@ ANYSCALE_API_KEY=
 # Ollama
 OLLAMA_API_BASE=http://host.docker.internal:11434
 
+# llmman (https://github.com/llmmanorg/llmman)
+LLMMAN_API_BASE=http://host.docker.internal:17434/v1
+
 # LM Studio
 LM_STUDIO_API_BASE=http://host.docker.internal:1234
 LM_STUDIO_API_KEY=1234

docs/cookbooks/local.md

@@ -11,6 +11,8 @@ For this cookbook, we'll serve our local models via Ollama. [You may follow the
 
 You can also follow along using LM Studio. To get started with LM Studio, see our [Local LLM documentation](/self-hosting/local-rag).
 
+Or with [llmman](https://github.com/llmmanorg/llmman), a local model runner that serves the Ollama API (alongside OpenAI- and Anthropic-compatible ones) on port 17434. Pull models with `llmman pull gemma4` (or straight from Hugging Face, e.g. `llmman pull hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF`), start it with `llmman serve`, then launch R2R with `R2R_CONFIG_NAME=llmman`. Use the `llmman/` model prefix and set `LLMMAN_API_BASE` (default `http://localhost:17434/v1`) if the server runs elsewhere.
+
 R2R supports [LiteLLM](https://github.com/BerriAI/litellm) for routing embedding and completion requests. This allows for OpenAI-compatible endpoints to be called and seamlessly routed to, if you are serving local models another way.
 </Note>
 

py/core/configs/llmman.toml

@@ -0,0 +1,56 @@
+# llmman (https://github.com/llmmanorg/llmman) serves the Ollama and OpenAI APIs on
+# port 17434. Start it with `llmman serve` and pull the models below with
+# `llmman pull gemma4` and `llmman pull hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF`.
+# Override the server location with `LLMMAN_API_BASE` (default http://localhost:17434/v1).
+
+[app]
+# LLM used for internal operations, like deriving conversation names
+fast_llm = "llmman/gemma4"
+
+# LLM used for user-facing output, like RAG replies
+quality_llm = "llmman/gemma4"
+
+# LLM used for ingesting visual inputs
+vlm = "llmman/gemma4" # TODO - Replace with viable candidate
+
+# LLM used for transcription
+audio_lm = "llmman/gemma4" # TODO - Replace with viable candidate
+
+
+# Reasoning model, used for `research` agent
+reasoning_llm = "llmman/gemma4"
+# Planning model, used for `research` agent
+planning_llm = "llmman/gemma4"
+
+# Embeddings go through llmman's OpenAI `/v1/embeddings` route; no API key is needed.
+[embedding]
+provider = "litellm"
+base_model = "openai/hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF"
+base_dimension = 768
+batch_size = 128
+concurrent_request_limit = 2
+api_base = "http://localhost:17434/v1"
+api_key = "llmman"
+
+[completion_embedding]
+provider = "litellm"
+base_model = "openai/hf.co/nomic-ai/nomic-embed-text-v1.5-GGUF"
+base_dimension = 768
+batch_size = 128
+concurrent_request_limit = 2
+api_base = "http://localhost:17434/v1"
+api_key = "llmman"
+
+[agent]
+tools = ["search_file_knowledge"]
+
+# `r2r` routes the `llmman/` prefix to the OpenAI-compatible client at `LLMMAN_API_BASE`.
+[completion]
+provider = "r2r"
+concurrent_request_limit = 1
+
+  [completion.generation_config]
+  temperature = 0.1
+  top_p = 1
+  max_tokens_to_sample = 1_024
+  stream = false

py/core/providers/llm/openai.py

@@ -23,6 +23,8 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None:
         self.async_deepseek_client = None
         self.ollama_client = None
         self.async_ollama_client = None
+        self.llmman_client = None
+        self.async_llmman_client = None
         self.lmstudio_client = None
         self.async_lmstudio_client = None
         # NEW: Azure Foundry clients using the Azure Inference API
@@ -86,6 +88,22 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None:
             )
             logger.debug("Ollama OpenAI clients initialized successfully")
 
+        # Initialize llmman clients (https://github.com/llmmanorg/llmman).
+        # llmman serves the Ollama and OpenAI APIs on port 17434; no key needed.
+        llmman_api_base = os.getenv(
+            "LLMMAN_API_BASE", "http://localhost:17434/v1"
+        )
+        if llmman_api_base:
+            self.llmman_client = OpenAI(
+                api_key=os.getenv("LLMMAN_API_KEY", "dummy"),
+                base_url=llmman_api_base,
+            )
+            self.async_llmman_client = AsyncOpenAI(
+                api_key=os.getenv("LLMMAN_API_KEY", "dummy"),
+                base_url=llmman_api_base,
+            )
+            logger.debug("llmman OpenAI clients initialized successfully")
+
         # Initialize LMStudio clients
         lmstudio_api_base = os.getenv(
             "LMSTUDIO_API_BASE", "http://localhost:1234/v1"
@@ -135,14 +153,15 @@ def __init__(self, config: CompletionConfig, *args, **kwargs) -> None:
                 self.openai_client,
                 self.azure_client,
                 self.ollama_client,
+                self.llmman_client,
                 self.lmstudio_client,
                 self.azure_foundry_client,
             ]
         ):
             raise ValueError(
                 "No valid client credentials found. Please set either OPENAI_API_KEY, "
                 "both AZURE_API_KEY and AZURE_API_BASE environment variables, "
-                "OLLAMA_API_BASE, LMSTUDIO_API_BASE, or AZURE_FOUNDRY_API_KEY and AZURE_FOUNDRY_API_ENDPOINT."
+                "OLLAMA_API_BASE, LLMMAN_API_BASE, LMSTUDIO_API_BASE, or AZURE_FOUNDRY_API_KEY and AZURE_FOUNDRY_API_ENDPOINT."
             )
 
     def _get_client_and_model(self, model: str):
@@ -172,6 +191,12 @@ def _get_client_and_model(self, model: str):
                     "Ollama OpenAI credentials not configured but ollama/ model prefix used"
                 )
             return self.ollama_client, model[7:]  # Strip 'ollama/' prefix
+        elif model.startswith("llmman/"):
+            if not self.llmman_client:
+                raise ValueError(
+                    "llmman credentials not configured but llmman/ model prefix used"
+                )
+            return self.llmman_client, model[7:]  # Strip 'llmman/' prefix
         elif model.startswith("lmstudio/"):
             if not self.lmstudio_client:
                 raise ValueError(
@@ -228,6 +253,12 @@ def _get_async_client_and_model(self, model: str):
                     "Ollama OpenAI credentials not configured but ollama/ model prefix used"
                 )
             return self.async_ollama_client, model[7:]
+        elif model.startswith("llmman/"):
+            if not self.async_llmman_client:
+                raise ValueError(
+                    "llmman credentials not configured but llmman/ model prefix used"
+                )
+            return self.async_llmman_client, model[7:]
         elif model.startswith("lmstudio/"):
             if not self.async_lmstudio_client:
                 raise ValueError(

py/core/providers/llm/r2r_llm.py

@@ -17,7 +17,7 @@ class R2RCompletionProvider(CompletionProvider):
 
     - If `generation_config.model` starts with "anthropic/", call AnthropicCompletionProvider.
     - If it starts with "azure-foundry/", call AzureFoundryCompletionProvider.
-    - If it starts with one of the other OpenAI-like prefixes ("openai/", "azure/", "deepseek/", "ollama/", "lmstudio/")
+    - If it starts with one of the other OpenAI-like prefixes ("openai/", "azure/", "deepseek/", "ollama/", "llmman/", "lmstudio/")
       or has no prefix (e.g. "gpt-4", "gpt-3.5"), call OpenAICompletionProvider.
     - Otherwise, fallback to LiteLLMCompletionProvider.
     """
@@ -65,6 +65,7 @@ def _choose_subprovider_by_model(
             "azure/",
             "deepseek/",
             "ollama/",
+            "llmman/",
             "lmstudio/",
         ]
         if (