roostorg · haileyok · Jun 4, 2026 · Jun 4, 2026 · Jun 4, 2026 · Jun 4, 2026
@@ -125,6 +125,25 @@ def register_ast_validators() -> None:
     # Register AST validators
 ```
 
+### Available hooks
+
+Implement any subset of these in your plugin's `register_plugins.py`:
+
+| Hook | Returns | Notes |
+| --- | --- | --- |
+| `register_udfs` | `Sequence[Type[UDFBase]]` | Custom user-defined functions. |
+| `register_output_sinks` | `Sequence[BaseOutputSink]` | Where execution results go. |
+| `register_ast_validators` | `Sequence[Type[BaseValidator]]` | Extra SML validators. |
+| `register_action_proto_deserializer` | `ActionProtoDeserializer \| None` | Custom action proto → JSON. |
+| `register_input_stream` | `BaseInputStream` | Single-provider (`firstresult`). |
+| `register_execution_result_store` | `ExecutionResultStore` | Single-provider (`firstresult`). |
+| `register_labels_service_or_provider` | `LabelsServiceBase \| LabelsProvider` | Single-provider (`firstresult`). |
+| `register_llm_provider` | `BaseLLMProvider` | Single-provider (`firstresult`). LLM API access for AI-assisted features. |
+
+The `register_llm_provider` hook and the vendor-neutral tool-calling helpers
+(`@tool`, `ToolRegistry`, `run_tool_loop`) have their own page:
+[LLM provider & tool calling](llm.md).
+
 ## Rules
 
 Rules are written in SML, some examples are provided in `example_rules/` with YAML config, the rules are mounted to the worker processes when the containers start via environment variables. ex:

@@ -10,6 +10,7 @@
   - [Release Process](development/releases.md)
 - [User Interface](UI.md)
 - [Writing Rules](rules.md)
+- [LLM Provider & Tool Calling](llm.md)
 - [User Research & Personas](user_personas.md)
 
 ---

@@ -0,0 +1,65 @@
+# LLM Provider & Tool Calling
+
+Osprey exposes an optional, vendor-neutral interface for LLM API access, used by
+AI-assisted features such as natural-language query building. It lives in
+`osprey.worker.lib.llm` and is **tool-calling aware**: you pass `ToolDefinition`s
+in, the model may return `ToolCall`s, and you feed `ToolResult`s back on the next
+`chat()` call.
+
+## The `register_llm_provider` hook
+
+A plugin supplies the LLM API client by implementing the `register_llm_provider`
+hook:
+
+```python
+from osprey.worker.lib.config import Config
+from osprey.worker.lib.llm.base import BaseLLMProvider
+
+@hookimpl_osprey
+def register_llm_provider(config: Config) -> BaseLLMProvider:
+    return MyLLMProvider(config)
+```
+
+Only one provider may be registered (`firstresult=True`). Retrieve it with
+`bootstrap_llm_provider(config)` from `osprey.worker.adaptor.plugin_manager`, which
+returns `None` when no plugin registers one — so callers should null-check.
+
+A direct Anthropic implementation is provided as a reference in
+`example_plugins/src/llm/anthropic_provider.py`, including the request/response and
+`tool_use` translation. The `anthropic` SDK is a dependency of `example_plugins`
+(installed by `uv sync`); set `ANTHROPIC_API_KEY` (or the
+`OSPREY_LLM_ANTHROPIC_API_KEY` config key) to use it. The example plugins do **not**
+register it by default — add your own `register_llm_provider` hookimpl to enable it.
+
+## Declaring tools and running a tool loop
+
+`osprey.worker.lib.llm` includes an optional, vendor-neutral tool-calling layer.
+Declare tools with the `@tool` decorator on a `ToolRegistry` (it compiles
+`ToolParameter`s into the `ToolDefinition` JSON Schema the provider consumes), then
+let `run_tool_loop` drive the call/dispatch/feed-back exchange until the model
+returns a final answer. Handlers are plain synchronous callables.
+
+```python
+from osprey.worker.lib.llm import ToolParameter, ToolRegistry, run_tool_loop, LLMMessage
+
+registry = ToolRegistry()
+
+@registry.tool(
+    name='lookup_user',
+    description='Look up a user by id.',
+    parameters=[ToolParameter(name='id', type='integer', description='User id')],
+)
+def lookup_user(id: int) -> dict:
+    return {'id': id, 'name': '...'}
+
+response = run_tool_loop(
+    provider,                       # any BaseLLMProvider
+    messages=[LLMMessage(role='user', content='Who is user 7?')],
+    registry=registry,
+)
+```
+
+`registry.dispatch(tool_call)` runs a single tool and captures errors as a
+`ToolResult` with `is_error=True` (fed back to the model rather than aborting).
+`run_tool_loop` raises `ToolLoopLimitExceeded` if the model keeps requesting tools
+past `max_iterations`.
@@ -4,7 +4,8 @@ version = "0.1.0"
 description = "Example plugins for Osprey"
 requires-python = ">=3.11"
 dependencies = [
-    "pluggy==1.5.0"
+    "pluggy==1.5.0",
+    "anthropic>=0.40.0",
 ]
 
 [tool.setuptools]

@@ -0,0 +1,5 @@
+"""Example LLM provider plugins for Osprey.
+
+See :mod:`llm.anthropic_provider` for a direct Anthropic Messages API implementation
+of :class:`osprey.worker.lib.llm.base.BaseLLMProvider`.
+"""
@@ -0,0 +1,245 @@
+"""Example LLM provider backed directly by the Anthropic Messages API.
+
+This demonstrates implementing :class:`osprey.worker.lib.llm.base.BaseLLMProvider`,
+including tool calling: it translates the vendor-neutral ``LLMMessage`` /
+``ToolDefinition`` types into Anthropic's request format, and maps the response
+(including ``tool_use`` blocks) back into ``LLMResponse`` / ``ToolCall``.
+
+The ``anthropic`` SDK is a dependency of ``example_plugins``. The client is built
+lazily on first use, so instantiating the provider does not require an API key.
+
+Configuration (via Osprey ``Config`` or environment):
+
+- API key: ``OSPREY_LLM_ANTHROPIC_API_KEY`` config key, else the ``ANTHROPIC_API_KEY``
+  environment variable (read by the SDK itself if neither is set explicitly).
+- Default model: ``OSPREY_LLM_ANTHROPIC_MODEL`` config key
+  (default: ``claude-sonnet-4-6``).
+- Default max tokens: ``OSPREY_LLM_ANTHROPIC_MAX_TOKENS`` config key (default: ``1024``).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union
+
+from osprey.worker.lib.config import Config
+from osprey.worker.lib.llm.base import (
+    BaseLLMProvider,
+    CacheControl,
+    LLMMessage,
+    LLMResponse,
+    LLMUsage,
+    ToolCall,
+    ToolDefinition,
+)
+
+if TYPE_CHECKING:
+    import anthropic
+
+DEFAULT_MODEL = 'claude-sonnet-4-6'
+DEFAULT_MAX_TOKENS = 1024
+
+
+class AnthropicLLMProvider(BaseLLMProvider):
+    """A :class:`BaseLLMProvider` that calls the Anthropic Messages API directly."""
+
+    def __init__(self, config: Config, client: Optional[anthropic.Anthropic] = None) -> None:
+        self._config = config
+        self._default_model = config.get_str('OSPREY_LLM_ANTHROPIC_MODEL', DEFAULT_MODEL)
+        self._default_max_tokens = config.get_int('OSPREY_LLM_ANTHROPIC_MAX_TOKENS', DEFAULT_MAX_TOKENS)
+        # Allow injecting a client (used in tests); otherwise build lazily on first use.
+        self._client = client
+
+    def _get_client(self) -> anthropic.Anthropic:
+        if self._client is not None:
+            return self._client
+
+        try:
+            import anthropic
+        except ImportError as exc:  # pragma: no cover - anthropic is a declared dependency
+            raise RuntimeError(
+                "The 'anthropic' package is required to use AnthropicLLMProvider. "
+                'It is a dependency of example_plugins, so run `uv sync` to install it.'
+            ) from exc
+
+        api_key = self._config.get_optional_str('OSPREY_LLM_ANTHROPIC_API_KEY') or os.environ.get('ANTHROPIC_API_KEY')
+        # If api_key is None the SDK still reads ANTHROPIC_API_KEY from the environment itself.
+        self._client = anthropic.Anthropic(api_key=api_key) if api_key else anthropic.Anthropic()
+        return self._client
+
+    def chat(
+        self,
+        *,
+        messages: Sequence[LLMMessage],
+        system: Optional[str] = None,
+        tools: Optional[Sequence[ToolDefinition]] = None,
+        model: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+        temperature: Optional[float] = None,
+        **params: Any,
+    ) -> LLMResponse:
+        request: Dict[str, Any] = {
+            # Use `is not None` rather than `or` so an explicit max_tokens=0 or
+            # model='' is passed through (and rejected by the API) instead of
+            # silently falling back to the default.
+            'model': model if model is not None else self._default_model,
+            'max_tokens': max_tokens if max_tokens is not None else self._default_max_tokens,
+            'messages': self._to_anthropic_messages(messages),
+        }
+
+        system_value = self._build_system(system, messages)
+        if system_value is not None:
+            request['system'] = system_value
+
+        if tools:
+            request['tools'] = [self._to_anthropic_tool(tool) for tool in tools]
+
+        if temperature is not None:
+            request['temperature'] = temperature
+
+        # Provider-specific passthrough (e.g. top_p, stop_sequences, tool_choice).
+        request.update(params)
+
+        response = self._get_client().messages.create(**request)
+        return self._from_anthropic_response(response)
+
+    # --- request translation ------------------------------------------------
+
+    @classmethod
+    def _build_system(
+        cls, system: Optional[str], messages: Sequence[LLMMessage]
+    ) -> Union[str, List[Dict[str, Any]], None]:
+        # Anthropic carries the system prompt as a top-level field, not a message,
+        # so the `system` argument and any role='system' messages are folded here.
+        # Each part keeps its own optional cache_control breakpoint.
+        parts: List[Tuple[str, Optional[CacheControl]]] = []
+        if system:
+            parts.append((system, None))
+        for message in messages:
+            if message.role == 'system' and message.content:
+                parts.append((message.content, message.cache_control))
+
+        if not parts:
+            return None
+
+        # When nothing needs a cache breakpoint, the simple string form suffices.
+        if all(cache_control is None for _, cache_control in parts):
+            return '\n\n'.join(text for text, _ in parts)
+
+        # Otherwise emit the structured block form so per-part cache_control is
+        # preserved (a string `system` cannot carry cache breakpoints).
+        blocks: List[Dict[str, Any]] = []
+        for text, cache_control in parts:
+            block: Dict[str, Any] = {'type': 'text', 'text': text}
+            if cache_control is not None:
+                block['cache_control'] = cls._cache_control_dict(cache_control)
+            blocks.append(block)
+        return blocks
+
+    @staticmethod
+    def _cache_control_dict(cache_control: CacheControl) -> Dict[str, Any]:
+        result: Dict[str, Any] = {'type': 'ephemeral'}
+        if cache_control.ttl is not None:
+            # Opt into a non-default cache duration (e.g. '1h'); the default 5m
+            # ephemeral cache needs no ttl field. The 1h TTL is generally available
+            # on the Claude API — set via the ttl field, no beta header required.
+            result['ttl'] = cache_control.ttl
+        return result
+
+    @staticmethod
+    def _to_anthropic_tool(tool: ToolDefinition) -> Dict[str, Any]:
+        return {
+            'name': tool.name,
+            'description': tool.description,
+            'input_schema': tool.input_schema,
+        }
+
+    @classmethod
+    def _to_anthropic_messages(cls, messages: Sequence[LLMMessage]) -> List[Dict[str, Any]]:
+        out: List[Dict[str, Any]] = []
+        for message in messages:
+            # System messages are handled separately via the top-level `system` field.
+            if message.role == 'system':
+                continue
+
+            blocks = cls._message_content_blocks(message)
+            if not blocks:
+                continue
+
+            if message.cache_control is not None:
+                blocks[-1]['cache_control'] = cls._cache_control_dict(message.cache_control)
+
+            # Tool results are surfaced to Anthropic as a user-role message.
+            role = 'user' if message.role == 'tool' else message.role
+            out.append({'role': role, 'content': blocks})
+        return out
+
+    @staticmethod
+    def _message_content_blocks(message: LLMMessage) -> List[Dict[str, Any]]:
+        blocks: List[Dict[str, Any]] = []
+
+        if message.content:
+            blocks.append({'type': 'text', 'text': message.content})
+
+        for tool_call in message.tool_calls:
+            blocks.append(
+                {
+                    'type': 'tool_use',
+                    'id': tool_call.id,
+                    'name': tool_call.name,
+                    'input': tool_call.arguments,
+                }
+            )
+
+        for tool_result in message.tool_results:
+            blocks.append(
+                {
+                    'type': 'tool_result',
+                    'tool_use_id': tool_result.tool_call_id,
+                    'content': tool_result.content,
+                    'is_error': tool_result.is_error,
+                }
+            )
+
+        return blocks
+
+    # --- response translation ------------------------------------------------
+
+    @staticmethod
+    def _from_anthropic_response(response: Any) -> LLMResponse:
+        text_parts: List[str] = []
+        tool_calls: List[ToolCall] = []
+
+        for block in getattr(response, 'content', None) or []:
+            block_type = getattr(block, 'type', None)
+            if block_type == 'text':
+                text_parts.append(getattr(block, 'text', '') or '')
+            elif block_type == 'tool_use':
+                raw_input = getattr(block, 'input', None)
+                tool_calls.append(
+                    ToolCall(
+                        id=getattr(block, 'id', ''),
+                        name=getattr(block, 'name', ''),
+                        # Degrade gracefully (like the other fields) if the SDK ever
+                        # hands back a non-dict input rather than crashing.
+                        arguments=dict(raw_input) if isinstance(raw_input, dict) else {},
+                    )
+                )
+
+        usage: Optional[LLMUsage] = None
+        raw_usage = getattr(response, 'usage', None)
+        if raw_usage is not None:
+            usage = LLMUsage(
+                input_tokens=getattr(raw_usage, 'input_tokens', 0) or 0,
+                output_tokens=getattr(raw_usage, 'output_tokens', 0) or 0,
+                cache_read_tokens=getattr(raw_usage, 'cache_read_input_tokens', 0) or 0,
+                cache_write_tokens=getattr(raw_usage, 'cache_creation_input_tokens', 0) or 0,
+            )
+
+        return LLMResponse(
+            text=''.join(text_parts),
+            tool_calls=tool_calls,
+            stop_reason=getattr(response, 'stop_reason', None),
+            usage=usage,
+            raw=response,
+        )