From a8fae7929d5433bd8935f26602b674f104ebd2e9 Mon Sep 17 00:00:00 2001 From: kesku Date: Sun, 26 Jul 2026 21:29:22 +0100 Subject: [PATCH] feat(web): add native Exa web search provider --- README.md | 1 + config/config.example.json | 6 + docs/reference/tools_configuration.md | 37 ++++ docs/security/security_configuration.md | 18 +- pkg/config/config.go | 25 +++ pkg/config/defaults.go | 4 + pkg/config/example_security_usage.go | 14 +- pkg/config/security_integration_test.go | 13 ++ pkg/tools/integration/web.go | 164 +++++++++++++++++- pkg/tools/integration/web_test.go | 7 + pkg/tools/integration_facade.go | 1 + web/backend/api/tools.go | 20 +++ .../tools/web-search-provider-settings.tsx | 1 + 13 files changed, 301 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a351c98d69..9aef18a06e 100644 --- a/README.md +++ b/README.md @@ -511,6 +511,7 @@ PicoClaw can search the web to provide up-to-date information. Configure in `too | DuckDuckGo | Not needed | Unlimited | Built-in fallback | | [Gemini Google Search](https://aistudio.google.com/apikey) | Required | Varies | Gemini with Google Search grounding | | [Baidu Search](https://cloud.baidu.com/doc/qianfan-api/s/Wmbq4z7e5) | Required | 1500/month (daily allocation) | AI-powered, China-optimized | +| [Exa](https://exa.ai) | Required | Free $20 in credits on signup, $10 every month after that | Semantic search with highlights | | [Tavily](https://tavily.com) | Required | 1000 queries/month | Optimized for AI Agents | | [Brave Search](https://brave.com/search/api) | Required | 2000 queries/month | Fast and private | | [Kagi Search](https://help.kagi.com/kagi/api/search.html) | Required | Paid/limited by API setup | Premium search results | diff --git a/config/config.example.json b/config/config.example.json index de12d84cce..b47b9450cf 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -334,6 +334,12 @@ "base_url": "", "max_results": 0 }, + "exa": { + "enabled": false, + "api_key": "", + "api_keys": ["YOUR_EXA_API_KEY"], + "max_results": 5 + }, "kagi": { "enabled": false, "api_key": "", diff --git a/docs/reference/tools_configuration.md b/docs/reference/tools_configuration.md index c3a9140b58..ef27cfcf83 100644 --- a/docs/reference/tools_configuration.md +++ b/docs/reference/tools_configuration.md @@ -173,6 +173,43 @@ Kagi API usage may be billed or limited separately from a normal Kagi subscripti | `base_url` | string | - | Custom Tavily API base URL | | `max_results` | int | 5 | Maximum number of results | +### Exa + +[Exa](https://exa.ai) is a semantic web search API. PicoClaw calls `POST /search` with `type: "auto"` and requests +`contents.highlights`, so each result carries a relevant excerpt of the page. + +| Config | Type | Default | Description | +|---------------|----------|---------|------------------------------------------------| +| `enabled` | bool | false | Enable Exa search | +| `api_key` | string | - | Exa API key | +| `api_keys` | string[] | - | Multiple API keys for rotation (takes priority over `api_key`) | +| `max_results` | int | 5 | Maximum number of results | + +```json +{ + "tools": { + "web": { + "provider": "exa", + "exa": { + "enabled": true, + "max_results": 5 + } + } + } +} +``` + +Store Exa API keys in `.security.yml`: + +```yaml +web: + exa: + api_keys: + - "YOUR_EXA_API_KEY" +``` + +For Exa, `d`, `w`, `m`, and `y` map to a `startPublishedDate` filter relative to the current time. + ### SearXNG | Config | Type | Default | Description | diff --git a/docs/security/security_configuration.md b/docs/security/security_configuration.md index 4b5e4fe619..fc87871b84 100644 --- a/docs/security/security_configuration.md +++ b/docs/security/security_configuration.md @@ -248,7 +248,7 @@ channel_list: ### Web Tools -**Brave, Tavily, Perplexity, Kagi:** +**Brave, Tavily, Perplexity, Kagi, Exa:** ```yaml web: brave: @@ -258,6 +258,9 @@ web: kagi: api_keys: - "your-kagi-api-key" + exa: + api_keys: + - "your-exa-api-key" ``` - Use `api_keys` (plural) array format @@ -318,7 +321,7 @@ model_list: - **Rate limit management**: Distribute usage across multiple keys - **High availability**: Reduce downtime during API provider issues -### Web Tools (Brave/Tavily/Perplexity/Kagi) - Single key +### Web Tools (Brave/Tavily/Perplexity/Kagi/Exa) - Single key ```yaml web: @@ -328,9 +331,12 @@ web: kagi: api_keys: - "your-kagi-api-key" + exa: + api_keys: + - "your-exa-api-key" ``` -### Web Tools (Brave/Tavily/Perplexity/Kagi) - Multiple keys +### Web Tools (Brave/Tavily/Perplexity/Kagi/Exa) - Multiple keys ```yaml web: @@ -342,6 +348,10 @@ web: api_keys: - "kagi-key-1" - "kagi-key-2" + exa: + api_keys: + - "exa-key-1" + - "exa-key-2" ``` ### Web Tool (GLMSearch/BaiduSearch) - Single key only @@ -568,7 +578,7 @@ go test ./pkg/config -run TestSecurityConfig - Ensure you're using `api_keys` (plural) in `.security.yml` for models and web tools (except GLMSearch/BaiduSearch) - Check that the array format is correct in YAML (proper indentation with dashes) -- Remember: Models, Brave, Tavily, Perplexity, Kagi MUST use `api_keys` (array format) +- Remember: Models, Brave, Tavily, Perplexity, Kagi, Exa MUST use `api_keys` (array format) - GLMSearch and BaiduSearch MUST use `api_key` (single string format) ### Load Balancing/Failover Issues diff --git a/pkg/config/config.go b/pkg/config/config.go index df232f771a..d25432ebd2 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -941,6 +941,30 @@ func (c *TavilyConfig) SetAPIKeys(keys []string) { } } +type ExaConfig struct { + Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_TOOLS_WEB_EXA_ENABLED"` + APIKeys SecureStrings `json:"api_keys,omitzero" yaml:"api_keys,omitempty" env:"PICOCLAW_TOOLS_WEB_EXA_API_KEYS"` + MaxResults int `json:"max_results" yaml:"-" env:"PICOCLAW_TOOLS_WEB_EXA_MAX_RESULTS"` +} + +// APIKey returns the Exa API key +func (c *ExaConfig) APIKey() string { + if len(c.APIKeys) == 0 { + return "" + } + return c.APIKeys[0].String() +} + +// SetAPIKey sets the Exa API key +func (c *ExaConfig) SetAPIKey(key string) { + c.APIKeys = SimpleSecureStrings(key) +} + +// SetAPIKeys sets the Exa API keys +func (c *ExaConfig) SetAPIKeys(keys []string) { + c.APIKeys = SimpleSecureStrings(keys...) +} + type KagiConfig struct { Enabled bool `json:"enabled" yaml:"-" env:"PICOCLAW_TOOLS_WEB_KAGI_ENABLED"` APIKeys SecureStrings `json:"api_keys,omitzero" yaml:"api_keys,omitempty" env:"PICOCLAW_TOOLS_WEB_KAGI_API_KEYS"` @@ -1029,6 +1053,7 @@ type WebToolsConfig struct { ToolConfig ` yaml:"-" envPrefix:"PICOCLAW_TOOLS_WEB_"` Brave BraveConfig `yaml:"brave,omitempty" json:"brave"` Tavily TavilyConfig `yaml:"tavily,omitempty" json:"tavily"` + Exa ExaConfig `yaml:"exa,omitempty" json:"exa"` Kagi KagiConfig `yaml:"kagi,omitempty" json:"kagi"` Sogou SogouConfig `yaml:"-" json:"sogou"` DuckDuckGo DuckDuckGoConfig `yaml:"-" json:"duckduckgo"` diff --git a/pkg/config/defaults.go b/pkg/config/defaults.go index 96ce5f0f48..819e232182 100644 --- a/pkg/config/defaults.go +++ b/pkg/config/defaults.go @@ -341,6 +341,10 @@ func DefaultConfig() *Config { Enabled: false, MaxResults: 5, }, + Exa: ExaConfig{ + Enabled: false, + MaxResults: 5, + }, Kagi: KagiConfig{ Enabled: false, BaseURL: "https://kagi.com/api/v1/search", diff --git a/pkg/config/example_security_usage.go b/pkg/config/example_security_usage.go index de80f541a4..3c60504dd1 100644 --- a/pkg/config/example_security_usage.go +++ b/pkg/config/example_security_usage.go @@ -51,7 +51,7 @@ channels: token: "your-discord-bot-token" # Web Tool Keys -# Brave, Tavily, Perplexity, Kagi: Use 'api_keys' array +# Brave, Tavily, Perplexity, Kagi, Exa: Use 'api_keys' array # GLMSearch, BaiduSearch: Use 'api_key' single string web: @@ -68,6 +68,9 @@ web: kagi: api_keys: - "your-kagi-api-key" # Single key in array format + exa: + api_keys: + - "your-exa-api-key" # Single key in array format glm_search: api_key: "your-glm-search-api-key" # Single key (not array) baidu_search: @@ -242,7 +245,7 @@ channels: ## Web Tool API Keys -**Brave, Tavily, Perplexity, Kagi:** +**Brave, Tavily, Perplexity, Kagi, Exa:** ```yaml web: @@ -259,6 +262,9 @@ web: kagi: api_keys: - "kagi-key" + exa: + api_keys: + - "exa-key" ``` Use `api_keys` (plural) array format. @@ -449,7 +455,7 @@ web: ## Single Key Format -**Models, Brave, Tavily, Perplexity, Kagi:** +**Models, Brave, Tavily, Perplexity, Kagi, Exa:** ```yaml model_list: @@ -571,7 +577,7 @@ and .security.yml values. ## Multiple API Keys Not Working - Ensure you're using `api_keys` (plural) in .security.yml for models and web tools (except GLMSearch/BaiduSearch) - Check that the array format is correct in YAML (proper indentation with dashes) -- Remember: Models, Brave, Tavily, Perplexity, Kagi MUST use `api_keys` (array format) +- Remember: Models, Brave, Tavily, Perplexity, Kagi, Exa MUST use `api_keys` (array format) - GLMSearch and BaiduSearch MUST use `api_key` (single string format) ## Keys Not Being Applied diff --git a/pkg/config/security_integration_test.go b/pkg/config/security_integration_test.go index cac7690901..e876997286 100644 --- a/pkg/config/security_integration_test.go +++ b/pkg/config/security_integration_test.go @@ -195,6 +195,10 @@ func TestAllSecurityKeysAccessible(t *testing.T) { err = os.WriteFile(kagiAPIKeyFile, []byte("kagi-from-file-33333"), 0o600) require.NoError(t, err) + exaAPIKeyFile := filepath.Join(tmpDir, "exa_api_key.txt") + err = os.WriteFile(exaAPIKeyFile, []byte("exa-from-file-44444"), 0o600) + require.NoError(t, err) + githubTokenFile := filepath.Join(tmpDir, "github_token.txt") err = os.WriteFile(githubTokenFile, []byte("ghp-github-from-file-abc123"), 0o600) require.NoError(t, err) @@ -277,6 +281,9 @@ func TestAllSecurityKeysAccessible(t *testing.T) { "kagi": { "enabled": true }, + "exa": { + "enabled": true + }, "glm_search": { "enabled": true } @@ -341,6 +348,9 @@ web: kagi: api_keys: - "file://kagi_api_key.txt" + exa: + api_keys: + - "file://exa_api_key.txt" glm_search: api_key: "glm-test-glm-search-key" @@ -469,6 +479,9 @@ skills: assert.Equal(t, "kagi-from-file-33333", cfg.Tools.Web.Kagi.APIKey()) t.Logf("Kagi APIKey(): %s", cfg.Tools.Web.Kagi.APIKey()) + assert.Equal(t, "exa-from-file-44444", cfg.Tools.Web.Exa.APIKey()) + t.Logf("Exa APIKey(): %s", cfg.Tools.Web.Exa.APIKey()) + // GLM Search - Note: GLM uses SetAPIKey (lowercase) internally t.Logf("GLMSearch APIKey(): %s", cfg.Tools.Web.GLMSearch.APIKey.String()) assert.Equal(t, "glm-test-glm-search-key", cfg.Tools.Web.GLMSearch.APIKey.String()) diff --git a/pkg/tools/integration/web.go b/pkg/tools/integration/web.go index 3220941934..6159d77142 100644 --- a/pkg/tools/integration/web.go +++ b/pkg/tools/integration/web.go @@ -31,7 +31,7 @@ const ( userAgentHonest = "picoclaw/%s (+https://github.com/sipeed/picoclaw; AI assistant bot)" // HTTP client timeouts for web tool providers. - searchTimeout = 10 * time.Second // Brave, Tavily, DuckDuckGo + searchTimeout = 10 * time.Second // Brave, Tavily, Exa, DuckDuckGo perplexityTimeout = 30 * time.Second // Perplexity (LLM-based, slower) fetchTimeout = 60 * time.Second // WebFetchTool @@ -177,6 +177,25 @@ func mapTavilyTimeRange(rangeCode string) string { } } +// mapExaStartPublishedDate converts a range code into an Exa +// startPublishedDate filter, relative to now. +func mapExaStartPublishedDate(rangeCode string, now time.Time) string { + var start time.Time + switch rangeCode { + case "d": + start = now.AddDate(0, 0, -1) + case "w": + start = now.AddDate(0, 0, -7) + case "m": + start = now.AddDate(0, -1, 0) + case "y": + start = now.AddDate(-1, 0, 0) + default: + return "" + } + return start.UTC().Format(time.RFC3339) +} + func mapPerplexityRecencyFilter(rangeCode string) string { switch rangeCode { case "d": @@ -501,6 +520,119 @@ func (p *TavilySearchProvider) Search( return "", fmt.Errorf("all api keys failed, last error: %w", lastErr) } +type ExaSearchProvider struct { + keyPool *APIKeyPool + proxy string + client *http.Client +} + +func (p *ExaSearchProvider) Search( + ctx context.Context, + query string, + count int, + rangeCode string, +) (string, error) { + if p.keyPool == nil || len(p.keyPool.keys) == 0 { + return "", errors.New("no API key provided") + } + + searchURL := "https://api.exa.ai/search" + + var lastErr error + iter := p.keyPool.NewIterator() + + for { + apiKey, ok := iter.Next() + if !ok { + break + } + + payload := map[string]any{ + "query": query, + "type": "auto", + "numResults": count, + "contents": map[string]any{ + "highlights": true, + }, + } + if startDate := mapExaStartPublishedDate(rangeCode, time.Now()); startDate != "" { + payload["startPublishedDate"] = startDate + } + + bodyBytes, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("failed to marshal payload: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", searchURL, bytes.NewBuffer(bodyBytes)) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Api-Key", apiKey) + + resp, err := p.client.Do(req) + if err != nil { + lastErr = fmt.Errorf("request failed: %w", err) + continue + } + + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if err != nil { + lastErr = fmt.Errorf("failed to read response: %w", err) + continue + } + + if resp.StatusCode != http.StatusOK { + lastErr = fmt.Errorf("exa api error (status %d): %s", resp.StatusCode, string(body)) + if resp.StatusCode == http.StatusTooManyRequests || + resp.StatusCode == http.StatusUnauthorized || + resp.StatusCode == http.StatusForbidden || + resp.StatusCode >= 500 { + continue + } + return "", lastErr + } + + var searchResp struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Highlights []string `json:"highlights"` + } `json:"results"` + } + + if err := json.Unmarshal(body, &searchResp); err != nil { + return "", fmt.Errorf("failed to parse response: %w", err) + } + + results := searchResp.Results + if len(results) == 0 { + return fmt.Sprintf("No results for: %s", query), nil + } + + var lines []string + lines = append(lines, fmt.Sprintf("Results for: %s (via Exa)", query)) + for i, item := range results { + if i >= count { + break + } + lines = append(lines, fmt.Sprintf("%d. %s\n %s", i+1, item.Title, item.URL)) + // Highlights are multi-line page excerpts; flatten them into one line. + if snippet := strings.Join(strings.Fields(strings.Join(item.Highlights, " ... ")), " "); snippet != "" { + lines = append(lines, fmt.Sprintf(" %s", snippet)) + } + } + + return strings.Join(lines, "\n"), nil + } + + return "", fmt.Errorf("all api keys failed, last error: %w", lastErr) +} + type KagiSearchProvider struct { keyPool *APIKeyPool baseURL string @@ -1472,6 +1604,9 @@ type WebSearchToolOptions struct { TavilyBaseURL string TavilyMaxResults int TavilyEnabled bool + ExaAPIKeys []string + ExaMaxResults int + ExaEnabled bool KagiAPIKeys []string KagiBaseURL string KagiMaxResults int @@ -1512,6 +1647,9 @@ func WebSearchToolOptionsFromConfig(cfg *config.Config) WebSearchToolOptions { TavilyBaseURL: cfg.Tools.Web.Tavily.BaseURL, TavilyMaxResults: cfg.Tools.Web.Tavily.MaxResults, TavilyEnabled: cfg.Tools.Web.Tavily.Enabled, + ExaAPIKeys: cfg.Tools.Web.Exa.APIKeys.Values(), + ExaMaxResults: cfg.Tools.Web.Exa.MaxResults, + ExaEnabled: cfg.Tools.Web.Exa.Enabled, KagiAPIKeys: cfg.Tools.Web.Kagi.APIKeys.Values(), KagiBaseURL: cfg.Tools.Web.Kagi.BaseURL, KagiMaxResults: cfg.Tools.Web.Kagi.MaxResults, @@ -1558,13 +1696,16 @@ var ( "gemini", "brave", "tavily", + "exa", "kagi", "perplexity", "searxng", "glm_search", "baidu_search", } - autoPrimaryWebSearchProviders = []string{"perplexity", "brave", "kagi", "searxng", "tavily", "gemini"} + autoPrimaryWebSearchProviders = []string{ + "exa", "perplexity", "brave", "kagi", "searxng", "tavily", "gemini", + } autoFallbackWebSearchProviders = []string{"baidu_search", "glm_search"} ) @@ -1590,6 +1731,8 @@ func (opts WebSearchToolOptions) providerReady(name string) bool { return opts.BraveEnabled && len(opts.BraveAPIKeys) > 0 case "tavily": return opts.TavilyEnabled && len(opts.TavilyAPIKeys) > 0 + case "exa": + return opts.ExaEnabled && len(opts.ExaAPIKeys) > 0 case "kagi": return opts.KagiEnabled && len(opts.KagiAPIKeys) > 0 case "perplexity": @@ -1759,6 +1902,23 @@ func (opts WebSearchToolOptions) providerByName(name string) (SearchProvider, in proxy: opts.Proxy, client: client, }, maxResults, nil + case "exa": + if !opts.providerReady("exa") { + return nil, 0, nil + } + client, err := utils.CreateHTTPClient(opts.Proxy, searchTimeout) + if err != nil { + return nil, 0, fmt.Errorf("failed to create HTTP client for Exa: %w", err) + } + maxResults := 10 + if opts.ExaMaxResults > 0 { + maxResults = min(opts.ExaMaxResults, 10) + } + return &ExaSearchProvider{ + keyPool: NewAPIKeyPool(opts.ExaAPIKeys), + proxy: opts.Proxy, + client: client, + }, maxResults, nil case "kagi": if !opts.providerReady("kagi") { return nil, 0, nil diff --git a/pkg/tools/integration/web_test.go b/pkg/tools/integration/web_test.go index c69f734f8b..3be80db3ce 100644 --- a/pkg/tools/integration/web_test.go +++ b/pkg/tools/integration/web_test.go @@ -470,6 +470,13 @@ func TestSearchRangeMappings(t *testing.T) { if got := mapTavilyTimeRange("w"); got != "week" { t.Fatalf("mapTavilyTimeRange(w) = %q, want week", got) } + if got := mapExaStartPublishedDate("w", time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)); got != + "2026-03-08T00:00:00Z" { + t.Fatalf("mapExaStartPublishedDate(w) = %q, want 2026-03-08T00:00:00Z", got) + } + if got := mapExaStartPublishedDate("", time.Now()); got != "" { + t.Fatalf("mapExaStartPublishedDate() = %q, want empty", got) + } if got := mapPerplexityRecencyFilter("m"); got != "month" { t.Fatalf("mapPerplexityRecencyFilter(m) = %q, want month", got) } diff --git a/pkg/tools/integration_facade.go b/pkg/tools/integration_facade.go index dc28fe54b9..b9df701e30 100644 --- a/pkg/tools/integration_facade.go +++ b/pkg/tools/integration_facade.go @@ -26,6 +26,7 @@ type ( SearchResultItem = integrationtools.SearchResultItem BraveSearchProvider = integrationtools.BraveSearchProvider TavilySearchProvider = integrationtools.TavilySearchProvider + ExaSearchProvider = integrationtools.ExaSearchProvider SogouSearchProvider = integrationtools.SogouSearchProvider DuckDuckGoSearchProvider = integrationtools.DuckDuckGoSearchProvider GeminiSearchProvider = integrationtools.GeminiSearchProvider diff --git a/web/backend/api/tools.go b/web/backend/api/tools.go index f751925989..c4a1b659b4 100644 --- a/web/backend/api/tools.go +++ b/web/backend/api/tools.go @@ -470,6 +470,13 @@ func (h *Handler) handleUpdateWebSearchConfig(w http.ResponseWriter, r *http.Req cfg.Tools.Web.Tavily.SetAPIKeys(keys) } } + if settings, ok := req.Settings["exa"]; ok { + cfg.Tools.Web.Exa.Enabled = settings.Enabled + cfg.Tools.Web.Exa.MaxResults = settings.MaxResults + if keys, ok := normalizeWebSearchAPIKeys(settings.APIKeys, settings.APIKey); ok { + cfg.Tools.Web.Exa.SetAPIKeys(keys) + } + } if settings, ok := req.Settings["kagi"]; ok { cfg.Tools.Web.Kagi.Enabled = settings.Enabled cfg.Tools.Web.Kagi.MaxResults = settings.MaxResults @@ -525,6 +532,7 @@ func normalizeWebSearchProvider(provider string) string { case "sogou", "brave", "tavily", + "exa", "kagi", "duckduckgo", "gemini", @@ -592,6 +600,11 @@ func buildWebSearchConfigResponse(cfg *config.Config) webSearchConfigResponse { BaseURL: cfg.Tools.Web.Tavily.BaseURL, APIKeySet: len(cfg.Tools.Web.Tavily.APIKeys.Values()) > 0, }, + "exa": { + Enabled: cfg.Tools.Web.Exa.Enabled, + MaxResults: cfg.Tools.Web.Exa.MaxResults, + APIKeySet: len(cfg.Tools.Web.Exa.APIKeys.Values()) > 0, + }, "kagi": { Enabled: cfg.Tools.Web.Kagi.Enabled, MaxResults: cfg.Tools.Web.Kagi.MaxResults, @@ -663,6 +676,13 @@ func buildWebSearchConfigResponse(cfg *config.Config) webSearchConfigResponse { Current: current == "tavily", RequiresAuth: true, }, + { + ID: "exa", + Label: "Exa", + Configured: picotools.WebSearchProviderReady(opts, "exa"), + Current: current == "exa", + RequiresAuth: true, + }, { ID: "kagi", Label: "Kagi Search", diff --git a/web/frontend/src/components/agent/tools/web-search-provider-settings.tsx b/web/frontend/src/components/agent/tools/web-search-provider-settings.tsx index 23bad4ad84..4f58af525d 100644 --- a/web/frontend/src/components/agent/tools/web-search-provider-settings.tsx +++ b/web/frontend/src/components/agent/tools/web-search-provider-settings.tsx @@ -30,6 +30,7 @@ const baseUrlProviders = new Set([ const apiKeyProviders = new Set([ "brave", "tavily", + "exa", "kagi", "perplexity", "gemini",