[pull] main from firecrawl:main

apps/api/AGENT_HINTS.md

@@ -0,0 +1,35 @@
+# Response guidance for agents
+
+The v2 `POST /search`, `/scrape`, `/parse`, and `/map` routes may return an optional top-level `agent_hints: string[]`. Each response contains at most one result-dependent cross-endpoint suggestion and one low-credit notice. Existing data, errors, warning fields, and HTTP statuses remain unchanged.
+
+Hints are disabled by default. Trusted agent adapters such as the Firecrawl MCP server and CLI can enable them for one request with:
+
+```http
+X-Firecrawl-Agent-Hints: true
+```
+
+The header applies to the business request. No request-body schema changes are required. SDKs and other adapters should retain the top-level field when unwrapping `data` and should preserve it on error results. The strings recommend conditional next steps; receiving one does not execute another request or indicate user authorization.
+
+## SDK and adapter delivery
+
+The accompanying JavaScript and Python SDK changes preserve `agent_hints` alongside normal flattened Document, SearchData, and MapData results and on typed errors. Python covers synchronous and asynchronous clients. Markdown and HTML retain their existing content.
+
+Raw HTTP and ordinary SDK callers receive no hints unless their adapter explicitly sets the header above. This change does not add a client-wide SDK hint option. The SDK release must precede upgrading pinned CLI/MCP dependencies for their keyed SDK paths to retain the field.
+
+## Initial rules
+
+- Search excerpts: offer Scrape when a web result has no markdown, HTML, or raw HTML. Inspect actual output per result, not the requested scrape setting.
+- Empty web search: offer another Search with a broader or alternative query only when a web result collection is explicitly present and empty.
+- Search result cluster: for authenticated callers only, when at least four valid HTTP(S) result URLs are present and at least three share one origin representing at least 75% of valid result URLs, offer Map for URL discovery or Crawl for multi-page content. Keyless Search callers cannot use Map or Crawl, so they receive no cluster suggestion.
+- Source page 401: when database authentication is enabled and a successful Scrape envelope includes both a page status of 401 and a scrape ID in `metadata.scrapeId` or top-level `scrape_id`, offer Interact using that concrete scrape ID for authentication or page interaction. Self-hosted deployments without database authentication do not receive this suggestion because Interact returns 501 there.
+- Source page 404/410: offer Search for a current location or alternative. An API cache-miss 404, 429, authentication failure, or timeout does not fire this rule.
+- Truncated PDF: when Scrape reports `totalPages > numPages`, offer another Scrape only when its PDF `maxPages` can be raised, capped at the API maximum of 10,000.
+- Low credits: when the authoritative billing preflight reports fewer than 100 credits, ask the agent to let the user know they should add more credits. This notice is independent of endpoint success for responses that reach the hint middleware, appears before any cross-endpoint suggestion, and does not replace that suggestion.
+
+Selection reads only request/response fields and billing state already in memory. It makes no additional network/model calls, reads no additional session/database state, and does not scan page text or classify user intent. No hint appears solely to fill an available slot.
+
+Static feedback instructions do not belong in response hints. Adapters that expose a feedback tool should document its contract in the relevant tool descriptions, where the guidance is available before invocation and can reference the adapter's actual feedback tool and identifiers.
+
+## Validation
+
+The focused selector tests exercise excerpt detection, source-page failures, error suppression, the low-credit threshold, and combined hints. Express route fixtures exercise response preservation, opt-in, opt-out, low-credit delivery, and responses without a useful next step. Hosted snips cover a completed scrape, map opt-out, and validation failure through the actual API. The snips use the harness and existing test service; they are not a live paid API smoke test.

apps/api/src/__tests__/routes/check-credits-middleware.test.ts

@@ -65,6 +65,7 @@ function runMiddleware(
     };
 
     const res: any = {
+      locals: {},
       status: vi.fn((..._args: any[]) => {
         // 402 / 403 paths terminate via res.status(...).json(...) without next()
         setImmediate(() => settle({ res }));
@@ -95,6 +96,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
 
     expect(res.status).not.toHaveBeenCalled();
     expect(req.account.remainingCredits).toBe(Infinity);
+    expect(res.locals.agentCreditsRemaining).toBe(0);
     // request body limit must NOT have been clamped down to 0
     expect(req.body.limit).toBe(100);
   });
@@ -106,6 +108,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
     const { res } = await runMiddleware(req);
 
     expect(res.status).toHaveBeenCalledWith(402);
+    expect(res.locals.agentCreditsRemaining).toBe(0);
   });
 
   it("adjusts crawl limit down when Autumn denies but some credits remain", async () => {
@@ -117,6 +120,7 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
     const { res } = await runMiddleware(req);
 
     expect(res.status).not.toHaveBeenCalled();
+    expect(res.locals.agentCreditsRemaining).toBe(5);
     expect(req.body.limit).toBe(5);
     expect(checkCreditsMock).toHaveBeenLastCalledWith(
       expect.objectContaining({

apps/api/src/__tests__/snips/v2/agent-hints.test.ts

@@ -0,0 +1,89 @@
+import { describeIf, TEST_PRODUCTION, TEST_SUITE_WEBSITE } from "../lib";
+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";
+
+let identity: Identity;
+beforeAll(async () => {
+  identity = await idmux({
+    name: "agent-hints",
+    concurrency: 10,
+    credits: 1000,
+  });
+}, 20000);
+
+describeIf(TEST_PRODUCTION)("Agent hints", () => {
+  const missingPage = `${TEST_SUITE_WEBSITE}/agent-hints-not-found`;
+
+  it(
+    "adds a scrape-to-search hint for a missing source page",
+    async () => {
+      const response = await request(TEST_API_URL)
+        .post("/v2/scrape")
+        .set("Authorization", `Bearer ${identity.apiKey}`)
+        .set("X-Firecrawl-Agent-Hints", "true")
+        .send({ url: missingPage, timeout: scrapeTimeout });
+      expect(response.statusCode).toBe(200);
+      expect(response.body.success).toBe(true);
+      expect(response.body.data.metadata.statusCode).toBe(404);
+      expect(response.body.agent_hints).toEqual([
+        expect.stringContaining("POST /v2/search"),
+      ]);
+    },
+    scrapeTimeout + 10000,
+  );
+
+  it(
+    "keeps hints off when the opt-in header is false despite a hint condition",
+    async () => {
+      const response = await request(TEST_API_URL)
+        .post("/v2/scrape")
+        .set("Authorization", `Bearer ${identity.apiKey}`)
+        .set("X-Firecrawl-Agent-Hints", "false")
+        .send({ url: missingPage, timeout: scrapeTimeout });
+      expect(response.statusCode).toBe(200);
+      expect(response.body.success).toBe(true);
+      expect(response.body.data.metadata.statusCode).toBe(404);
+      expect(response.body).not.toHaveProperty("agent_hints");
+    },
+    scrapeTimeout + 10000,
+  );
+
+  it(
+    "does not add static feedback guidance to a completed scrape",
+    async () => {
+      const response = await request(TEST_API_URL)
+        .post("/v2/scrape")
+        .set("Authorization", `Bearer ${identity.apiKey}`)
+        .set("X-Firecrawl-Agent-Hints", "true")
+        .send({ url: TEST_SUITE_WEBSITE, timeout: scrapeTimeout });
+      expect(response.statusCode).toBe(200);
+      expect(response.body.success).toBe(true);
+      expect(response.body).not.toHaveProperty("agent_hints");
+    },
+    scrapeTimeout + 10000,
+  );
+
+  it(
+    "leaves hints off by default on a completed map",
+    async () => {
+      const response = await request(TEST_API_URL)
+        .post("/v2/map")
+        .set("Authorization", `Bearer ${identity.apiKey}`)
+        .send({ url: TEST_SUITE_WEBSITE, limit: 1, timeout: scrapeTimeout });
+      expect(response.statusCode).toBe(200);
+      expect(response.body).not.toHaveProperty("agent_hints");
+    },
+    scrapeTimeout + 10000,
+  );
+
+  it("preserves validation errors without inventing hints", async () => {
+    const response = await request(TEST_API_URL)
+      .post("/v2/scrape")
+      .set("Authorization", `Bearer ${identity.apiKey}`)
+      .set("X-Firecrawl-Agent-Hints", "true")
+      .send({ url: "not-a-url" });
+    expect(response.statusCode).toBe(400);
+    expect(response.body.success).toBe(false);
+    expect(typeof response.body.error).toBe("string");
+    expect(response.body).not.toHaveProperty("agent_hints");
+  });
+});

apps/api/src/controllers/v2/types.ts

@@ -1541,6 +1541,7 @@ export type VideoItem = {
 };
 
 export type ErrorResponse = {
+  agent_hints?: string[];
   success: false;
   code?: ErrorCodes;
   error: string;
@@ -1553,6 +1554,7 @@ export type ScrapeResponse =
   | ErrorResponse
   | {
       success: true;
+      agent_hints?: string[];
       warning?: string;
       data: Document & {
         tools?: import("../../services/alexandria/contracts").DiscoveredTool[];
@@ -1821,6 +1823,7 @@ export type MapResponse =
   | ErrorResponse
   | {
       success: true;
+      agent_hints?: string[];
       id: string;
       links?: MapDocument[];
       warning?: string;
@@ -2561,20 +2564,23 @@ export type SearchResponse =
   | ErrorResponse
   | {
       success: true;
+      agent_hints?: string[];
       warning?: string;
       data: Document[];
       creditsUsed: number;
       id: string;
     }
   | {
       success: true;
+      agent_hints?: string[];
       warning?: string;
       data: import("../../lib/entities").SearchV2Response;
       creditsUsed: number;
       id: string;
     }
   | {
       success: true;
+      agent_hints?: string[];
       warning?: string;
       data: import("../../lib/entities").SearchV2Response;
       scrapeIds: {