[pull] main from firecrawl:main

apps/api/native/Cargo.toml

@@ -25,6 +25,7 @@ serde_json = "1.0.143"
 strsim = "0.11"
 texting_robots = "0.2.2"
 url = "2.5.7"
+zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
 tokio = "1.48.0"
 # Transitive dep (pdf-inspector -> unicode-normalization -> tinyvec). Pinned because
 # tinyvec 1.13.0 fails to compile (https://github.com/Lokathor/tinyvec/issues/225)

apps/api/native/src/document.rs

@@ -1,5 +1,42 @@
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
+use roxmltree::Document;
+use std::collections::{HashMap, HashSet};
+use std::io::{Cursor, Read};
+
+const MAX_OOXML_PART_BYTES: u64 = 128 * 1024 * 1024;
+const MAX_OOXML_TOTAL_BYTES: u64 = 128 * 1024 * 1024;
+const MAX_OOXML_PARTS: usize = 256;
+const MAX_HEADER_FOOTER_REFERENCES: usize = 256;
+const WORDPROCESSINGML_NAMESPACES: [&str; 2] = [
+  "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+  "http://purl.oclc.org/ooxml/wordprocessingml/main",
+];
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+enum RepeatingPartKind {
+  Header,
+  Footer,
+}
+
+struct Relationship {
+  target: String,
+  kind: RepeatingPartKind,
+}
+
+struct ZipReadBudget {
+  bytes_remaining: u64,
+  parts_remaining: usize,
+}
+
+impl Default for ZipReadBudget {
+  fn default() -> Self {
+    Self {
+      bytes_remaining: MAX_OOXML_TOTAL_BYTES,
+      parts_remaining: MAX_OOXML_PARTS,
+    }
+  }
+}
 
 /// Convert a document (doc, docx, odt/ods/odp, rtf, xls/xlsx, ppt/pptx, epub,
 /// csv, ...) to GitHub-Flavored Markdown using anydoc. The format is detected
@@ -14,6 +51,241 @@ pub fn convert_document_to_markdown(data: &[u8], extension_hint: Option<String>)
       .and_then(anydoc::Format::from_extension)
   });
 
-  anydoc::to_markdown_bytes(data, format)
-    .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))
+  let markdown = anydoc::to_markdown_bytes(data, format)
+    .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?;
+
+  if format != Some(anydoc::Format::Docx) {
+    return Ok(markdown);
+  }
+
+  let (headers, footers) = extract_docx_headers_and_footers(data);
+  if headers.is_empty() && footers.is_empty() {
+    return Ok(markdown);
+  }
+  let sections = [headers, markdown.trim().to_string(), footers]
+    .into_iter()
+    .filter(|section| !section.is_empty())
+    .collect::<Vec<_>>();
+
+  Ok(format!("{}\n", sections.join("\n\n")))
+}
+
+fn extract_docx_headers_and_footers(data: &[u8]) -> (String, String) {
+  let Ok(mut archive) = zip::ZipArchive::new(Cursor::new(data)) else {
+    return (String::new(), String::new());
+  };
+  let mut budget = ZipReadBudget::default();
+
+  let main_part = read_zip_text(&mut archive, "_rels/.rels", &mut budget)
+    .and_then(|xml| office_document_target(&xml))
+    .unwrap_or_else(|| "word/document.xml".to_string());
+  let Some(document_xml) = read_zip_text(&mut archive, &main_part, &mut budget) else {
+    return (String::new(), String::new());
+  };
+  let relationship_part = relationship_part_for(&main_part);
+  let Some(relationships_xml) = read_zip_text(&mut archive, &relationship_part, &mut budget) else {
+    return (String::new(), String::new());
+  };
+  let relationships = parse_repeating_relationships(&relationships_xml);
+  let Ok(document) = Document::parse(&document_xml) else {
+    return (String::new(), String::new());
+  };
+
+  let mut seen = HashSet::new();
+  let mut headers = Vec::new();
+  let mut footers = Vec::new();
+  let mut reference_count = 0;
+
+  for reference in document
+    .descendants()
+    .filter(|node| is_wordprocessing_node(*node))
+  {
+    let reference_kind = match reference.tag_name().name() {
+      "headerReference" => RepeatingPartKind::Header,
+      "footerReference" => RepeatingPartKind::Footer,
+      _ => continue,
+    };
+    reference_count += 1;
+    if reference_count > MAX_HEADER_FOOTER_REFERENCES {
+      break;
+    }
+    let Some(id) = reference
+      .attributes()
+      .find(|attribute| attribute.name() == "id")
+      .map(|attribute| attribute.value())
+    else {
+      continue;
+    };
+    let Some(relationship) = relationships.get(id) else {
+      continue;
+    };
+    if relationship.kind != reference_kind {
+      continue;
+    }
+    let Some(part) = resolve_part(&main_part, &relationship.target) else {
+      continue;
+    };
+    if !seen.insert((reference_kind, part.clone())) {
+      continue;
+    }
+    let Some(xml) = read_zip_text(&mut archive, &part, &mut budget) else {
+      continue;
+    };
+    let Some(text) = extract_wordprocessing_text(&xml) else {
+      continue;
+    };
+
+    match reference_kind {
+      RepeatingPartKind::Header => headers.push(text),
+      RepeatingPartKind::Footer => footers.push(text),
+    }
+  }
+
+  (headers.join("\n\n"), footers.join("\n\n"))
+}
+
+fn read_zip_text<R: Read + std::io::Seek>(
+  archive: &mut zip::ZipArchive<R>,
+  name: &str,
+  budget: &mut ZipReadBudget,
+) -> Option<String> {
+  if budget.parts_remaining == 0 || budget.bytes_remaining == 0 {
+    return None;
+  }
+  let mut file = archive.by_name(name.trim_start_matches('/')).ok()?;
+  if file.size() > MAX_OOXML_PART_BYTES || file.size() > budget.bytes_remaining {
+    return None;
+  }
+  budget.parts_remaining -= 1;
+  let mut bytes = Vec::with_capacity(file.size() as usize);
+  let read_limit = MAX_OOXML_PART_BYTES.min(budget.bytes_remaining);
+  let read_result = file.by_ref().take(read_limit + 1).read_to_end(&mut bytes);
+  budget.bytes_remaining = budget.bytes_remaining.saturating_sub(bytes.len() as u64);
+  if read_result.is_err() {
+    return None;
+  }
+  if bytes.len() as u64 > read_limit {
+    budget.bytes_remaining = 0;
+    return None;
+  }
+  String::from_utf8(bytes).ok()
+}
+
+fn office_document_target(xml: &str) -> Option<String> {
+  let document = Document::parse(xml).ok()?;
+  let relationship = document.descendants().find(|node| {
+    node.is_element()
+      && node.tag_name().name() == "Relationship"
+      && node
+        .attribute("Type")
+        .is_some_and(|value| value.ends_with("/officeDocument"))
+  })?;
+  normalize_part(relationship.attribute("Target")?)
+}
+
+fn parse_repeating_relationships(xml: &str) -> HashMap<String, Relationship> {
+  let Ok(document) = Document::parse(xml) else {
+    return HashMap::new();
+  };
+
+  document
+    .descendants()
+    .filter_map(|node| {
+      if !node.is_element()
+        || node.tag_name().name() != "Relationship"
+        || node
+          .attribute("TargetMode")
+          .is_some_and(|value| value.eq_ignore_ascii_case("external"))
+      {
+        return None;
+      }
+      let kind = match node.attribute("Type")? {
+        value if value.ends_with("/header") => RepeatingPartKind::Header,
+        value if value.ends_with("/footer") => RepeatingPartKind::Footer,
+        _ => return None,
+      };
+      Some((
+        node.attribute("Id")?.to_string(),
+        Relationship {
+          target: node.attribute("Target")?.to_string(),
+          kind,
+        },
+      ))
+    })
+    .collect()
+}
+
+fn relationship_part_for(part: &str) -> String {
+  match part.rsplit_once('/') {
+    Some((parent, file_name)) => format!("{parent}/_rels/{file_name}.rels"),
+    None => format!("_rels/{part}.rels"),
+  }
+}
+
+fn resolve_part(base: &str, target: &str) -> Option<String> {
+  if target.starts_with('/') {
+    return normalize_part(target);
+  }
+  let parent = base
+    .rsplit_once('/')
+    .map(|(parent, _)| parent)
+    .unwrap_or("");
+  normalize_part(&format!("{parent}/{target}"))
+}
+
+fn normalize_part(part: &str) -> Option<String> {
+  let mut normalized = Vec::new();
+  for component in part.trim_start_matches('/').split('/') {
+    match component {
+      "" | "." => {}
+      ".." => normalized.pop().map(|_| ())?,
+      value => normalized.push(value),
+    }
+  }
+  (!normalized.is_empty()).then(|| normalized.join("/"))
+}
+
+fn extract_wordprocessing_text(xml: &str) -> Option<String> {
+  let document = Document::parse(xml).ok()?;
+  let paragraphs = document
+    .descendants()
+    .filter(|node| {
+      is_wordprocessing_node(*node)
+        && node.tag_name().name() == "p"
+        && !node
+          .ancestors()
+          .skip(1)
+          .any(|ancestor| is_wordprocessing_node(ancestor) && ancestor.tag_name().name() == "p")
+    })
+    .filter_map(|paragraph| {
+      let mut text = String::new();
+      for node in paragraph
+        .descendants()
+        .filter(|node| is_wordprocessing_node(*node))
+      {
+        match node.tag_name().name() {
+          "t" => {
+            if let Some(value) = node.text() {
+              text.push_str(value);
+            }
+          }
+          "tab" => text.push('\t'),
+          "br" | "cr" => text.push('\n'),
+          _ => {}
+        }
+      }
+      let text = text.trim();
+      (!text.is_empty()).then(|| text.to_string())
+    })
+    .collect::<Vec<_>>();
+
+  (!paragraphs.is_empty()).then(|| paragraphs.join("\n\n"))
+}
+
+fn is_wordprocessing_node(node: roxmltree::Node<'_, '_>) -> bool {
+  node.is_element()
+    && node
+      .tag_name()
+      .namespace()
+      .is_some_and(|namespace| WORDPROCESSINGML_NAMESPACES.contains(&namespace))
 }

apps/api/src/__tests__/snips/v2/parse.test.ts

@@ -21,7 +21,7 @@ import { config } from "../../../config";
 import { getRedisConnection } from "../../../services/queue-service";
 
 const DOCX_FIXTURE_BASE64 =
-  "UEsDBBQAAAAIAKtlbVzXeYTq8QAAALgBAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbH2QzU7DMBCE730Ky9cqccoBIZSkB36OwKE8wMreJFb9J69b2rdn00KREOVozXwz62nXB+/EHjPZGDq5qhspMOhobBg7+b55ru6koALBgIsBO3lEkut+0W6OCUkwHKiTUynpXinSE3qgOiYMrAwxeyj8zKNKoLcworppmlulYygYSlXmDNkvhGgfcYCdK+LpwMr5loyOpHg4e+e6TkJKzmoorKt9ML+Kqq+SmsmThyabaMkGqa6VzOL1jh/0lSfK1qB4g1xewLNRfcRslIl65xmu/0/649o4DFbjhZ/TUo4aiXh77+qL4sGG71+06jR8/wlQSwMEFAAAAAgAq2VtXCAbhuqyAAAALgEAAAsAAABfcmVscy8ucmVsc43Puw6CMBQG4J2naM4uBQdjDIXFmLAafICmPZRGeklbL7y9HRzEODie23fyN93TzOSOIWpnGdRlBQStcFJbxeAynDZ7IDFxK/nsLDJYMELXFs0ZZ57yTZy0jyQjNjKYUvIHSqOY0PBYOo82T0YXDE+5DIp6Lq5cId1W1Y6GTwPagpAVS3rJIPSyBjIsHv/h3ThqgUcnbgZt+vHlayPLPChMDB4uSCrf7TKzQHNKuorZvgBQSwMEFAAAAAgAq2VtXCCNfXOwAAAA7AAAABEAAAB3b3JkL2RvY3VtZW50LnhtbDWOMQvCMBCFd3/FkV1THURKGwfFVQcF19icWmjuQi5a/fcmBZeP93jw3TXbjx/gjVF6plYtF5UCpI5dT49WXc6H+UaBJEvODkzYqi+K2ppZM9aOu5dHSpANJPXYqmdKodZauid6KwsOSHm7c/Q25RofeuToQuQORfIBP+hVVa21tz0pMwPI1hu7b4lTCSYjFiRzslEQ9sfdFS5hYOvgjJIaXbbCODFMGv33lPT/0/wAUEsBAhQDFAAAAAgAq2VtXNd5hOrxAAAAuAEAABMAAAAAAAAAAAAAAIABAAAAAFtDb250ZW50X1R5cGVzXS54bWxQSwECFAMUAAAACACrZW1cIBuG6rIAAAAuAQAACwAAAAAAAAAAAAAAgAEiAQAAX3JlbHMvLnJlbHNQSwECFAMUAAAACACrZW1cII19c7AAAADsAAAAEQAAAAAAAAAAAAAAgAH9AQAAd29yZC9kb2N1bWVudC54bWxQSwUGAAAAAAMAAwC5AAAA3AIAAAAA";
+  "UEsDBBQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbLWSu07DMBSGXyXyWtUnZWBASToAKzDwAkf2cWLhm+zTUt4epykdUKELjPZ/+X5L7rYH75o95WJj6MVGtmI7dK8fiUpTlVB6MTGnO4CiJvJYZEwUqmJi9sj1mEdIqN5wJLhp21tQMTAFXvPcIYbugQzuHDePh3q9UDK5Ipr7xTizeoEpOauQqw77oL9R1ieCrMmjp0w2lVU1CLhImJWfAafcc312tpqaF8z8hL664D1mDTqqna9J+XvNhZ3RGKvonJ/bUo6KSrFh9E6eFY82rK7tmAg15c3fz1iKr/JNjPwv/KX4iw/H7zZ8AlBLAwQUAAAACABiQjVdXzOVUpUAAAAHAQAACwAAAF9yZWxzLy5yZWxzjc87DsIwDAbgq0Q+QJ0yMKCmXVi6Ii4QJW5T0TzkhNftycBAEQOjf//6LHfDw6/iRpyXGBS0jYSh70606lKD7JaURW2ErMCVkg6I2TjyOjcxUaibKbLXpY48Y9LmomfCnZR75E8DtqYYrQIebQvi/Ez0jx2naTF0jObqKZQfJ74aVdY8U1Fwj2zRvuOmsoB9h5sX+xdQSwMEFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAB3b3JkL2RvY3VtZW50LnhtbI2QQU8DIRCF/wrhB5TVg4fN7jbRptGTjdHEK8LQJQGGzFDX/nthrem1l0fIfO/Ng2H7E4P4BmKPaZR3m05up2HpLZpThFREHSful1HOpeReKTYzRM0bzJDqzCFFXeqVjmpBspnQALNPxxjUfdc9qKh9kpcYuiUGnfMGdpcCfyEEQZfakGefWbaCX2jP7cxNqEmZDpoYxO716VN85IDainfgIh4rOqgGNKVVVxuDKYfVO4O2QG/ggCAZEBU+ZxilBadPoUhBvbejpBf7vJJSNZdDLLe59ivZXOq6Vv2/Ql3/e/oFUEsDBBQAAAAIAGJCNV2/NrkppgAAAIMBAAAcAAAAd29yZC9fcmVscy9kb2N1bWVudC54bWwucmVsc62QywrCMBBFfyXkAzqtCxfSx0ak3Yo/ENJJU2weJKPo3xvUioUuXLi8M8O5hymbm5nYFUMcna14keW8qcsjToLSIOrRR5YubKy4JvI7gCg1GhEz59GmjXLBCEoxDOCFPIsBYZPnWwjfDL5ksq6veOj6FkWPgbPT3eMvBU6pUeLeyYtBSys9oGeiCANSYj5zkSUQh3WJg3P0Xwk1E98Sr/yRgMV76wdQSwMEFAAAAAgAYkI1XRIFb3mKAAAAqQAAABAAAAB3b3JkL2hlYWRlcjEueG1sHc5NCsMgEAXgq4gHiLaLLoIxi3aRXbtooVuJ0yj4x4w0PX41mw8ebxiemn8xsC8g+Zwmfhokn7XaR2eRtSbRuE/c1VpGIWh1EA0NuUBq3SdjNLVF3MSe0RbMKxD5tMUgzlJeRDQ+8f6tdLBT9cMgAbvdr2/2KiEby55AlS1gLKAS/aSLh+WwjdF/UEsDBBQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAd29yZC9mb290ZXIxLnhtbB3OPQrDMAwF4KsYHyB2O3QITjK0dG2HFrqaRPmB2DKSqHv82lk+eDwhnht+YVdfIN4wdvrUWD30LrezkCpN5DZ3ehVJrTE8rhA8N5gglm5GCl5KpMVkpCkRjsC8xSXs5mztxQS/RV2/pQpVpH96YlC3x/Wj3mlHP6kXsKg7ogA5U0+qdJgOy5j+D1BLAQIUAxQAAAAIAGJCNV0B3PB/6gAAAIsCAAATAAAAAAAAAAAAAACAAQAAAABbQ29udGVudF9UeXBlc10ueG1sUEsBAhQDFAAAAAgAYkI1XV8zlVKVAAAABwEAAAsAAAAAAAAAAAAAAIABGwEAAF9yZWxzLy5yZWxzUEsBAhQDFAAAAAgAYkI1Xf2I2NHjAAAAkQEAABEAAAAAAAAAAAAAAIAB2QEAAHdvcmQvZG9jdW1lbnQueG1sUEsBAhQDFAAAAAgAYkI1Xb82uSmmAAAAgwEAABwAAAAAAAAAAAAAAIAB6wIAAHdvcmQvX3JlbHMvZG9jdW1lbnQueG1sLnJlbHNQSwECFAMUAAAACABiQjVdEgVveYoAAACpAAAAEAAAAAAAAAAAAAAAgAHLAwAAd29yZC9oZWFkZXIxLnhtbFBLAQIUAxQAAAAIAGJCNV0TuxHRigAAAKkAAAAQAAAAAAAAAAAAAACAAYMEAAB3b3JkL2Zvb3RlcjEueG1sUEsFBgAAAAAGAAYAfwEAADsFAAAAAA==";
 
 const htmlFixture = `
 <!DOCTYPE html>
@@ -572,7 +572,12 @@ describe("/v2/parse", () => {
         identity,
       );
 
-      expect(result.markdown).toMatch(/Parse DOCX Upload Test/i);
+      const markdown = result.markdown!;
+      expect(markdown).toContain("Parse DOCX Upload Test Header");
+      expect(markdown).toContain("Parse DOCX Upload Test Body");
+      expect(markdown).toContain("Parse DOCX Upload Test Footer");
+      expect(markdown.indexOf("Header")).toBeLessThan(markdown.indexOf("Body"));
+      expect(markdown.indexOf("Body")).toBeLessThan(markdown.indexOf("Footer"));
       expect(result.metadata.creditsUsed).toBe(1);
     },
     scrapeTimeout,

apps/api/src/controllers/v2/agent.ts

@@ -1,4 +1,5 @@
 import { v7 as uuidv7 } from "uuid";
+import { AGENT_REQUEST_CREDITS_SHARDS } from "../../lib/request-credits-store";
 import { Response } from "express";
 import {
   AgentRequest,
@@ -205,7 +206,7 @@ export async function agentController(
     target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",
     zeroDataRetention: false, // not supported for agent
     api_key_id: req.acuc?.api_key_id ?? null,
-    creditsShards: 8,
+    creditsShards: AGENT_REQUEST_CREDITS_SHARDS,
   });
 
   const passthrough = await fetch(

apps/api/src/controllers/v2/batch-scrape.ts

@@ -50,7 +50,11 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
 import { billTeam } from "../../services/billing/credit_billing";
 import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
 import { UnsupportedSiteError } from "../../lib/error";
-import { requestCreditsShards } from "../../lib/request-credits-store";
+import {
+  AGENT_REQUEST_CREDITS_SHARDS,
+  initializeRequestCredits,
+  requestCreditsShards,
+} from "../../lib/request-credits-store";
 
 export async function batchScrapeController(
   req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -351,6 +355,35 @@ export async function batchScrapeController(
       ),
       creditsShards: requestCreditsShards(urls.length),
     });
+  } else if (!req.body.appendToId) {
+    // An agent-started batch is recorded under the agent's request rather
+    // than as a request of its own, so logRequest never created a credit
+    // row for it. Its status is still polled: v2 reads credits under the
+    // saved agent request id, v1 under the batch id, and its children record
+    // credits under the agent request id. Make sure both rows exist. The
+    // agent row is created once with the agent controller's shard count;
+    // for an agent that came through that controller this is a no-op, for
+    // one that reaches the API first through interop it is the only writer.
+    const agentRequestId = req.body.__agentInterop?.requestId;
+    if (agentRequestId) {
+      await initializeRequestCredits(
+        agentRequestId,
+        AGENT_REQUEST_CREDITS_SHARDS,
+      ).catch(error => {
+        logger.warn("Failed to initialize Bigtable request credits", {
+          error,
+          requestId: agentRequestId,
+          shards: AGENT_REQUEST_CREDITS_SHARDS,
+        });
+      });
+    }
+    const creditsShards = requestCreditsShards(urls.length);
+    await initializeRequestCredits(id, creditsShards).catch(error => {
+      logger.warn("Failed to initialize Bigtable request credits", {
+        error,
+        shards: creditsShards,
+      });
+    });
   }
 
   const sc: StoredCrawl = req.body.appendToId

apps/api/src/controllers/v2/scrape-alexandria.ts

@@ -30,8 +30,8 @@ const providerScrapeSchema = z.strictObject({
     .number()
     .int()
     .positive()
-    .default(50000)
-    .transform(value => Math.min(value, 50000)),
+    .default(120_000)
+    .transform(value => Math.min(value, 120_000)),
   origin: z.string().default("api"),
   integration: z.string().nullable().optional(),
   __agentInterop: z