From e773217fcef3e0e9687cf7c9acf26ea8a142801f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 03:57:14 +0800 Subject: [PATCH 001/150] feat(office): add canonical XLSX workbook selectors --- docs/office-selectors.md | 15 +++++++++------ office/pkg.generated.mbti | 6 ++++++ office/selector_adapters.mbt | 20 ++++++++++++++++++++ office/selector_adapters_test.mbt | 5 +++++ office/selector_parse.mbt | 10 +++++++++- office/selector_test.mbt | 19 ++++++++++++++----- office/selector_types.mbt | 12 ++++++++++++ 7 files changed, 75 insertions(+), 12 deletions(-) diff --git a/docs/office-selectors.md b/docs/office-selectors.md index 8f944def..80fcf685 100644 --- a/docs/office-selectors.md +++ b/docs/office-selectors.md @@ -38,10 +38,11 @@ child kinds without parsing the path again. DOCX named selectors use the `id` key; arbitrary XPath predicates, regexes, and expressions are not part of this grammar. -XLSX selectors address a sheet by stable name or snapshot-relative position, -then optionally end in a typed A1 coordinate: +XLSX selectors address the workbook singleton or a sheet by stable name or +snapshot-relative position. Sheet selectors may end in a typed A1 coordinate: ```text +/xlsx/workbook /xlsx/sheet[name="Data"]/cell[A1] /xlsx/sheet[name="O'Brien / Q1"]/range[A1:C12] /xlsx/sheet[2] @@ -56,7 +57,7 @@ cross-sheet formula syntax are intentionally outside selector syntax. Every parsed selector reports one of two address classes: -- `Stable`: it contains only format roots and named keys. For example, +- `Stable`: it contains only singleton roots and named keys. For example, `/docx/comments/comment[id="7"]` and `/xlsx/sheet[name="Data"]`. - `SnapshotRelative`: it contains any positional selector or A1 coordinate. Inserts, deletes, reordering, or sheet edits can move the addressed content. @@ -85,9 +86,11 @@ surrogates fail before an AST is created or UTF-8 encoding is attempted. `selector_from_docx_projection_path` converts the existing `/body/...`, `/header[n]/...`, and annotation paths emitted by the DOCX tools. The -`selector_for_xlsx_cell` and `selector_for_xlsx_range` helpers quote worksheet -names and validate A1 coordinates. These helpers adapt syntax only: they do not -open a package or establish that an addressed object exists. +`selector_for_xlsx_workbook`, `selector_for_xlsx_sheet`, +`selector_for_xlsx_cell`, and `selector_for_xlsx_range` construct the canonical +XLSX shapes, quote worksheet names, and validate A1 coordinates. These helpers +adapt syntax only: they do not open a package or establish that an addressed +object exists. Format records in `office help --json` expose the selector schema, root, examples, and the resolver status for that format. DOCX is `read-resolved`: diff --git a/office/pkg.generated.mbti b/office/pkg.generated.mbti index 07731dee..30421d6f 100644 --- a/office/pkg.generated.mbti +++ b/office/pkg.generated.mbti @@ -71,6 +71,10 @@ pub fn selector_for_xlsx_cell(String, String) -> OfficeSelector raise SelectorEr pub fn selector_for_xlsx_range(String, String) -> OfficeSelector raise SelectorError +pub fn selector_for_xlsx_sheet(String) -> OfficeSelector raise SelectorError + +pub fn selector_for_xlsx_workbook() -> OfficeSelector raise SelectorError + pub fn selector_from_docx_projection_path(String) -> OfficeSelector raise SelectorError // Errors @@ -157,7 +161,9 @@ pub struct CellAddress { column : Int row : Int } +pub fn CellAddress::column(Self) -> Int pub fn CellAddress::render(Self) -> String +pub fn CellAddress::row(Self) -> Int pub(all) enum DocumentFormat { Xlsx diff --git a/office/selector_adapters.mbt b/office/selector_adapters.mbt index 442928a6..9c08fd59 100644 --- a/office/selector_adapters.mbt +++ b/office/selector_adapters.mbt @@ -121,6 +121,26 @@ fn selector_for_xlsx_coordinate( ) } +///| +/// Builds the canonical selector for the workbook singleton. +pub fn selector_for_xlsx_workbook() -> OfficeSelector raise SelectorError { + parse_selector("/xlsx/workbook") +} + +///| +/// Builds a stable canonical selector for one named worksheet or chart sheet. +/// It validates and quotes the name but does not look it up in a workbook. +pub fn selector_for_xlsx_sheet( + sheet_name : String, +) -> OfficeSelector raise SelectorError { + check_xlsx_adapter_argument( + sheet_name, + "worksheet name", + SELECTOR_MAX_VALUE_LENGTH, + ) + parse_selector("/xlsx/sheet[name=\{Json::string(sheet_name).stringify()}]") +} + ///| /// Builds a canonical, syntax-validated selector for one cell on a named /// worksheet. It does not look up the sheet or cell in a workbook. diff --git a/office/selector_adapters_test.mbt b/office/selector_adapters_test.mbt index 90d9a570..94123772 100644 --- a/office/selector_adapters_test.mbt +++ b/office/selector_adapters_test.mbt @@ -56,6 +56,10 @@ test "DOCX projection adapter bounds legacy diagnostics" { ///| test "XLSX adapters quote sheet names and normalize coordinates" { + let workbook = @office.selector_for_xlsx_workbook() + assert_eq(workbook.render(), "/xlsx/workbook") + let sheet = @office.selector_for_xlsx_sheet("财务 / \"Q1\"") + assert_eq(sheet.render(), "/xlsx/sheet[name=\"财务 / \\\"Q1\\\"\"]") let cell = @office.selector_for_xlsx_cell("财务 / \"Q1\"", "xfd1048576") assert_eq( cell.render(), @@ -71,6 +75,7 @@ test "XLSX adapters reject oversized raw arguments before rendering" { let oversized_reference = "A".repeat(22) for run in [ + () => @office.selector_for_xlsx_sheet(oversized_name), () => @office.selector_for_xlsx_cell(oversized_name, "A1"), () => @office.selector_for_xlsx_range("Data", oversized_reference), ] { diff --git a/office/selector_parse.mbt b/office/selector_parse.mbt index 5845c8bb..5a78838b 100644 --- a/office/selector_parse.mbt +++ b/office/selector_parse.mbt @@ -558,9 +558,17 @@ fn validate_xlsx_selector( segments : Array[SelectorSegment], coordinate : SelectorCoordinate?, ) -> SelectorStability raise SelectorError { + if segments.length() == 1 && segments[0].name == "workbook" { + if segments[0].selection is Some(_) || coordinate is Some(_) { + raise parser.fail( + "office.selector.invalid_shape", "XLSX workbook is a singleton and cannot take a selection or coordinate leaf", + ) + } + return Stable + } if segments.length() != 1 || segments[0].name != "sheet" { raise parser.fail( - "office.selector.invalid_shape", "XLSX selectors require exactly one sheet segment before an optional cell/range leaf", + "office.selector.invalid_shape", "XLSX selectors require workbook or exactly one sheet segment before an optional cell/range leaf", ) } match segments[0].selection { diff --git a/office/selector_test.mbt b/office/selector_test.mbt index 160dab23..ef1a60a5 100644 --- a/office/selector_test.mbt +++ b/office/selector_test.mbt @@ -21,8 +21,9 @@ test "canonical selectors parse, normalize, render, and classify" { let inputs = [ "/docx/body", "/docx/body/p[1]/r[2]", "/docx/header[1]/p[1]", "/docx/footer[1]/tbl[1]/tr[2]/tc[1]/p[1]", "/docx/footnotes/note[id=\"7\"]/p[1]", "/docx/comments/comment[id=\"review/one]=\\\"ready\\\"\"]", - "/xlsx/sheet[name=\"Data\"]", "/xlsx/sheet[name=\"Data\"]/cell[A1]", "/xlsx/sheet[name=\"O'Brien / Q1\"]/range[c12:a1]", - "/xlsx/sheet[name=\"财务/一季度\"]/cell[xfd1048576]", "/xlsx/sheet[2]", + "/xlsx/workbook", "/xlsx/sheet[name=\"Data\"]", "/xlsx/sheet[name=\"Data\"]/cell[A1]", + "/xlsx/sheet[name=\"O'Brien / Q1\"]/range[c12:a1]", "/xlsx/sheet[name=\"财务/一季度\"]/cell[xfd1048576]", + "/xlsx/sheet[2]", ] let rendered : Array[String] = [] for input in inputs { @@ -44,6 +45,7 @@ test "canonical selectors parse, normalize, render, and classify" { #|/docx/footer[1]/tbl[1]/tr[2]/tc[1]/p[1] | snapshot-relative #|/docx/footnotes/note[id="7"]/p[1] | snapshot-relative #|/docx/comments/comment[id="review/one]=\"ready\""] | stable + #|/xlsx/workbook | stable #|/xlsx/sheet[name="Data"] | stable #|/xlsx/sheet[name="Data"]/cell[A1] | snapshot-relative #|/xlsx/sheet[name="O'Brien / Q1"]/range[A1:C12] | snapshot-relative @@ -72,6 +74,10 @@ test "selector AST exposes adapter-ready segments, keys, and coordinates" { } assert_eq(start.render(), "A1") assert_eq(finish.render(), "C12") + assert_eq(start.column(), 1) + assert_eq(start.row(), 1) + assert_eq(finish.column(), 3) + assert_eq(finish.row(), 12) } ///| @@ -80,8 +86,9 @@ test "invalid selector forms have deterministic structured errors" { "", "docx/body", "/ppt/body", "/docx", "/docx/", "/docx//p[1]", "/docx/body/p[0]", "/docx/body/p[1234567890]", "/docx/body/p", "/docx/header/p[1]", "/docx/comments/comment[@id=7]", "/docx/comments/comment[class=\"x\"]", "/docx/body/p[1", "/docx/comments/comment[id=\"unterminated]", - "/docx/sheet[name=\"Data\"]/cell[A1]", "/xlsx/body/p[1]", "/xlsx/sheet", "/xlsx/sheet[id=\"Data\"]", - "/xlsx/sheet[name=\"\"]/cell[A1]", "/xlsx/sheet[name=\"Data\"]/cell[A0]", "/xlsx/sheet[name=\"Data\"]/cell[A01]", + "/docx/sheet[name=\"Data\"]/cell[A1]", "/xlsx/body/p[1]", "/xlsx/workbook[1]", + "/xlsx/workbook/cell[A1]", "/xlsx/sheet", "/xlsx/sheet[id=\"Data\"]", "/xlsx/sheet[name=\"\"]/cell[A1]", + "/xlsx/sheet[name=\"Data\"]/cell[A0]", "/xlsx/sheet[name=\"Data\"]/cell[A01]", "/xlsx/sheet[name=\"Data\"]/cell[XFE1]", "/xlsx/sheet[name=\"Data\"]/cell[A1048577]", "/xlsx/sheet[name=\"Data\"]/cell[$A$1]", "/xlsx/sheet[name=\"Data\"]/range[A1]", "/xlsx/sheet[name=\"Data\"]/range[A1:B2:C3]", "/xlsx/sheet[name=\"Data\"]/cell[A1]/range[A1:B2]", @@ -106,7 +113,9 @@ test "invalid selector forms have deterministic structured errors" { #|office.selector.unclosed_selection@14:/docx/body/p[1:expected ']' after segment selection #|office.selector.unclosed_string@26:/docx/comments/comment[id="unterminated]:unterminated JSON string in named selector #|office.selector.unsupported_predicate@30:/docx/sheet[name="Data"]/cell[A1]:expected a positive index or key="value" selection - #|office.selector.invalid_shape@15:/xlsx/body/p[1]:XLSX selectors require exactly one sheet segment before an optional cell/range leaf + #|office.selector.invalid_shape@15:/xlsx/body/p[1]:XLSX selectors require workbook or exactly one sheet segment before an optional cell/range leaf + #|office.selector.invalid_shape@17:/xlsx/workbook[1]:XLSX workbook is a singleton and cannot take a selection or coordinate leaf + #|office.selector.invalid_shape@23:/xlsx/workbook/cell[A1]:XLSX workbook is a singleton and cannot take a selection or coordinate leaf #|office.selector.invalid_shape@11:/xlsx/sheet:XLSX sheet requires [index] or [name="value"] #|office.selector.unsupported_predicate@22:/xlsx/sheet[id="Data"]:XLSX sheet selectors use the 'name' key, not 'id' #|office.selector.empty_value@17:/xlsx/sheet[name=""]/cell[A1]:named selector values cannot be empty diff --git a/office/selector_types.mbt b/office/selector_types.mbt index a8c96a01..9bd544ea 100644 --- a/office/selector_types.mbt +++ b/office/selector_types.mbt @@ -152,6 +152,18 @@ pub fn CellAddress::render(self : CellAddress) -> String { selector_column_name(self.column) + self.row.to_string() } +///| +/// Returns the validated 1-based column index (`A` = 1, `XFD` = 16384). +pub fn CellAddress::column(self : CellAddress) -> Int { + self.column +} + +///| +/// Returns the validated 1-based row index (`1` through `1048576`). +pub fn CellAddress::row(self : CellAddress) -> Int { + self.row +} + ///| fn render_segment_selection(selection : SegmentSelection) -> String { match selection { From f60e9ebdf65b91d73d61665882e8e78317171069 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 03:58:29 +0800 Subject: [PATCH 002/150] refactor(office): share bounded output accounting --- office/cmd/office/docx_projection.mbt | 14 +---- office/cmd/office/docx_read_output.mbt | 81 ++++++++++++++------------ office/cmd/office/docx_read_wbtest.mbt | 4 +- office/cmd/office/output.mbt | 24 ++++++++ 4 files changed, 71 insertions(+), 52 deletions(-) diff --git a/office/cmd/office/docx_projection.mbt b/office/cmd/office/docx_projection.mbt index 9baa4b0c..0d1d4e7b 100644 --- a/office/cmd/office/docx_projection.mbt +++ b/office/cmd/office/docx_projection.mbt @@ -113,19 +113,7 @@ fn docx_resource_failure( limit : Int, actual? : Int, ) -> CliFailure { - let fields : Map[String, Json] = { - "resource": Json::string(resource), - "limit": Json::number(limit.to_double()), - } - match actual { - Some(value) => fields["actual"] = Json::number(value.to_double()) - None => () - } - docx_cli_failure( - "office.docx.resource_limit", - "DOCX \{resource} exceeds the configured limit of \{limit}", - details=Json::object(fields), - ) + format_resource_failure("docx", resource, limit, actual?) } ///| diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index 47bb2c3c..ecaa5caa 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -178,9 +178,9 @@ fn PreparedDocxQuery::build( } ///| -struct BoundedDocxPayload { +struct BoundedOfficePayload { data : Json - budget : DocxOutputBudget + budget : OfficeOutputBudget warnings : Array[@lib.ProtocolWarning] } @@ -188,24 +188,29 @@ struct BoundedDocxPayload { /// Counts compact JSON output before allocation and bounds retained records /// while they are produced. The counter measures Unicode scalar characters, /// including the successful command's explicit trailing line feed. -struct DocxOutputBudget { +struct OfficeOutputBudget { + format : String maximum : Int mut used : Int } ///| -fn DocxOutputBudget::new(maximum : Int) -> DocxOutputBudget { - { maximum, used: docx_cli_output_framing_chars } +fn OfficeOutputBudget::new( + maximum : Int, + format? : String = "docx", +) -> OfficeOutputBudget { + { format, maximum, used: docx_cli_output_framing_chars } } ///| -fn DocxOutputBudget::reserve_chars( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_chars( + self : OfficeOutputBudget, count : Int, ) -> Unit raise CliFailure { if count < 0 || count > self.maximum - self.used { let actual = if count < 0 { self.used } else { self.used + count } - raise docx_resource_failure( + raise format_resource_failure( + self.format, "successful command output characters", self.maximum, actual~, @@ -215,8 +220,8 @@ fn DocxOutputBudget::reserve_chars( } ///| -fn DocxOutputBudget::reserve_plain_text( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_plain_text( + self : OfficeOutputBudget, value : String, ) -> Unit raise CliFailure { for _ in value { @@ -225,8 +230,8 @@ fn DocxOutputBudget::reserve_plain_text( } ///| -fn DocxOutputBudget::reserve_json_string( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_json_string( + self : OfficeOutputBudget, value : String, ) -> Unit raise CliFailure { self.reserve_chars(2) @@ -243,8 +248,8 @@ fn DocxOutputBudget::reserve_json_string( } ///| -fn DocxOutputBudget::reserve_json( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_json( + self : OfficeOutputBudget, value : Json, ) -> Unit raise CliFailure { match value { @@ -283,8 +288,8 @@ fn DocxOutputBudget::reserve_json( } ///| -fn DocxOutputBudget::reserve_array_item( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_array_item( + self : OfficeOutputBudget, value : Json, has_previous : Bool, ) -> Unit raise CliFailure { @@ -297,13 +302,13 @@ fn DocxOutputBudget::reserve_array_item( } ///| -fn DocxOutputBudget::remaining(self : DocxOutputBudget) -> Int { +fn OfficeOutputBudget::remaining(self : OfficeOutputBudget) -> Int { self.maximum - self.used } ///| -fn DocxOutputBudget::reserve_json_field( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_json_field( + self : OfficeOutputBudget, name : String, value : Json, has_previous : Bool, @@ -317,8 +322,8 @@ fn DocxOutputBudget::reserve_json_field( } ///| -fn DocxOutputBudget::reserve_warning( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_warning( + self : OfficeOutputBudget, warning : @lib.ProtocolWarning, ) -> Unit raise CliFailure { self.reserve_chars(1) @@ -328,8 +333,8 @@ fn DocxOutputBudget::reserve_warning( } ///| -fn DocxOutputBudget::reserve_success_envelope( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_success_envelope( + self : OfficeOutputBudget, data : Json, warnings : Array[@lib.ProtocolWarning], ) -> Unit raise CliFailure { @@ -353,8 +358,8 @@ fn DocxOutputBudget::reserve_success_envelope( } ///| -fn DocxOutputBudget::reserve_json_string_body( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_json_string_body( + self : OfficeOutputBudget, value : String, ) -> Unit raise CliFailure { for character in value { @@ -370,8 +375,8 @@ fn DocxOutputBudget::reserve_json_string_body( } ///| -fn DocxOutputBudget::reserve_nonnegative_integer_growth( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_nonnegative_integer_growth( + self : OfficeOutputBudget, value : Int, ) -> Unit raise CliFailure { let mut remaining = value @@ -385,8 +390,8 @@ fn DocxOutputBudget::reserve_nonnegative_integer_growth( } ///| -fn DocxOutputBudget::reserve_boolean_growth_from_true( - self : DocxOutputBudget, +fn OfficeOutputBudget::reserve_boolean_growth_from_true( + self : OfficeOutputBudget, value : Bool, ) -> Unit raise CliFailure { // `true` is the four-character skeleton placeholder; `false` needs one more. @@ -434,16 +439,16 @@ fn bounded_docx_payload( data : Json, warnings : Array[@lib.ProtocolWarning], maximum : Int, -) -> BoundedDocxPayload raise CliFailure { +) -> BoundedOfficePayload raise CliFailure { ignore(checked_output_limit(maximum)) - let budget = DocxOutputBudget::new(maximum) + let budget = OfficeOutputBudget::new(maximum) budget.reserve_success_envelope(data, warnings) { data, budget, warnings } } ///| fn checked_docx_json_output( - payload : BoundedDocxPayload, + payload : BoundedOfficePayload, ) -> String raise CliFailure { let envelope = @lib.output_success(payload.data, warnings=payload.warnings) // Compact output makes the preflight count exact and avoids indentation @@ -484,7 +489,7 @@ fn get_docx_payload( projection : DocxProjection, entry : DocxProjectionEntry, max_output_chars : Int, -) -> BoundedDocxPayload raise CliFailure { +) -> BoundedOfficePayload raise CliFailure { let children : Array[Json] = [] let fields = entry_base_json(entry, children) fields["schema"] = Json::string(@lib.SCHEMA_DOCX_ELEMENT) @@ -543,7 +548,7 @@ fn docx_text_payload( offset : Int, limit : Int, max_output_chars : Int, -) -> BoundedDocxPayload raise CliFailure { +) -> BoundedOfficePayload raise CliFailure { let entries : Array[Json] = [] let fields : Map[String, Json] = { "schema": Json::string(@lib.SCHEMA_DOCX_TEXT), @@ -741,7 +746,7 @@ fn docx_query_payload( under : DocxProjectionEntry?, spec : DocxQuerySpec, max_output_chars : Int, -) -> BoundedDocxPayload raise CliFailure { +) -> BoundedOfficePayload raise CliFailure { let filter_properties : Array[Json] = [] let filters : Map[String, Json] = { "properties": Json::array(filter_properties), @@ -888,7 +893,7 @@ fn section_ref_json(reference : @document.SectionRef) -> Json { ///| fn bounded_section_json( section : @document.DocumentSection, - budget : DocxOutputBudget, + budget : OfficeOutputBudget, has_previous : Bool, ) -> Json raise CliFailure { let headers : Array[Json] = [] @@ -919,7 +924,7 @@ fn bounded_section_json( ///| fn reader_diagnostics( projection : DocxProjection, - retention_budget? : DocxOutputBudget, + retention_budget? : OfficeOutputBudget, ) -> Array[Json] raise CliFailure { let values : Array[Json] = [] for message in projection.annotated.result().messages { @@ -944,7 +949,7 @@ fn reader_diagnostics( fn docx_outline_payload( projection : DocxProjection, max_output_chars? : Int = docx_cli_default_max_output_chars, -) -> BoundedDocxPayload raise CliFailure { +) -> BoundedOfficePayload raise CliFailure { ignore(checked_output_limit(max_output_chars)) let counts : Map[String, Int] = Map([]) let mut footnotes = 0 diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index fb06443c..80234037 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -616,7 +616,9 @@ test "compact JSON preflight matches serializer character accounting" { for _ in serialized { characters += 1 } - let budget = DocxOutputBudget::new(characters + docx_cli_output_framing_chars) + let budget = OfficeOutputBudget::new( + characters + docx_cli_output_framing_chars, + ) budget.reserve_json(value) assert_eq(budget.used, characters + docx_cli_output_framing_chars) } diff --git a/office/cmd/office/output.mbt b/office/cmd/office/output.mbt index 6efaa3cd..f0b63c47 100644 --- a/office/cmd/office/output.mbt +++ b/office/cmd/office/output.mbt @@ -5,6 +5,30 @@ enum OutputMode { JsonLines } +///| +fn format_resource_failure( + format : String, + resource : String, + limit : Int, + actual? : Int, +) -> CliFailure { + let fields : Map[String, Json] = { + "resource": Json::string(resource), + "limit": Json::number(limit.to_double()), + } + match actual { + Some(value) => fields["actual"] = Json::number(value.to_double()) + None => () + } + CliFailure( + @lib.protocol_error( + "office.\{format}.resource_limit", + "\{format.to_upper()} \{resource} exceeds the configured limit of \{limit}", + details=Json::object(fields), + ), + ) +} + ///| fn terminal_hex_digit(value : Int) -> Char { match value { From cd4e0be64d623639fc4886a0dc01c5043962e1de Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:07:47 +0800 Subject: [PATCH 003/150] feat(office): add bounded XLSX read projection --- office/cmd/office/moon.pkg | 2 + office/cmd/office/read_package.mbt | 163 +++++++ office/cmd/office/xlsx_read_model.mbt | 566 +++++++++++++++++++++++++ office/cmd/office/xlsx_read_wbtest.mbt | 115 +++++ 4 files changed, 846 insertions(+) create mode 100644 office/cmd/office/read_package.mbt create mode 100644 office/cmd/office/xlsx_read_model.mbt create mode 100644 office/cmd/office/xlsx_read_wbtest.mbt diff --git a/office/cmd/office/moon.pkg b/office/cmd/office/moon.pkg index 21b5bc8f..2bd4b776 100644 --- a/office/cmd/office/moon.pkg +++ b/office/cmd/office/moon.pkg @@ -6,6 +6,8 @@ import { "bobzhang/docx2html/opc", "bobzhang/docx2html/paths" @docx_paths, "bobzhang/docx2html/xml", + "bobzhang/mbtexcel/inspect", + "bobzhang/mbtexcel/xlsx", "bobzhang/mbtexcel/zip", "bobzhang/office" @lib, "bobzhang/office/raw", diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt new file mode 100644 index 00000000..9cb11736 --- /dev/null +++ b/office/cmd/office/read_package.mbt @@ -0,0 +1,163 @@ +///| +let office_read_max_package_bytes : Int = 64 * 1024 * 1024 + +///| +let office_read_max_zip_entries : Int = 4096 + +///| +let office_read_max_zip_entry_bytes : Int = 32 * 1024 * 1024 + +///| +let office_read_max_zip_total_bytes : Int = 128 * 1024 * 1024 + +///| +let office_read_max_preserved_source_bytes : Int = 64 * 1024 * 1024 + 65_535 + +///| +let office_read_max_xml_part_bytes : Int = 16 * 1024 * 1024 + +///| +struct OfficeReadPackage { + file : String + format : @lib.DocumentFormat + archive : @zip.Archive +} + +///| +fn office_read_format_hint( + file : String, +) -> @lib.DocumentFormat raise CliFailure { + let lower = file.to_lower() + if lower.has_suffix(".xlsx") { + Xlsx + } else if lower.has_suffix(".docx") { + Docx + } else { + raise CliFailure(identify_error(@lib.UnsupportedFileExtension(file), file)) + } +} + +///| +fn office_zip_read_failure( + error : @zip.ZipError, + file : String, + expected : @lib.DocumentFormat, +) -> CliFailure { + match error { + @zip.ResourceLimitExceeded(kind~, limit~, actual~) => + CliFailure( + @lib.protocol_error( + "office.\{expected.name()}.resource_limit", + "\{expected.name().to_upper()} ZIP \{bounded_text(kind, 80)} exceeds the configured limit", + details=Json::object({ + "file": Json::string(bounded_text(file, 160)), + "resource": Json::string(bounded_text(kind, 80)), + "limit": Json::number(limit.to_double()), + "actual": Json::number(actual.to_double()), + }), + ), + ) + @zip.OutputLimitExceeded(limit~) => + format_resource_failure(expected.name(), "ZIP output", limit) + _ => + CliFailure( + @lib.protocol_error( + "office.invalid_package", + "invalid Office package: archive is not a readable bounded ZIP", + details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), + ), + ) + } +} + +///| +fn office_xlsx_read_limits() -> @xlsx.ReadLimits raise @xlsx.XlsxError { + @xlsx.ReadLimits::with_values( + max_package_bytes=office_read_max_package_bytes, + max_archive_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + max_xml_part_bytes=office_read_max_xml_part_bytes, + ) +} + +///| +async fn read_office_package(file : String) -> OfficeReadPackage { + let expected = office_read_format_hint(file) + let data = read_bounded_file( + file, + office_read_max_package_bytes, + "office.file_read_failed", + "input package", + limit_code="office.\{expected.name()}.resource_limit", + ) + let archive = @zip.read_limited( + data, + max_package_bytes=office_read_max_package_bytes, + max_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + ) catch { + error => raise office_zip_read_failure(error, file, expected) + } + let format = @lib.detect_archive_format(file, archive) catch { + error => raise CliFailure(identify_error(error, file)) + } + { file, format, archive } +} + +///| +fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { + match error { + @xlsx.ResourceLimitExceeded(kind~, limit~, actual~) => + CliFailure( + @lib.protocol_error( + "office.xlsx.resource_limit", + "XLSX \{bounded_text(kind, 80)} exceeds the configured limit", + details=Json::object({ + "file": Json::string(bounded_text(file, 160)), + "resource": Json::string(bounded_text(kind, 80)), + "limit": Json::number(limit.to_double()), + "actual": Json::number(actual.to_double()), + }), + ), + ) + @xlsx.InvalidPackage(msg~) | @xlsx.InvalidXml(msg~) => + CliFailure( + @lib.protocol_error( + "office.invalid_package", + "invalid XLSX package: " + bounded_text(msg, 320), + details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), + ), + ) + _ => + CliFailure( + @lib.protocol_error( + "office.xlsx.read_failed", + "could not read XLSX workbook", + details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), + ), + ) + } +} + +///| +fn open_xlsx_read_package( + source : OfficeReadPackage, +) -> @xlsx.Workbook raise CliFailure { + if source.format is Docx { + raise CliFailure( + @lib.protocol_error( + "office.xlsx.format_mismatch", "XLSX reader received a validated DOCX package", + ), + ) + } + let limits = office_xlsx_read_limits() catch { + error => raise xlsx_read_failure(error, source.file) + } + @xlsx.read_bounded_archive(source.archive, limits~) catch { + error => raise xlsx_read_failure(error, source.file) + } +} diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt new file mode 100644 index 00000000..66c9be89 --- /dev/null +++ b/office/cmd/office/xlsx_read_model.mbt @@ -0,0 +1,566 @@ +///| +let xlsx_cli_hard_max_scan_cells : Int = 100_000 + +///| +let xlsx_cli_max_cell_string_chars : Int = 1024 * 1024 + +///| +let xlsx_cli_max_scanned_string_chars : Int = 16 * 1024 * 1024 + +///| +let xlsx_cli_max_metadata_items : Int = 10_000 + +///| +let xlsx_cli_max_metadata_string_chars : Int = 8 * 1024 * 1024 + +///| +enum XlsxSheetContent { + Worksheet(@xlsx.Worksheet) + ChartSheet(@xlsx.ChartSheet) +} + +///| +struct XlsxSheetTarget { + name : String + index : Int + path : String + content : XlsxSheetContent +} + +///| +struct XlsxProjection { + file : String + workbook : @xlsx.Workbook + sheet_names : Array[String] + max_scan_cells : Int +} + +///| +struct XlsxRect { + col_lo : Int + row_lo : Int + col_hi : Int + row_hi : Int +} + +///| +enum XlsxResolvedSelector { + Workbook(path~ : String) + Sheet(XlsxSheetTarget) + Cell(XlsxSheetTarget, XlsxRect, path~ : String) + Range(XlsxSheetTarget, XlsxRect, path~ : String) +} + +///| +struct XlsxCellSnapshot { + path : String + reference : String + row : Int + column : Int + raw : @xlsx.CellValue? + formatted : String? + formula : String? + style_id : Int +} + +///| +struct XlsxStringBudget { + maximum : Int + per_value_maximum : Int + mut used : Int +} + +///| +struct XlsxScanBudget { + maximum : Int + strings : XlsxStringBudget + mut scanned : Int +} + +///| +struct XlsxMetadataBudget { + maximum_items : Int + maximum_string_chars : Int + mut items : Int + mut string_chars : Int +} + +///| +fn xlsx_cli_failure( + code : String, + message : String, + details? : Json, +) -> CliFailure { + CliFailure(@lib.protocol_error(code, message, details?)) +} + +///| +fn xlsx_resource_failure( + resource : String, + limit : Int, + actual? : Int, +) -> CliFailure { + format_resource_failure("xlsx", resource, limit, actual?) +} + +///| +fn xlsx_scan_resource_failure( + resource : String, + limit : Int, + actual : Int64, +) -> CliFailure { + xlsx_cli_failure( + "office.xlsx.resource_limit", + "XLSX \{resource} exceeds the configured limit", + details=Json::object({ + "format": Json::string("xlsx"), + "resource": Json::string(resource), + "limit": Json::number(limit.to_double()), + "actual": Json::number(actual.to_double()), + }), + ) +} + +///| +fn[T] xlsx_call( + file : String, + action : () -> T raise @xlsx.XlsxError, +) -> T raise CliFailure { + action() catch { + error => raise xlsx_read_failure(error, file) + } +} + +///| +fn canonical_xlsx_sheet_path(name : String) -> String raise CliFailure { + @lib.selector_for_xlsx_sheet(name).render() catch { + error => raise selector_failure(error) + } +} + +///| +fn canonical_xlsx_cell_path( + sheet : String, + reference : String, +) -> String raise CliFailure { + @lib.selector_for_xlsx_cell(sheet, reference).render() catch { + error => raise selector_failure(error) + } +} + +///| +fn canonical_xlsx_range_path( + sheet : String, + reference : String, +) -> String raise CliFailure { + @lib.selector_for_xlsx_range(sheet, reference).render() catch { + error => raise selector_failure(error) + } +} + +///| +fn make_xlsx_projection( + file : String, + workbook : @xlsx.Workbook, + max_elements : Int, +) -> XlsxProjection raise CliFailure { + if max_elements < 1 || max_elements > docx_cli_hard_max_elements { + raise xlsx_cli_failure( + "office.invalid_arguments", + "--max-elements must be between 1 and \{docx_cli_hard_max_elements}", + details=Json::object({ + "argument": Json::string("max-elements"), + "minimum": Json::number(1), + "maximum": Json::number(docx_cli_hard_max_elements.to_double()), + }), + ) + } + { + file, + workbook, + sheet_names: workbook.get_sheet_list(), + max_scan_cells: max_elements.min(xlsx_cli_hard_max_scan_cells), + } +} + +///| +fn open_xlsx_projection( + source : OfficeReadPackage, + max_elements : Int, +) -> XlsxProjection raise CliFailure { + let workbook = open_xlsx_read_package(source) + make_xlsx_projection(source.file, workbook, max_elements) +} + +///| +fn XlsxRect::cell_count(self : XlsxRect) -> Int64 { + (self.col_hi - self.col_lo + 1).to_int64() * + (self.row_hi - self.row_lo + 1).to_int64() +} + +///| +fn XlsxRect::reference( + self : XlsxRect, + file : String, +) -> String raise CliFailure { + let first = xlsx_call(file, () => { + @xlsx.coordinates_to_cell_name(self.col_lo, self.row_lo) + }) + if self.col_lo == self.col_hi && self.row_lo == self.row_hi { + first + } else { + let last = xlsx_call(file, () => { + @xlsx.coordinates_to_cell_name(self.col_hi, self.row_hi) + }) + first + ":" + last + } +} + +///| +fn xlsx_rect_from_coordinate(coordinate : @lib.SelectorCoordinate) -> XlsxRect { + match coordinate { + Cell(address) => + { + col_lo: address.column(), + row_lo: address.row(), + col_hi: address.column(), + row_hi: address.row(), + } + Range(first, last) => + { + col_lo: first.column(), + row_lo: first.row(), + col_hi: last.column(), + row_hi: last.row(), + } + } +} + +///| +fn xlsx_used_rect(worksheet : @xlsx.Worksheet) -> XlsxRect? { + let max_col = worksheet.max_col() + let max_row = worksheet.max_row() + if max_col < 1 || max_row < 1 { + None + } else { + Some({ col_lo: 1, row_lo: 1, col_hi: max_col, row_hi: max_row }) + } +} + +///| +fn XlsxProjection::check_scan_rect( + self : XlsxProjection, + rect : XlsxRect, +) -> Unit raise CliFailure { + let actual = rect.cell_count() + if actual > self.max_scan_cells.to_int64() { + raise xlsx_scan_resource_failure( + "scanned cells", + self.max_scan_cells, + actual, + ) + } +} + +///| +fn XlsxProjection::find_sheet_by_name( + self : XlsxProjection, + name : String, +) -> XlsxSheetTarget? raise CliFailure { + let mut found_index = -1 + for index, candidate in self.sheet_names { + if candidate == name { + found_index = index + break + } + } + if found_index < 0 { + return None + } + let path = canonical_xlsx_sheet_path(name) + match self.workbook.sheet(name) { + Some(sheet) => + Some({ name, index: found_index, path, content: Worksheet(sheet) }) + None => + match self.workbook.chart_sheet(name) { + Some(sheet) => + Some({ name, index: found_index, path, content: ChartSheet(sheet) }) + None => None + } + } +} + +///| +fn XlsxProjection::resolve_sheet_segment( + self : XlsxProjection, + segment : @lib.SelectorSegment, + requested : String, +) -> XlsxSheetTarget raise CliFailure { + let target = match segment.key_value("name") { + Some(name) => self.find_sheet_by_name(name) + None => + match segment.position() { + Some(position) => + match self.sheet_names.get(position - 1) { + Some(name) => self.find_sheet_by_name(name) + None => None + } + None => None + } + } + match target { + Some(sheet) => sheet + None => + raise xlsx_cli_failure( + "office.xlsx.selector_not_found", + "XLSX selector did not resolve in this workbook snapshot", + details=Json::object({ + "selector": Json::string(bounded_text(requested, 240)), + }), + ) + } +} + +///| +fn XlsxSheetTarget::worksheet( + self : XlsxSheetTarget, + selector : String, +) -> @xlsx.Worksheet raise CliFailure { + match self.content { + Worksheet(sheet) => sheet + ChartSheet(_) => + raise xlsx_cli_failure( + "office.xlsx.unsupported_sheet_kind", + "cell and range selectors require a worksheet, not a chart sheet", + details=Json::object({ + "selector": Json::string(selector), + "sheet": Json::string(self.name), + "sheet_kind": Json::string("chart_sheet"), + }), + ) + } +} + +///| +fn resolve_xlsx_selector( + projection : XlsxProjection, + input : String, +) -> XlsxResolvedSelector raise CliFailure { + let selector = @lib.parse_selector(input) catch { + error => raise selector_failure(error) + } + if selector.document_format() is Docx { + raise xlsx_cli_failure( + "office.xlsx.selector_format_mismatch", + "XLSX structured reads require an /xlsx selector", + details=Json::object({ + "selector": Json::string(selector.render()), + "expected_format": Json::string("xlsx"), + "actual_format": Json::string("docx"), + }), + ) + } + let segments = selector.segments() + if segments.length() == 1 && segments[0].name() == "workbook" { + return Workbook(path="/xlsx/workbook") + } + let sheet = projection.resolve_sheet_segment(segments[0], selector.render()) + match selector.coordinate() { + None => Sheet(sheet) + Some(coordinate) => { + ignore(sheet.worksheet(selector.render())) + let rect = xlsx_rect_from_coordinate(coordinate) + projection.check_scan_rect(rect) + let reference = rect.reference(projection.file) + match coordinate { + Cell(_) => + Cell( + sheet, + rect, + path=canonical_xlsx_cell_path(sheet.name, reference), + ) + Range(_, _) => + Range( + sheet, + rect, + path=canonical_xlsx_range_path(sheet.name, reference), + ) + } + } + } +} + +///| +fn XlsxStringBudget::new( + maximum? : Int = xlsx_cli_max_scanned_string_chars, + per_value_maximum? : Int = xlsx_cli_max_cell_string_chars, +) -> XlsxStringBudget { + { maximum, per_value_maximum, used: 0 } +} + +///| +fn XlsxStringBudget::charge( + self : XlsxStringBudget, + resource : String, + value : String, +) -> Unit raise CliFailure { + let count = value.length() + if count > self.per_value_maximum { + raise xlsx_resource_failure( + "\{resource} characters", + self.per_value_maximum, + actual=count, + ) + } + if count > self.maximum - self.used { + raise xlsx_resource_failure( + "scanned string characters", + self.maximum, + actual=self.used + count, + ) + } + self.used += count +} + +///| +fn XlsxScanBudget::new(maximum : Int) -> XlsxScanBudget { + { maximum, strings: XlsxStringBudget::new(), scanned: 0 } +} + +///| +fn XlsxScanBudget::charge_cell(self : XlsxScanBudget) -> Unit raise CliFailure { + if self.scanned >= self.maximum { + raise xlsx_resource_failure( + "scanned cells", + self.maximum, + actual=self.scanned + 1, + ) + } + self.scanned += 1 +} + +///| +fn XlsxScanBudget::charge_snapshot_strings( + self : XlsxScanBudget, + snapshot : XlsxCellSnapshot, +) -> Unit raise CliFailure { + match snapshot.raw { + Some(String(value)) => self.strings.charge("cell value", value) + Some(Error(value)) => self.strings.charge("cell error", value) + _ => () + } + match snapshot.formatted { + Some(value) => self.strings.charge("formatted cell value", value) + None => () + } + match snapshot.formula { + Some(value) => self.strings.charge("cell formula", value) + None => () + } +} + +///| +fn xlsx_cell_snapshot( + projection : XlsxProjection, + sheet : XlsxSheetTarget, + row : Int, + column : Int, + budget : XlsxScanBudget, +) -> XlsxCellSnapshot raise CliFailure { + budget.charge_cell() + let reference = xlsx_call(projection.file, () => { + @xlsx.coordinates_to_cell_name(column, row) + }) + let raw = xlsx_call(projection.file, () => { + projection.workbook.get_cell_value_raw(sheet.name, reference) + }) + let normalized_raw = match raw { + Some(String("")) => None + value => value + } + let formula = match + xlsx_call(projection.file, () => { + projection.workbook.get_cell_formula(sheet.name, reference) + }) { + Some("") => None + value => value + } + let style_id = xlsx_call(projection.file, () => { + @inspect.effective_style_id( + projection.workbook, + sheet.name, + reference, + row~, + col=column, + ) + }) + let formatted = match normalized_raw { + Some(_) => + Some( + xlsx_call(projection.file, () => { + projection.workbook.get_cell_value(sheet.name, reference) + }).unwrap_or(""), + ) + None => None + } + let snapshot : XlsxCellSnapshot = { + path: canonical_xlsx_cell_path(sheet.name, reference), + reference, + row, + column, + raw: normalized_raw, + formatted, + formula, + style_id, + } + budget.charge_snapshot_strings(snapshot) + snapshot +} + +///| +fn XlsxCellSnapshot::is_present(self : XlsxCellSnapshot) -> Bool { + self.raw is Some(_) || self.formula is Some(_) || self.style_id != 0 +} + +///| +fn XlsxMetadataBudget::new() -> XlsxMetadataBudget { + { + maximum_items: xlsx_cli_max_metadata_items, + maximum_string_chars: xlsx_cli_max_metadata_string_chars, + items: 0, + string_chars: 0, + } +} + +///| +fn XlsxMetadataBudget::charge_item( + self : XlsxMetadataBudget, + strings : Array[String], +) -> Unit raise CliFailure { + if self.items >= self.maximum_items { + raise xlsx_resource_failure( + "metadata items", + self.maximum_items, + actual=self.items + 1, + ) + } + self.items += 1 + for value in strings { + let count = value.length() + if count > xlsx_cli_max_cell_string_chars { + raise xlsx_resource_failure( + "metadata string characters", + xlsx_cli_max_cell_string_chars, + actual=count, + ) + } + if count > self.maximum_string_chars - self.string_chars { + raise xlsx_resource_failure( + "metadata string characters", + self.maximum_string_chars, + actual=self.string_chars + count, + ) + } + self.string_chars += count + } +} diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt new file mode 100644 index 00000000..0a2cbc4f --- /dev/null +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -0,0 +1,115 @@ +///| +fn xlsx_read_test_projection() -> XlsxProjection raise { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + ignore(workbook.new_sheet("财务 \"Q1\"")) + workbook.set_cell_str("Data", "A1", "alpha") + workbook.set_cell_formula("Data", "B2", "SUM(1,2)", value="3") + make_xlsx_projection("fixture.xlsx", workbook, 100) +} + +///| +fn xlsx_cli_error( + run : () -> Unit raise CliFailure, +) -> @lib.ProtocolError raise { + try run() catch { + CliFailure(error) => error + } noraise { + _ => fail("expected XLSX CLI failure") + } +} + +///| +test "XLSX selectors resolve to stable named canonical paths" { + let projection = xlsx_read_test_projection() + guard resolve_xlsx_selector(projection, "/xlsx/sheet[2]") is Sheet(second) else { + fail("expected sheet selector") + } + assert_eq(second.name, "财务 \"Q1\"") + assert_eq(second.index, 1) + assert_eq(second.path, "/xlsx/sheet[name=\"财务 \\\"Q1\\\"\"]") + assert_eq(@lib.parse_selector(second.path).render(), second.path) + guard resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/range[B2:A1]", + ) + is Range(sheet, rect, path~) else { + fail("expected range selector") + } + assert_eq(sheet.name, "Data") + assert_eq(rect.cell_count(), 4L) + assert_eq(path, "/xlsx/sheet[name=\"Data\"]/range[A1:B2]") + assert_eq(@lib.parse_selector(path).render(), path) +} + +///| +test "XLSX selector failures distinguish format, absence, and scan limits" { + let projection = xlsx_read_test_projection() + let mismatch = xlsx_cli_error(() => { + ignore(resolve_xlsx_selector(projection, "/docx/body")) + }) + assert_eq(mismatch.code, "office.xlsx.selector_format_mismatch") + let missing = xlsx_cli_error(() => { + ignore(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Missing\"]")) + }) + assert_eq(missing.code, "office.xlsx.selector_not_found") + let oversized = xlsx_cli_error(() => { + ignore( + resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/range[A1:K10]", + ), + ) + }) + assert_eq(oversized.code, "office.xlsx.resource_limit") + guard oversized.details is Some(Object(fields)) else { + fail("expected scan-limit details") + } + assert_eq(fields.get("resource"), Some(Json::string("scanned cells"))) + assert_eq(fields.get("limit"), Some(Json::number(100))) + assert_eq(fields.get("actual"), Some(Json::number(110))) +} + +///| +test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" { + let projection = xlsx_read_test_projection() + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[B2]") + is Cell(sheet, rect, path~) else { + fail("expected cell selector") + } + let budget = XlsxScanBudget::new(1) + let snapshot = xlsx_cell_snapshot( + projection, + sheet, + rect.row_lo, + rect.col_lo, + budget, + ) + assert_true(snapshot.is_present()) + assert_eq(snapshot.path, path) + assert_eq(snapshot.reference, "B2") + assert_eq(snapshot.row, 2) + assert_eq(snapshot.column, 2) + assert_eq(snapshot.formula, Some("SUM(1,2)")) + assert_eq(snapshot.formatted, Some("3")) + assert_true(snapshot.raw is Some(String("3"))) + assert_eq(budget.scanned, 1) +} + +///| +test "XLSX string and metadata accounting fail before unbounded retention" { + let per_value = XlsxStringBudget::new(maximum=20, per_value_maximum=3) + let too_wide = xlsx_cli_error(() => per_value.charge("cell value", "four")) + assert_eq(too_wide.code, "office.xlsx.resource_limit") + let aggregate = XlsxStringBudget::new(maximum=5, per_value_maximum=5) + aggregate.charge("cell value", "abc") + let too_many = xlsx_cli_error(() => aggregate.charge("cell formula", "def")) + assert_eq(too_many.code, "office.xlsx.resource_limit") + let metadata : XlsxMetadataBudget = { + maximum_items: 1, + maximum_string_chars: 4, + items: 0, + string_chars: 0, + } + metadata.charge_item(["name"]) + let metadata_limit = xlsx_cli_error(() => metadata.charge_item([])) + assert_eq(metadata_limit.code, "office.xlsx.resource_limit") +} From e5e6bd26c52d808d3a25423bb05be252e1bda3b5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:15:17 +0800 Subject: [PATCH 004/150] feat(office): add bounded XLSX read outputs --- office/capabilities.mbt | 12 + office/cmd/office/docx_read_output.mbt | 78 ++- office/cmd/office/moon.pkg | 1 + office/cmd/office/pkg.generated.mbti | 40 +- office/cmd/office/xlsx_query.mbt | 267 ++++++++++ office/cmd/office/xlsx_read_model.mbt | 63 +++ office/cmd/office/xlsx_read_output.mbt | 704 +++++++++++++++++++++++++ office/cmd/office/xlsx_read_wbtest.mbt | 139 +++++ office/pkg.generated.mbti | 8 + 9 files changed, 1290 insertions(+), 22 deletions(-) create mode 100644 office/cmd/office/xlsx_query.mbt create mode 100644 office/cmd/office/xlsx_read_output.mbt diff --git a/office/capabilities.mbt b/office/capabilities.mbt index 0cb0e61e..8c4866be 100644 --- a/office/capabilities.mbt +++ b/office/capabilities.mbt @@ -32,6 +32,18 @@ pub const SCHEMA_DOCX_TEXT : String = "office.docx.text/1" ///| pub const SCHEMA_DOCX_QUERY : String = "office.docx.query/1" +///| +pub const SCHEMA_XLSX_OUTLINE : String = "office.xlsx.outline/1" + +///| +pub const SCHEMA_XLSX_ELEMENT : String = "office.xlsx.element/1" + +///| +pub const SCHEMA_XLSX_TEXT : String = "office.xlsx.text/1" + +///| +pub const SCHEMA_XLSX_QUERY : String = "office.xlsx.query/1" + ///| /// One declared input or output field in the Office capability registry. pub(all) struct CapabilityField { diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index ecaa5caa..c83d4e6d 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -417,15 +417,17 @@ fn checked_output_limit(maximum : Int) -> Int raise CliFailure { } ///| -fn ensure_docx_output_limit( +fn ensure_office_output_limit( output : String, maximum : Int, + format : String, ) -> Unit raise CliFailure { let mut count = docx_cli_output_framing_chars for _ in output { count += 1 if count > maximum { - raise docx_resource_failure( + raise format_resource_failure( + format, "successful command output characters", maximum, actual=count, @@ -435,55 +437,87 @@ fn ensure_docx_output_limit( } ///| -fn bounded_docx_payload( +fn bounded_office_payload( data : Json, warnings : Array[@lib.ProtocolWarning], maximum : Int, + format : String, ) -> BoundedOfficePayload raise CliFailure { ignore(checked_output_limit(maximum)) - let budget = OfficeOutputBudget::new(maximum) + let budget = OfficeOutputBudget::new(maximum, format~) budget.reserve_success_envelope(data, warnings) { data, budget, warnings } } ///| -fn checked_docx_json_output( +fn bounded_docx_payload( + data : Json, + warnings : Array[@lib.ProtocolWarning], + maximum : Int, +) -> BoundedOfficePayload raise CliFailure { + bounded_office_payload(data, warnings, maximum, "docx") +} + +///| +fn checked_office_json_output( payload : BoundedOfficePayload, ) -> String raise CliFailure { let envelope = @lib.output_success(payload.data, warnings=payload.warnings) // Compact output makes the preflight count exact and avoids indentation // overhead that would otherwise need a second allocation-sized pass. let output = envelope.stringify() - ensure_docx_output_limit(output, payload.budget.maximum) + ensure_office_output_limit( + output, + payload.budget.maximum, + payload.budget.format, + ) let mut actual = 0 for _ in output { actual += 1 } if actual + docx_cli_output_framing_chars != payload.budget.used { - raise docx_cli_failure( - "office.docx.output_accounting_mismatch", - "internal DOCX output accounting did not match serialization", - details=Json::object({ - "accounted": Json::number(payload.budget.used.to_double()), - "actual": Json::number( - (actual + docx_cli_output_framing_chars).to_double(), - ), - }), + raise CliFailure( + @lib.protocol_error( + "office.\{payload.budget.format}.output_accounting_mismatch", + "internal \{payload.budget.format.to_upper()} output accounting did not match serialization", + details=Json::object({ + "accounted": Json::number(payload.budget.used.to_double()), + "actual": Json::number( + (actual + docx_cli_output_framing_chars).to_double(), + ), + }), + ), ) } output } ///| -fn checked_docx_human_output( +fn checked_docx_json_output( + payload : BoundedOfficePayload, +) -> String raise CliFailure { + checked_office_json_output(payload) +} + +///| +fn checked_office_human_output( output : String, maximum : Int, + format : String, ) -> String raise CliFailure { ignore(checked_output_limit(maximum)) - ensure_docx_output_limit(output, maximum) + ensure_office_output_limit(output, maximum, format) output } +///| +fn checked_docx_human_output( + output : String, + maximum : Int, +) -> String raise CliFailure { + checked_office_human_output(output, maximum, "docx") +} + ///| fn get_docx_payload( projection : DocxProjection, @@ -1098,6 +1132,7 @@ fn docx_outline_payload( ///| struct DocxHumanWriter { + format : String maximum : Int output : StringBuilder mut used : Int @@ -1105,9 +1140,13 @@ struct DocxHumanWriter { } ///| -fn DocxHumanWriter::new(maximum : Int) -> DocxHumanWriter raise CliFailure { +fn DocxHumanWriter::new( + maximum : Int, + format? : String = "docx", +) -> DocxHumanWriter raise CliFailure { ignore(checked_output_limit(maximum)) { + format, maximum, output: StringBuilder::new(), used: docx_cli_output_framing_chars, @@ -1126,7 +1165,8 @@ fn DocxHumanWriter::write_char( value : Char, ) -> Unit raise CliFailure { if self.used >= self.maximum { - raise docx_resource_failure( + raise format_resource_failure( + self.format, "successful command output characters", self.maximum, actual=self.used + 1, diff --git a/office/cmd/office/moon.pkg b/office/cmd/office/moon.pkg index 2bd4b776..cff6e2d5 100644 --- a/office/cmd/office/moon.pkg +++ b/office/cmd/office/moon.pkg @@ -20,6 +20,7 @@ import { "moonbitlang/core/env", "moonbitlang/core/json", "moonbitlang/core/sorted_map", + "moonbitlang/core/string", "tonyfettes/unicode", } diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 27037ce2..e3d0a6d8 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -19,7 +19,7 @@ type AnnotationProjectionIndex type AnnotationStoryPlan -type BoundedDocxPayload +type BoundedOfficePayload type DocxCollectedText @@ -27,8 +27,6 @@ type DocxHumanWriter type DocxLinearPattern -type DocxOutputBudget - type DocxProjection type DocxProjectionBuilder @@ -49,10 +47,46 @@ type DocxTextCollector type HelpQuery +type OfficeOutputBudget + +type OfficeReadPackage + type OutputMode type PreparedDocxQuery +type XlsxCellFilter + +type XlsxCellKind + +type XlsxCellPage + +type XlsxCellPredicate + +type XlsxCellSnapshot + +type XlsxMetadataBudget + +type XlsxNumberOperator + +type XlsxProjection + +type XlsxQuerySpec + +type XlsxRect + +type XlsxResolvedSelector + +type XlsxScanBudget + +type XlsxScanRegion + +type XlsxSheetContent + +type XlsxSheetTarget + +type XlsxStringBudget + // Type aliases // Traits diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt new file mode 100644 index 00000000..dd688a15 --- /dev/null +++ b/office/cmd/office/xlsx_query.mbt @@ -0,0 +1,267 @@ +///| +let xlsx_cli_max_query_selector_chars : Int = 4096 + +///| +let xlsx_cli_max_query_predicates : Int = 16 + +///| +let xlsx_cli_max_query_value_chars : Int = 1024 + +///| +enum XlsxCellKind { + Formula + Number + StringValue + Boolean + ErrorValue +} + +///| +enum XlsxNumberOperator { + Greater + GreaterOrEqual + Less + LessOrEqual + Equal + NotEqual +} + +///| +enum XlsxCellPredicate { + HasType(XlsxCellKind) + ValueCompare(XlsxNumberOperator, Double) + HasFormula + FormulaContains(String) + TextEquals(String) + TextContains(String) +} + +///| +struct XlsxQuerySpec { + selector : String + predicates : Array[XlsxCellPredicate] + offset : Int + limit : Int +} + +///| +fn xlsx_query_failure(message : String, selector : String) -> CliFailure { + xlsx_cli_failure( + "office.xlsx.invalid_query", + message, + details=Json::object({ + "selector": Json::string(bounded_text(selector, 240)), + }), + ) +} + +///| +fn require_xlsx_query_value( + label : String, + value : String, + selector : String, +) -> String raise CliFailure { + let trimmed = value.trim().to_owned() + if trimmed == "" { + raise xlsx_query_failure("\{label} needs a non-empty value", selector) + } + if !has_at_most_chars(trimmed, xlsx_cli_max_query_value_chars) { + raise xlsx_query_failure( + "\{label} value exceeds \{xlsx_cli_max_query_value_chars} characters", + selector, + ) + } + trimmed +} + +///| +fn parse_xlsx_number_predicate( + rest : String, + selector : String, +) -> XlsxCellPredicate raise CliFailure { + let (operator, number_text) = if rest.has_prefix(">=") { + (GreaterOrEqual, rest[2:].to_owned()) + } else if rest.has_prefix("<=") { + (LessOrEqual, rest[2:].to_owned()) + } else if rest.has_prefix("!=") { + (NotEqual, rest[2:].to_owned()) + } else if rest.has_prefix(">") { + (Greater, rest[1:].to_owned()) + } else if rest.has_prefix("<") { + (Less, rest[1:].to_owned()) + } else if rest.has_prefix("=") { + (Equal, rest[1:].to_owned()) + } else { + raise xlsx_query_failure( + "a value predicate requires one of >, >=, <, <=, =, or !=", selector, + ) + } + let bounded = require_xlsx_query_value( + "value comparison", number_text, selector, + ) + let number = @string.parse_double(bounded) catch { + _ => + raise xlsx_query_failure( + "value predicate expects a finite number", selector, + ) + } + if number != number || number.is_inf() { + raise xlsx_query_failure( + "value predicate expects a finite number", selector, + ) + } + ValueCompare(operator, number) +} + +///| +fn parse_xlsx_predicate( + body : String, + selector : String, +) -> XlsxCellPredicate raise CliFailure { + let predicate = body.trim().to_owned() + if predicate.has_prefix("type=") { + match predicate[5:].to_owned() { + "formula" => HasType(Formula) + "number" => HasType(Number) + "string" => HasType(StringValue) + "bool" => HasType(Boolean) + "error" => HasType(ErrorValue) + _ => + raise xlsx_query_failure( + "type predicate expects formula, number, string, bool, or error", selector, + ) + } + } else if predicate.has_prefix("value") { + parse_xlsx_number_predicate(predicate[5:].to_owned(), selector) + } else if predicate.has_prefix("formula~=") { + FormulaContains( + require_xlsx_query_value("formula~=", predicate[9:].to_owned(), selector), + ) + } else if predicate == "formula" { + HasFormula + } else if predicate.has_prefix("text~=") { + TextContains( + require_xlsx_query_value("text~=", predicate[6:].to_owned(), selector), + ) + } else if predicate.has_prefix("text=") { + TextEquals( + require_xlsx_query_value("text=", predicate[5:].to_owned(), selector), + ) + } else { + raise xlsx_query_failure("unknown XLSX cell predicate", selector) + } +} + +///| +fn parse_xlsx_query_selector( + source : String, +) -> Array[XlsxCellPredicate] raise CliFailure { + if !has_at_most_chars(source, xlsx_cli_max_query_selector_chars) { + raise xlsx_query_failure( + "query selector exceeds \{xlsx_cli_max_query_selector_chars} characters", + source, + ) + } + let selector = source.trim().to_owned() + if selector != "cell" && !selector.has_prefix("cell[") { + raise xlsx_query_failure( + "selector must be 'cell' followed by optional [predicate] groups", selector, + ) + } + let predicates : Array[XlsxCellPredicate] = [] + let mut rest = selector[4:].to_owned() + while rest != "" { + if predicates.length() >= xlsx_cli_max_query_predicates { + raise xlsx_cli_failure( + "office.xlsx.query_predicate_limit", + "at most \{xlsx_cli_max_query_predicates} XLSX query predicates are allowed", + details=Json::object({ + "limit": Json::number(xlsx_cli_max_query_predicates.to_double()), + }), + ) + } + if !rest.has_prefix("[") { + raise xlsx_query_failure("expected '[' in query selector", selector) + } + let close = match rest.find("]") { + Some(index) => index + None => + raise xlsx_query_failure( + "query selector is missing a closing ']'", selector, + ) + } + predicates.push(parse_xlsx_predicate(rest[1:close].to_owned(), selector)) + rest = rest[close + 1:].to_owned() + } + predicates +} + +///| +fn XlsxNumberOperator::matches( + self : XlsxNumberOperator, + value : Double, + bound : Double, +) -> Bool { + match self { + Greater => value > bound + GreaterOrEqual => value >= bound + Less => value < bound + LessOrEqual => value <= bound + Equal => value == bound + NotEqual => value != bound + } +} + +///| +fn XlsxCellPredicate::matches( + self : XlsxCellPredicate, + cell : XlsxCellSnapshot, +) -> Bool { + match self { + HasType(kind) => + match kind { + Formula => cell.formula is Some(_) + Number => cell.raw is Some(Numeric(_)) + StringValue => cell.raw is Some(String(_)) + Boolean => cell.raw is Some(Bool(_)) + ErrorValue => cell.raw is Some(Error(_)) + } + ValueCompare(operator, bound) => + match cell.raw { + Some(Numeric(value)) => operator.matches(value, bound) + _ => false + } + HasFormula => cell.formula is Some(_) + FormulaContains(needle) => + match cell.formula { + Some(formula) => formula.contains(needle) + None => false + } + TextEquals(expected) => + match cell.raw { + Some(String(value)) => value == expected + _ => false + } + TextContains(needle) => + match cell.raw { + Some(String(value)) => value.contains(needle) + _ => false + } + } +} + +///| +fn xlsx_cell_matches( + cell : XlsxCellSnapshot, + predicates : Array[XlsxCellPredicate], +) -> Bool { + if !cell.is_present() { + return false + } + for predicate in predicates { + if !predicate.matches(cell) { + return false + } + } + true +} diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 66c9be89..56dcc1c5 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -63,6 +63,12 @@ struct XlsxCellSnapshot { style_id : Int } +///| +struct XlsxScanRegion { + sheet : XlsxSheetTarget + rect : XlsxRect +} + ///| struct XlsxStringBudget { maximum : Int @@ -390,6 +396,63 @@ fn resolve_xlsx_selector( } } +///| +fn XlsxResolvedSelector::path(self : XlsxResolvedSelector) -> String { + match self { + Workbook(path~) => path + Sheet(sheet) => sheet.path + Cell(_, _, path~) | Range(_, _, path~) => path + } +} + +///| +fn xlsx_scan_regions( + projection : XlsxProjection, + under : XlsxResolvedSelector?, +) -> Array[XlsxScanRegion] raise CliFailure { + let regions : Array[XlsxScanRegion] = [] + match under { + Some(Cell(sheet, rect, ..)) | Some(Range(sheet, rect, ..)) => + regions.push({ sheet, rect }) + Some(Sheet(sheet)) => + match sheet.content { + Worksheet(worksheet) => + match xlsx_used_rect(worksheet) { + Some(rect) => regions.push({ sheet, rect }) + None => () + } + ChartSheet(_) => () + } + Some(Workbook(..)) | None => + for name in projection.sheet_names { + match projection.find_sheet_by_name(name) { + Some(sheet) => + match sheet.content { + Worksheet(worksheet) => + match xlsx_used_rect(worksheet) { + Some(rect) => regions.push({ sheet, rect }) + None => () + } + ChartSheet(_) => () + } + None => () + } + } + } + let mut total = 0L + for region in regions { + total += region.rect.cell_count() + if total > projection.max_scan_cells.to_int64() { + raise xlsx_scan_resource_failure( + "scanned cells", + projection.max_scan_cells, + total, + ) + } + } + regions +} + ///| fn XlsxStringBudget::new( maximum? : Int = xlsx_cli_max_scanned_string_chars, diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt new file mode 100644 index 00000000..4a33d6d6 --- /dev/null +++ b/office/cmd/office/xlsx_read_output.mbt @@ -0,0 +1,704 @@ +///| +enum XlsxCellFilter { + PresentCells + TextCells + QueryCells(Array[XlsxCellPredicate]) +} + +///| +struct XlsxCellPage { + cells : Array[XlsxCellSnapshot] + matched_total : Int + scanned_cells : Int + offset : Int + limit : Int +} + +///| +fn xlsx_sheet_state_name(state : @xlsx.SheetState) -> String { + match state { + Visible => "visible" + Hidden => "hidden" + VeryHidden => "very-hidden" + } +} + +///| +fn xlsx_conditional_format_range_count( + projection : XlsxProjection, + sheet : String, +) -> Int raise CliFailure { + let formats = xlsx_call(projection.file, () => { + projection.workbook.get_conditional_formats(sheet) + }) + let mut count = 0 + for ranges, _ in formats { + for part in ranges.split(" ") { + if part.length() > 0 { + count += 1 + } + } + } + count +} + +///| +fn xlsx_sheet_summary_json( + projection : XlsxProjection, + sheet : XlsxSheetTarget, + metadata : XlsxMetadataBudget, +) -> Json raise CliFailure { + metadata.charge_item([sheet.name, sheet.path]) + let fields : Map[String, Json] = { + "path": Json::string(sheet.path), + "name": Json::string(sheet.name), + "index": Json::number((sheet.index + 1).to_double()), + } + match sheet.content { + ChartSheet(chart_sheet) => { + fields["kind"] = Json::string("chart-sheet") + fields["state"] = Json::string(xlsx_sheet_state_name(chart_sheet.state())) + fields["counts"] = Json::object({ "charts": Json::number(1) }) + } + Worksheet(worksheet) => { + fields["kind"] = Json::string("worksheet") + fields["state"] = Json::string(xlsx_sheet_state_name(worksheet.state())) + let max_row = worksheet.max_row() + let max_col = worksheet.max_col() + fields["max_row"] = Json::number(max_row.to_double()) + fields["max_column"] = Json::number(max_col.to_double()) + match xlsx_used_rect(worksheet) { + Some(rect) => { + let reference = rect.reference(projection.file) + let path = canonical_xlsx_range_path(sheet.name, reference) + metadata.charge_item([reference, path]) + fields["used_range"] = Json::object({ + "path": Json::string(path), + "reference": Json::string(reference), + "cell_count": Json::number(rect.cell_count().to_double()), + }) + } + None => () + } + let merges = xlsx_call(projection.file, () => { + projection.workbook.get_merge_cells(sheet.name) + }) + let tables = xlsx_call(projection.file, () => { + projection.workbook.get_tables(sheet.name) + }) + let comments = xlsx_call(projection.file, () => { + projection.workbook.get_comments(sheet.name) + }) + let hyperlinks = xlsx_call(projection.file, () => { + projection.workbook.get_hyperlink_cells(sheet.name) + }) + let validations = xlsx_call(projection.file, () => { + projection.workbook.get_data_validations(sheet.name) + }) + let pivots = xlsx_call(projection.file, () => { + projection.workbook.get_pivot_tables(sheet.name) + }) + let slicers = xlsx_call(projection.file, () => { + projection.workbook.get_slicers(sheet.name) + }) + fields["counts"] = Json::object({ + "merges": Json::number(merges.length().to_double()), + "tables": Json::number(tables.length().to_double()), + "charts": Json::number(worksheet.charts().length().to_double()), + "images": Json::number(worksheet.images().length().to_double()), + "pivot_tables": Json::number(pivots.length().to_double()), + "comments": Json::number(comments.length().to_double()), + "hyperlinks": Json::number(hyperlinks.length().to_double()), + "data_validations": Json::number(validations.length().to_double()), + "conditional_format_ranges": Json::number( + xlsx_conditional_format_range_count(projection, sheet.name).to_double(), + ), + "slicers": Json::number(slicers.length().to_double()), + }) + } + } + Json::object(fields) +} + +///| +fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { + let metadata = XlsxMetadataBudget::new() + let sheets : Array[Json] = [] + for name in projection.sheet_names { + match projection.find_sheet_by_name(name) { + Some(sheet) => + sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) + None => + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook tab order contains an unresolved sheet", + details=Json::object({ + "sheet": Json::string(bounded_text(name, 160)), + }), + ) + } + } + let defined_names : Array[Json] = [] + for defined in projection.workbook.get_defined_names() { + metadata.charge_item([ + defined.name, + defined.refers_to, + defined.scope, + defined.comment, + ]) + let fields : Map[String, Json] = { + "name": Json::string(defined.name), + "refers_to": Json::string(defined.refers_to), + } + if defined.scope != "" { + fields["scope"] = Json::string(defined.scope) + } + if defined.comment != "" { + fields["comment"] = Json::string(defined.comment) + } + defined_names.push(Json::object(fields)) + } + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_OUTLINE), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string("/xlsx/workbook"), + "sheet_count": Json::number(sheets.length().to_double()), + "sheets": Json::array(sheets), + "defined_names": Json::array(defined_names), + "limits": Json::object({ + "max_scan_cells": Json::number(projection.max_scan_cells.to_double()), + "max_metadata_items": Json::number(metadata.maximum_items.to_double()), + "max_metadata_string_chars": Json::number( + metadata.maximum_string_chars.to_double(), + ), + }), + } + if !projection.sheet_names.is_empty() { + let active_index = projection.workbook.get_active_sheet_index() + match projection.sheet_names.get(active_index) { + Some(name) => + match projection.find_sheet_by_name(name) { + Some(sheet) => + fields["active_sheet"] = Json::object({ + "path": Json::string(sheet.path), + "name": Json::string(sheet.name), + "index": Json::number((sheet.index + 1).to_double()), + }) + None => () + } + None => () + } + } + Json::object(fields) +} + +///| +fn xlsx_outline_payload( + projection : XlsxProjection, + max_output_chars : Int, +) -> BoundedOfficePayload raise CliFailure { + bounded_office_payload( + xlsx_outline_data(projection), + [], + max_output_chars, + "xlsx", + ) +} + +///| +fn xlsx_raw_value_json(value : @xlsx.CellValue) -> Json { + match value { + String(text) => + Json::object({ + "type": Json::string("string"), + "value": Json::string(text), + }) + Numeric(number) => + Json::object({ + "type": Json::string("number"), + "value": Json::number(number), + }) + Bool(flag) => + Json::object({ + "type": Json::string("bool"), + "value": Json::boolean(flag), + }) + Error(code) => + Json::object({ + "type": Json::string("error"), + "value": Json::string(code), + }) + } +} + +///| +fn xlsx_cell_json(cell : XlsxCellSnapshot) -> Json { + let fields : Map[String, Json] = { + "path": Json::string(cell.path), + "reference": Json::string(cell.reference), + "row": Json::number(cell.row.to_double()), + "column": Json::number(cell.column.to_double()), + } + match cell.formatted { + Some(value) => fields["value"] = Json::string(value) + None => () + } + match cell.raw { + Some(value) => fields["raw"] = xlsx_raw_value_json(value) + None => () + } + match cell.formula { + Some(value) => fields["formula"] = Json::string(value) + None => () + } + if cell.style_id != 0 { + fields["style_id"] = Json::number(cell.style_id.to_double()) + } + Json::object(fields) +} + +///| +fn xlsx_styles_json( + projection : XlsxProjection, + cells : Array[XlsxCellSnapshot], +) -> Json raise CliFailure { + let fields : Map[String, Json] = Map([]) + for cell in cells { + if cell.style_id != 0 { + let key = cell.style_id.to_string() + if !fields.contains(key) { + let style = xlsx_call(projection.file, () => { + projection.workbook.get_style(cell.style_id) + }) + fields[key] = @inspect.style_to_json(style) + } + } + } + Json::object(fields) +} + +///| +fn xlsx_cell_text(cell : XlsxCellSnapshot) -> String? { + match cell.formatted { + Some(value) if value != "" => Some(value) + _ => + match cell.formula { + Some(formula) => Some("=" + formula) + None => None + } + } +} + +///| +fn XlsxCellFilter::matches( + self : XlsxCellFilter, + cell : XlsxCellSnapshot, +) -> Bool { + match self { + PresentCells => cell.is_present() + TextCells => xlsx_cell_text(cell) is Some(_) + QueryCells(predicates) => xlsx_cell_matches(cell, predicates) + } +} + +///| +fn collect_xlsx_cell_page( + projection : XlsxProjection, + regions : Array[XlsxScanRegion], + filter : XlsxCellFilter, + offset : Int, + limit : Int, +) -> XlsxCellPage raise CliFailure { + let scan = XlsxScanBudget::new(projection.max_scan_cells) + let cells : Array[XlsxCellSnapshot] = [] + let mut matched = 0 + for region in regions { + for row in region.rect.row_lo..<=region.rect.row_hi { + for column in region.rect.col_lo..<=region.rect.col_hi { + let cell = xlsx_cell_snapshot( + projection, + region.sheet, + row, + column, + scan, + ) + if filter.matches(cell) { + if matched >= offset && cells.length() < limit { + cells.push(cell) + } + matched += 1 + } + } + } + } + { cells, matched_total: matched, scanned_cells: scan.scanned, offset, limit } +} + +///| +fn xlsx_page_fields(page : XlsxCellPage) -> Map[String, Json] { + { + "matched_total": Json::number(page.matched_total.to_double()), + "offset": Json::number(page.offset.to_double()), + "limit": Json::number(page.limit.to_double()), + "returned": Json::number(page.cells.length().to_double()), + "truncated": Json::boolean( + page.offset + page.cells.length() < page.matched_total, + ), + "scanned_cells": Json::number(page.scanned_cells.to_double()), + } +} + +///| +fn xlsx_get_data( + projection : XlsxProjection, + resolved : XlsxResolvedSelector, +) -> Json raise CliFailure { + match resolved { + Workbook(path~) => { + let outline = xlsx_outline_data(projection) + guard outline is Object(outline_fields) else { abort("outline object") } + Json::object({ + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(path), + "kind": Json::string("workbook"), + "stability": Json::string("stable"), + "sheet_count": outline_fields + .get("sheet_count") + .unwrap_or(Json::number(0)), + "sheets": outline_fields.get("sheets").unwrap_or(Json::array([])), + "defined_names": outline_fields + .get("defined_names") + .unwrap_or(Json::array([])), + }) + } + Sheet(sheet) => + Json::object({ + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(sheet.path), + "kind": Json::string("sheet"), + "stability": Json::string("stable"), + "sheet": xlsx_sheet_summary_json( + projection, + sheet, + XlsxMetadataBudget::new(), + ), + }) + Cell(sheet, rect, path~) => { + let page = collect_xlsx_cell_page( + projection, + [{ sheet, rect }], + PresentCells, + 0, + 1, + ) + guard page.cells.get(0) is Some(cell) else { + raise xlsx_cli_failure( + "office.xlsx.selector_not_found", + "XLSX cell selector resolved to a blank, unstyled coordinate", + details=Json::object({ "selector": Json::string(path) }), + ) + } + Json::object({ + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(path), + "kind": Json::string("cell"), + "stability": Json::string("snapshot-relative"), + "parent": Json::string(sheet.path), + "cell": xlsx_cell_json(cell), + "styles": xlsx_styles_json(projection, [cell]), + "scanned_cells": Json::number(page.scanned_cells.to_double()), + }) + } + Range(sheet, rect, path~) => { + let page = collect_xlsx_cell_page( + projection, + [{ sheet, rect }], + PresentCells, + 0, + projection.max_scan_cells, + ) + Json::object({ + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(path), + "kind": Json::string("range"), + "stability": Json::string("snapshot-relative"), + "parent": Json::string(sheet.path), + "reference": Json::string(rect.reference(projection.file)), + "cells": Json::array(page.cells.map(xlsx_cell_json)), + "styles": xlsx_styles_json(projection, page.cells), + "scanned_cells": Json::number(page.scanned_cells.to_double()), + "returned": Json::number(page.cells.length().to_double()), + }) + } + } +} + +///| +fn xlsx_get_payload( + projection : XlsxProjection, + resolved : XlsxResolvedSelector, + max_output_chars : Int, +) -> BoundedOfficePayload raise CliFailure { + bounded_office_payload( + xlsx_get_data(projection, resolved), + [], + max_output_chars, + "xlsx", + ) +} + +///| +fn xlsx_text_data( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + offset : Int, + limit : Int, +) -> Json raise CliFailure { + let page = collect_xlsx_cell_page( + projection, + xlsx_scan_regions(projection, under), + TextCells, + offset, + limit, + ) + let entries : Array[Json] = [] + for cell in page.cells { + guard xlsx_cell_text(cell) is Some(text) else { continue } + entries.push( + Json::object({ + "path": Json::string(cell.path), + "stability": Json::string("snapshot-relative"), + "text": Json::string(text), + }), + ) + } + let fields = xlsx_page_fields(page) + fields["schema"] = Json::string(@lib.SCHEMA_XLSX_TEXT) + fields["file"] = Json::string(projection.file) + fields["format"] = Json::string("xlsx") + fields["entries"] = Json::array(entries) + match under { + Some(value) => fields["under"] = Json::string(value.path()) + None => () + } + Json::object(fields) +} + +///| +fn xlsx_text_payload( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + offset : Int, + limit : Int, + max_output_chars : Int, +) -> BoundedOfficePayload raise CliFailure { + bounded_office_payload( + xlsx_text_data(projection, under, offset, limit), + [], + max_output_chars, + "xlsx", + ) +} + +///| +fn xlsx_query_data( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + spec : XlsxQuerySpec, +) -> Json raise CliFailure { + let page = collect_xlsx_cell_page( + projection, + xlsx_scan_regions(projection, under), + QueryCells(spec.predicates), + spec.offset, + spec.limit, + ) + let fields = xlsx_page_fields(page) + fields["schema"] = Json::string(@lib.SCHEMA_XLSX_QUERY) + fields["file"] = Json::string(projection.file) + fields["format"] = Json::string("xlsx") + fields["selector"] = Json::string(spec.selector) + fields["matches"] = Json::array(page.cells.map(xlsx_cell_json)) + fields["styles"] = xlsx_styles_json(projection, page.cells) + match under { + Some(value) => fields["under"] = Json::string(value.path()) + None => () + } + Json::object(fields) +} + +///| +fn xlsx_query_payload( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + spec : XlsxQuerySpec, + max_output_chars : Int, +) -> BoundedOfficePayload raise CliFailure { + bounded_office_payload( + xlsx_query_data(projection, under, spec), + [], + max_output_chars, + "xlsx", + ) +} + +///| +fn xlsx_outline_human( + projection : XlsxProjection, + maximum : Int, +) -> String raise CliFailure { + let output = DocxHumanWriter::new(maximum, format="xlsx") + output.begin_line() + output.write_plain("xlsx: ") + output.write_terminal_safe(projection.file) + output.begin_line() + output.write_plain("workbook\t/xlsx/workbook\t") + output.write_plain("\{projection.sheet_names.length()} sheets") + for name in projection.sheet_names { + guard projection.find_sheet_by_name(name) is Some(sheet) else { continue } + output.begin_line() + output.write_terminal_safe(sheet.path) + output.write_char('\t') + output.write_terminal_safe(sheet.name, single_line=true) + match sheet.content { + ChartSheet(_) => output.write_plain("\tchart-sheet") + Worksheet(worksheet) => { + output.write_plain("\tworksheet") + match xlsx_used_rect(worksheet) { + Some(rect) => { + output.write_char('\t') + output.write_terminal_safe(rect.reference(projection.file)) + } + None => output.write_plain("\tempty") + } + } + } + } + output.finish() +} + +///| +fn xlsx_page_human( + page : XlsxCellPage, + include_kind : Bool, + maximum : Int, +) -> String raise CliFailure { + let output = DocxHumanWriter::new(maximum, format="xlsx") + for cell in page.cells { + output.begin_line() + output.write_terminal_safe(cell.path) + if include_kind { + output.write_plain("\tcell") + } + match xlsx_cell_text(cell) { + Some(text) => { + output.write_char('\t') + output.write_terminal_safe(text, single_line=true) + } + None => () + } + } + output.begin_line() + output.write_plain( + "# matched \{page.matched_total}; returned \{page.cells.length()}; scanned \{page.scanned_cells}; offset \{page.offset}", + ) + output.finish() +} + +///| +fn xlsx_get_human( + projection : XlsxProjection, + resolved : XlsxResolvedSelector, + maximum : Int, +) -> String raise CliFailure { + match resolved { + Workbook(..) => xlsx_outline_human(projection, maximum) + Sheet(sheet) => { + let output = DocxHumanWriter::new(maximum, format="xlsx") + output.begin_line() + output.write_terminal_safe(sheet.path) + output.write_char('\t') + output.write_terminal_safe(sheet.name, single_line=true) + output.finish() + } + Cell(sheet, rect, path~) => { + let page = collect_xlsx_cell_page( + projection, + [{ sheet, rect }], + PresentCells, + 0, + 1, + ) + if page.cells.is_empty() { + raise xlsx_cli_failure( + "office.xlsx.selector_not_found", + "XLSX cell selector resolved to a blank, unstyled coordinate", + details=Json::object({ "selector": Json::string(path) }), + ) + } + xlsx_page_human(page, false, maximum) + } + Range(sheet, rect, ..) => + xlsx_page_human( + collect_xlsx_cell_page( + projection, + [{ sheet, rect }], + PresentCells, + 0, + projection.max_scan_cells, + ), + false, + maximum, + ) + } +} + +///| +fn xlsx_text_human( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + offset : Int, + limit : Int, + maximum : Int, +) -> String raise CliFailure { + xlsx_page_human( + collect_xlsx_cell_page( + projection, + xlsx_scan_regions(projection, under), + TextCells, + offset, + limit, + ), + false, + maximum, + ) +} + +///| +fn xlsx_query_human( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + spec : XlsxQuerySpec, + maximum : Int, +) -> String raise CliFailure { + xlsx_page_human( + collect_xlsx_cell_page( + projection, + xlsx_scan_regions(projection, under), + QueryCells(spec.predicates), + spec.offset, + spec.limit, + ), + true, + maximum, + ) +} diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 0a2cbc4f..2c1de5fa 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -5,6 +5,7 @@ fn xlsx_read_test_projection() -> XlsxProjection raise { ignore(workbook.new_sheet("财务 \"Q1\"")) workbook.set_cell_str("Data", "A1", "alpha") workbook.set_cell_formula("Data", "B2", "SUM(1,2)", value="3") + workbook.set_cell_formula("Data", "C3", "NOW()") make_xlsx_projection("fixture.xlsx", workbook, 100) } @@ -113,3 +114,141 @@ test "XLSX string and metadata accounting fail before unbounded retention" { let metadata_limit = xlsx_cli_error(() => metadata.charge_item([])) assert_eq(metadata_limit.code, "office.xlsx.resource_limit") } + +///| +test "XLSX query selectors are bounded, deterministic predicate conjunctions" { + let projection = xlsx_read_test_projection() + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[B2]") + is Cell(sheet, rect, ..) else { + fail("expected cell selector") + } + let snapshot = xlsx_cell_snapshot( + projection, + sheet, + rect.row_lo, + rect.col_lo, + XlsxScanBudget::new(1), + ) + let predicates = parse_xlsx_query_selector( + "cell[type=formula][formula~=SUM][text=3]", + ) + assert_eq(predicates.length(), 3) + assert_true(xlsx_cell_matches(snapshot, predicates)) + assert_false( + xlsx_cell_matches(snapshot, parse_xlsx_query_selector("cell[value>3]")), + ) + let malformed = xlsx_cli_error(() => { + ignore(parse_xlsx_query_selector("row[type=number]")) + }) + assert_eq(malformed.code, "office.xlsx.invalid_query") + let too_many = "cell" + "[formula]".repeat(17) + let predicate_limit = xlsx_cli_error(() => { + ignore(parse_xlsx_query_selector(too_many)) + }) + assert_eq(predicate_limit.code, "office.xlsx.query_predicate_limit") +} + +///| +test "XLSX outline and get use office schemas with canonical cell records" { + let projection = xlsx_read_test_projection() + let outline = xlsx_outline_data(projection) + guard outline is Object(outline_fields) else { + fail("expected outline object") + } + assert_eq( + outline_fields.get("schema"), + Some(Json::string(@lib.SCHEMA_XLSX_OUTLINE)), + ) + assert_eq(outline_fields.get("sheet_count"), Some(Json::number(2))) + guard outline_fields.get("sheets") is Some(Array(sheets)) else { + fail("expected sheets") + } + assert_eq(sheets.length(), 2) + let resolved = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/range[A1:C3]", + ) + let data = xlsx_get_data(projection, resolved) + guard data is Object(fields) && fields.get("cells") is Some(Array(cells)) else { + fail("expected range cells") + } + assert_eq(fields.get("schema"), Some(Json::string(@lib.SCHEMA_XLSX_ELEMENT))) + assert_eq(cells.length(), 3) + assert_true(cells[0].stringify().contains("alpha")) + assert_true(cells[1].stringify().contains("SUM(1,2)")) + assert_true(cells[2].stringify().contains("NOW()")) + let missing = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/cell[D4]", + ) + let missing_error = xlsx_cli_error(() => { + ignore(xlsx_get_data(projection, missing)) + }) + assert_eq(missing_error.code, "office.xlsx.selector_not_found") +} + +///| +test "XLSX text and query scan fully while retaining only the requested page" { + let projection = xlsx_read_test_projection() + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), + ) + let text = xlsx_text_data(projection, under, 1, 1) + guard text is Object(text_fields) && + text_fields.get("entries") is Some(Array(entries)) else { + fail("expected text entries") + } + assert_eq(text_fields.get("matched_total"), Some(Json::number(3))) + assert_eq(text_fields.get("returned"), Some(Json::number(1))) + assert_eq(text_fields.get("truncated"), Some(Json::boolean(true))) + assert_eq(text_fields.get("scanned_cells"), Some(Json::number(9))) + assert_eq(entries.length(), 1) + assert_true(entries[0].stringify().contains("B2")) + let formula_query : XlsxQuerySpec = { + selector: "cell[formula]", + predicates: parse_xlsx_query_selector("cell[formula]"), + offset: 0, + limit: 10, + } + let query = xlsx_query_data(projection, under, formula_query) + guard query is Object(query_fields) && + query_fields.get("matches") is Some(Array(matches)) else { + fail("expected query matches") + } + assert_eq(query_fields.get("matched_total"), Some(Json::number(2))) + assert_eq(matches.length(), 2) + assert_true(matches[1].stringify().contains("NOW()")) +} + +///| +test "XLSX JSON and human output enforce format-specific stdout ceilings" { + let projection = xlsx_read_test_projection() + let payload = xlsx_outline_payload(projection, 100_000) + let json = checked_office_json_output(payload) + assert_true(json.contains("office.output/1")) + assert_true(json.contains("office.xlsx.outline/1")) + let too_small = xlsx_cli_error(() => { + ignore(xlsx_outline_payload(projection, 8)) + }) + assert_eq(too_small.code, "office.xlsx.resource_limit") + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), + ) + let human = xlsx_text_human(projection, under, 0, 10, 10_000) + assert_true(human.contains("/cell[A1]\talpha")) + assert_true(human.contains("/cell[C3]\t=NOW()")) +} + +///| +test "XLSX all-sheet scans reject sparse far extents before visiting cells" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Sparse")) + workbook.set_cell_str("Sparse", "K10", "far") + let projection = make_xlsx_projection("sparse.xlsx", workbook, 100) + let error = xlsx_cli_error(() => { + ignore(xlsx_text_data(projection, None, 0, 10)) + }) + assert_eq(error.code, "office.xlsx.resource_limit") + guard error.details is Some(Object(fields)) else { + fail("expected sparse-scan details") + } + assert_eq(fields.get("actual"), Some(Json::number(110))) +} diff --git a/office/pkg.generated.mbti b/office/pkg.generated.mbti index 30421d6f..21bfa4b9 100644 --- a/office/pkg.generated.mbti +++ b/office/pkg.generated.mbti @@ -29,6 +29,14 @@ pub const SCHEMA_RAW_RESULT : String = "office.raw.result/1" pub const SCHEMA_TRANSACTION : String = "office.transaction/1" +pub const SCHEMA_XLSX_ELEMENT : String = "office.xlsx.element/1" + +pub const SCHEMA_XLSX_OUTLINE : String = "office.xlsx.outline/1" + +pub const SCHEMA_XLSX_QUERY : String = "office.xlsx.query/1" + +pub const SCHEMA_XLSX_TEXT : String = "office.xlsx.text/1" + pub const SELECTOR_MAX_DEPTH : Int = 32 pub const SELECTOR_MAX_LENGTH : Int = 2048 From 807d00fd840a9ae1a19413e97627cbd51ff65554 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:36:11 +0800 Subject: [PATCH 005/150] feat(office): dispatch bounded structured reads to xlsx --- office/capabilities.mbt | 254 +++++++++++---- office/capabilities_test.mbt | 70 +++-- office/cmd/office/docx_commands.mbt | 410 ++++++++++++++++++------- office/cmd/office/docx_projection.mbt | 44 ++- office/cmd/office/docx_read_output.mbt | 130 +------- office/cmd/office/docx_read_wbtest.mbt | 16 +- office/cmd/office/pkg.generated.mbti | 10 +- office/cmd/office/query_work.mbt | 112 +++++++ office/cmd/office/read_package.mbt | 10 +- office/cmd/office/xlsx_query.mbt | 91 +++++- office/cmd/office/xlsx_read_model.mbt | 33 +- office/cmd/office/xlsx_read_output.mbt | 76 +++-- office/cmd/office/xlsx_read_wbtest.mbt | 35 ++- 13 files changed, 872 insertions(+), 419 deletions(-) create mode 100644 office/cmd/office/query_work.mbt diff --git a/office/capabilities.mbt b/office/capabilities.mbt index 8c4866be..57d4c120 100644 --- a/office/capabilities.mbt +++ b/office/capabilities.mbt @@ -129,6 +129,25 @@ fn capability_action( { name, requires, forbids, restrictions } } +///| +fn capability_variant( + name : String, + usage : String, + result_schema : String, + constraints : Array[String], +) -> CapabilityVariant { + { + name, + usage, + result_schema, + inputs: [], + outputs: [], + constraints, + actions: [], + output_modes: ["human", "json"], + } +} + ///| /// Returns the canonical document-format declarations in stable order. pub fn capability_formats() -> Array[CapabilityFormat] { @@ -152,11 +171,11 @@ pub fn capability_formats() -> Array[CapabilityFormat] { selector: { schema: "office.selector/1", root: "/xlsx", - status: "syntax-only", + status: "read-resolved", examples: [ "/xlsx/sheet[name=\"Data\"]/cell[A1]", "/xlsx/sheet[name=\"Data\"]/range[A1:C12]", ], - description: "bounded parse/render only; workbook resolution is not implemented", + description: "bounded canonical resolution for workbook, sheet, cell, and range selectors across outline, get, text, and cell queries", }, }, ] @@ -222,14 +241,14 @@ pub fn capability_commands() -> Array[CapabilityCommand] { }, { name: "outline", - summary: "Summarize bounded DOCX structure using canonical selectors", + summary: "Summarize bounded XLSX or DOCX structure using canonical selectors", usage: "office outline FILE [--max-elements N] [--max-output-chars N] [--json]", - formats: ["docx"], + formats: ["docx", "xlsx"], aliases: [], inputs: [ - capability_field("file", "path", true, "existing DOCX package"), + capability_field("file", "path", true, "existing XLSX or DOCX package"), capability_field( - "max-elements", "integer(1..200000)", false, "projection scan ceiling; default 50000", + "max-elements", "integer(1..200000)", false, "DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000", ), capability_field( "max-output-chars", "integer(1..4194304)", false, "successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576", @@ -238,55 +257,76 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ], outputs: [ capability_field( - "schema", "literal(office.docx.outline/1)", true, "versioned result schema", + "schema", "enum(office.docx.outline/1|office.xlsx.outline/1)", true, "format-specific versioned result schema", ), capability_field( "file", "path", true, "the input path exactly as supplied", ), capability_field( - "format", "literal(docx)", true, "resolved document format", + "format", "enum(docx|xlsx)", true, "resolved document format", + ), + capability_field( + "scanned_elements", "integer", false, "DOCX only: number of elements in the bounded projection", ), capability_field( - "scanned_elements", "integer", true, "number of elements in the bounded projection", + "counts", "object", false, "DOCX only: deterministic structural counts across every story", ), + capability_field("stories", "array", false, "DOCX only: story roots"), capability_field( - "counts", "object", true, "deterministic structural counts across every DOCX story", + "headings", "array", false, "DOCX only: bounded heading previews", ), capability_field( - "stories", "array", true, "body, header, footer, note, endnote, and comment story roots", + "styles_in_use", "array", false, "DOCX only: deduplicated referenced styles", ), + capability_field("images", "array", false, "DOCX only: image metadata"), capability_field( - "headings", "array", true, "bounded heading previews with canonical paths", + "sections", "array", false, "DOCX only: section boundaries and header/footer references", ), capability_field( - "styles_in_use", "array", true, "deduplicated styles referenced by projected elements", + "diagnostics", "array", false, "DOCX only: reader and source diagnostics", ), capability_field( - "images", "array", true, "image metadata with canonical paths and byte counts", + "path", "literal(/xlsx/workbook)", false, "XLSX only: canonical workbook selector", ), capability_field( - "sections", "array", true, "section boundaries and effective header/footer references", + "sheet_count", "integer", false, "XLSX only: workbook tab count", ), capability_field( - "diagnostics", "array", true, "reader and annotation source diagnostics", + "active_sheet", "object", false, "XLSX only: canonical active-sheet summary when the workbook has tabs", + ), + capability_field( + "sheets", "array", false, "XLSX only: bounded tab-order sheet summaries with canonical paths and used ranges", + ), + capability_field( + "defined_names", "array", false, "XLSX only: bounded workbook defined-name inventory", + ), + capability_field( + "limits", "object", false, "XLSX only: effective scan and metadata limits", ), ], output_modes: ["human", "json"], - variants: [], + variants: [ + capability_variant("docx", "office outline FILE", SCHEMA_DOCX_OUTLINE, [ + "format=docx", + ]), + capability_variant("xlsx", "office outline FILE", SCHEMA_XLSX_OUTLINE, [ + "format=xlsx", + ]), + ], }, { name: "get", - summary: "Resolve one canonical DOCX selector", + summary: "Resolve one canonical XLSX or DOCX selector", usage: "office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]", - formats: ["docx"], + formats: ["docx", "xlsx"], aliases: [], inputs: [ - capability_field("file", "path", true, "existing DOCX package"), + capability_field("file", "path", true, "existing XLSX or DOCX package"), capability_field( - "selector", "office.selector/1", true, "canonical /docx selector", + "selector", "office.selector/1", true, "canonical selector matching the validated package format", ), capability_field( - "max-elements", "integer(1..200000)", false, "projection scan ceiling; default 50000", + "max-elements", "integer(1..200000)", false, "DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000", ), capability_field( "max-output-chars", "integer(1..4194304)", false, "successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576", @@ -295,29 +335,29 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ], outputs: [ capability_field( - "schema", "literal(office.docx.element/1)", true, "versioned result schema", + "schema", "enum(office.docx.element/1|office.xlsx.element/1)", true, "format-specific versioned result schema", ), capability_field( "file", "path", true, "the input path exactly as supplied", ), capability_field( - "format", "literal(docx)", true, "resolved document format", + "format", "enum(docx|xlsx)", true, "resolved document format", ), capability_field( "path", "office.selector/1", true, "resolved canonical path", ), capability_field( - "kind", "string", true, "resolved story, annotation, or element kind", + "kind", "string", true, "resolved workbook, sheet, coordinate, story, annotation, or element kind", ), capability_field( "role", "enum(story-root|annotation-collection|annotation-item|element)", - true, "projection role", + false, "DOCX only: projection role", ), capability_field( "stability", "enum(stable|snapshot-relative)", true, "selector stability classification", ), capability_field( - "source", "object", true, "physical story source metadata", + "source", "object", false, "DOCX only: physical story source metadata", ), capability_field( "parent", "office.selector/1", false, "canonical parent path; absent for projection roots", @@ -326,38 +366,83 @@ pub fn capability_commands() -> Array[CapabilityCommand] { "id", "string", false, "annotation id when the resolved item carries one", ), capability_field( - "children", "array", true, "addressable direct children with canonical paths", + "children", "array", false, "DOCX only: addressable direct children", + ), + capability_field( + "properties", "object", false, "DOCX only: declared formatting and element summary", + ), + capability_field( + "metadata", "object", false, "DOCX only: role-specific metadata", + ), + capability_field( + "text", "string", false, "DOCX only: bounded raw text projection", + ), + capability_field( + "sheet_count", "integer", false, "XLSX workbook selectors only: workbook tab count", + ), + capability_field( + "sheets", "array", false, "XLSX workbook selectors only: bounded tab-order sheet summaries", + ), + capability_field( + "defined_names", "array", false, "XLSX workbook selectors only: bounded defined-name inventory", + ), + capability_field( + "sheet", "object", false, "XLSX sheet selectors only: bounded sheet summary", ), capability_field( - "properties", "object", true, "declared formatting and element summary", + "cell", "object", false, "XLSX cell selectors only: typed value, formula, style, and canonical path", ), capability_field( - "metadata", "object", true, "role-specific annotation or collection metadata", + "cells", "array", false, "XLSX range selectors only: populated cells in row-major order", + ), + capability_field( + "reference", "a1-range", false, "XLSX range selectors only: normalized A1 rectangle", + ), + capability_field( + "styles", "object", false, "XLSX coordinate selectors only: deduplicated referenced style definitions", + ), + capability_field( + "scanned_cells", "integer", false, "XLSX coordinate selectors only: exact rectangle scan count", + ), + capability_field( + "returned", "integer", false, "XLSX range selectors only: populated cell count", ), - capability_field("text", "string", true, "bounded raw text projection"), ], output_modes: ["human", "json"], - variants: [], + variants: [ + capability_variant( + "docx", + "office get FILE /docx/...", + SCHEMA_DOCX_ELEMENT, + ["format=docx"], + ), + capability_variant( + "xlsx", + "office get FILE /xlsx/...", + SCHEMA_XLSX_ELEMENT, + ["format=xlsx"], + ), + ], }, { name: "text", - summary: "Extract bounded path-tagged DOCX paragraph text", + summary: "Extract bounded path-tagged XLSX cell or DOCX paragraph text", usage: "office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]", - formats: ["docx"], + formats: ["docx", "xlsx"], aliases: [], inputs: [ - capability_field("file", "path", true, "existing DOCX package"), + capability_field("file", "path", true, "existing XLSX or DOCX package"), capability_field( - "under", "office.selector/1", false, "optional canonical subtree root", + "under", "office.selector/1", false, "optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope", ), capability_field( - "offset", "integer(0..200000)", false, "zero-based paragraph offset; default 0", + "offset", "integer(0..200000)", false, "zero-based matching paragraph or cell offset; default 0", ), capability_field( - "limit", "integer(0..10000)", false, "maximum returned paragraphs; default 2000", + "limit", "integer(0..10000)", false, "maximum returned paragraphs or cells; default 2000", ), capability_field( - "max-elements", "integer(1..200000)", false, "projection scan ceiling; default 50000", + "max-elements", "integer(1..200000)", false, "DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000", ), capability_field( "max-output-chars", "integer(1..4194304)", false, "successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576", @@ -366,19 +451,19 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ], outputs: [ capability_field( - "schema", "literal(office.docx.text/1)", true, "versioned result schema", + "schema", "enum(office.docx.text/1|office.xlsx.text/1)", true, "format-specific versioned result schema", ), capability_field( "file", "path", true, "the input path exactly as supplied", ), capability_field( - "format", "literal(docx)", true, "resolved document format", + "format", "enum(docx|xlsx)", true, "resolved document format", ), capability_field( - "entries", "array(object{path,text,stability})", true, "paragraphs in deterministic document order", + "entries", "array(object{path,text,stability})", true, "paragraphs in document order or cells in sheet/row-major order", ), capability_field( - "matched_total", "integer", true, "complete bounded-scan paragraph count", + "matched_total", "integer", true, "complete bounded-scan matching paragraph or cell count", ), capability_field( "returned", "integer", true, "number of returned entries", @@ -391,41 +476,60 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ), capability_field("limit", "integer", true, "applied page-size ceiling"), capability_field( - "scanned_elements", "integer", true, "number of elements in the bounded projection", + "scanned_elements", "integer", false, "DOCX only: projected element count", + ), + capability_field( + "scanned_cells", "integer", false, "XLSX only: exact bounded cell scan count", ), capability_field( - "under", "office.selector/1", false, "resolved subtree root when one was requested", + "under", "office.selector/1", false, "resolved canonical scope when one was requested", ), ], output_modes: ["human", "json"], - variants: [], + variants: [ + capability_variant( + "docx", + "office text FILE [--under /docx/...]", + SCHEMA_DOCX_TEXT, + ["format=docx"], + ), + capability_variant( + "xlsx", + "office text FILE [--under /xlsx/...]", + SCHEMA_XLSX_TEXT, + ["format=xlsx"], + ), + ], }, { name: "query", - summary: "Run bounded deterministic predicates over DOCX elements", - usage: "office query FILE [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]", - formats: ["docx"], + summary: "Run bounded deterministic predicates over XLSX cells or DOCX elements", + usage: "office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]", + formats: ["docx", "xlsx"], aliases: [], inputs: [ - capability_field("file", "path", true, "existing DOCX package"), + capability_field("file", "path", true, "existing XLSX or DOCX package"), capability_field( - "under", "office.selector/1", false, "optional canonical subtree root", + "selector", "xlsx-cell-selector", false, "XLSX only: cell followed by up to 16 bounded type, value, formula, or text predicates; default cell", + ), + capability_field( + "under", "office.selector/1", false, "optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope", ), capability_field( "kind", "enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)", - false, "exact kind; documented long-name aliases normalize before matching", + false, "DOCX only: exact kind; documented aliases normalize before matching", ), capability_field( - "text", "literal-string(1..1048576 chars)", false, "literal substring predicate; regular expressions are not accepted", + "text", "literal-string(1..1048576 chars)", false, "DOCX only: literal substring predicate; regular expressions are not accepted", ), capability_field( - "id", "string(1..1048576 chars)", false, "exact annotation id predicate, including ids outside selector path limits", + "id", "string(1..1048576 chars)", false, "DOCX only: exact annotation id predicate", ), capability_field( - "property", "array(NAME=VALUE)", false, "up to 16 exact predicates with values capped at 1048576 characters", + "property", "array(NAME=VALUE)", false, "DOCX only: up to 16 exact declared-property predicates", ), capability_field( - "ignore-case", "boolean", false, "locale-independent Unicode simple-case --text matching", + "ignore-case", "boolean", false, "DOCX only: locale-independent Unicode simple-case --text matching", ), capability_field( "offset", "integer(0..200000)", false, "zero-based match offset; default 0", @@ -434,7 +538,7 @@ pub fn capability_commands() -> Array[CapabilityCommand] { "limit", "integer(0..1000)", false, "maximum returned matches; default 100", ), capability_field( - "max-elements", "integer(1..200000)", false, "projection scan ceiling; default 50000", + "max-elements", "integer(1..200000)", false, "DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000", ), capability_field( "max-output-chars", "integer(1..4194304)", false, "successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576", @@ -443,19 +547,19 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ], outputs: [ capability_field( - "schema", "literal(office.docx.query/1)", true, "versioned result schema", + "schema", "enum(office.docx.query/1|office.xlsx.query/1)", true, "format-specific versioned result schema", ), capability_field( "file", "path", true, "the input path exactly as supplied", ), capability_field( - "format", "literal(docx)", true, "resolved document format", + "format", "enum(docx|xlsx)", true, "resolved document format", ), capability_field( - "filters", "object", true, "normalized predicates applied by this query", + "filters", "object", false, "DOCX only: normalized predicates applied by this query", ), capability_field( - "matches", "array", true, "deterministic document-order match records", + "matches", "array", true, "deterministic document-order or sheet/row-major match records", ), capability_field( "matched_total", "integer", true, "complete bounded-scan match count", @@ -471,14 +575,36 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ), capability_field("limit", "integer", true, "applied page-size ceiling"), capability_field( - "scanned_elements", "integer", true, "number of elements in the bounded projection", + "scanned_elements", "integer", false, "DOCX only: projected element count", + ), + capability_field( + "selector", "xlsx-cell-selector", false, "XLSX only: supplied bounded cell selector", ), capability_field( - "under", "office.selector/1", false, "resolved subtree root when one was requested", + "styles", "object", false, "XLSX only: deduplicated styles referenced by returned matches", + ), + capability_field( + "scanned_cells", "integer", false, "XLSX only: exact bounded cell scan count", + ), + capability_field( + "under", "office.selector/1", false, "resolved canonical scope when one was requested", ), ], output_modes: ["human", "json"], - variants: [], + variants: [ + capability_variant( + "docx", + "office query FILE [DOCX predicates]", + SCHEMA_DOCX_QUERY, + ["format=docx"], + ), + capability_variant( + "xlsx", + "office query FILE [cell[predicate]...]", + SCHEMA_XLSX_QUERY, + ["format=xlsx"], + ), + ], }, { name: "raw", diff --git a/office/capabilities_test.mbt b/office/capabilities_test.mbt index 35250443..cb2f93b9 100644 --- a/office/capabilities_test.mbt +++ b/office/capabilities_test.mbt @@ -144,7 +144,7 @@ test "raw capability variants describe every accepted option and conflict" { } ///| -test "capability formats distinguish resolved DOCX reads from XLSX syntax" { +test "capability formats advertise resolved structured reads for DOCX and XLSX" { let formats = @office.capability_formats() assert_eq(formats.length(), 2) assert_eq(formats[0].selector.schema, "office.selector/1") @@ -152,32 +152,39 @@ test "capability formats distinguish resolved DOCX reads from XLSX syntax" { assert_eq(formats[1].selector.root, "/xlsx") assert_eq(formats[0].selector.status, "read-resolved") assert_true(formats[0].selector.description.contains("bounded canonical")) - assert_eq(formats[1].selector.status, "syntax-only") - assert_true(formats[1].selector.description.contains("not implemented")) + assert_eq(formats[1].selector.status, "read-resolved") + assert_true(formats[1].selector.description.contains("bounded canonical")) assert_true(formats[0].selector.examples.length() >= 2) assert_true(formats[1].selector.examples.length() >= 2) } ///| -test "DOCX read capabilities publish every enforced bound and result schema" { +test "structured read capabilities publish both format schemas and enforced bounds" { for case in [ - ("outline", @office.SCHEMA_DOCX_OUTLINE), - ("get", @office.SCHEMA_DOCX_ELEMENT), - ("text", @office.SCHEMA_DOCX_TEXT), - ("query", @office.SCHEMA_DOCX_QUERY), + ("outline", @office.SCHEMA_DOCX_OUTLINE, @office.SCHEMA_XLSX_OUTLINE), + ("get", @office.SCHEMA_DOCX_ELEMENT, @office.SCHEMA_XLSX_ELEMENT), + ("text", @office.SCHEMA_DOCX_TEXT, @office.SCHEMA_XLSX_TEXT), + ("query", @office.SCHEMA_DOCX_QUERY, @office.SCHEMA_XLSX_QUERY), ] { - let (name, schema) = case + let (name, docx_schema, xlsx_schema) = case guard @office.find_capability_command(name) is Some(command) else { fail("missing command: \{name}") } - assert_eq(command.formats, ["docx"]) + assert_eq(command.formats, ["docx", "xlsx"]) assert_true(command.inputs.any(field => field.name == "max-elements")) assert_true(command.inputs.any(field => field.name == "max-output-chars")) + guard command.outputs.filter(field => field.name == "schema").get(0) + is Some(schema) else { + fail("missing schema output: \{name}") + } + assert_true(schema.type_name.contains(docx_schema)) + assert_true(schema.type_name.contains(xlsx_schema)) + assert_true( + command.variants.any(variant => variant.result_schema == docx_schema), + ) assert_true( - command.outputs.any(field => { - field.name == "schema" && field.type_name.contains(schema) - }), + command.variants.any(variant => variant.result_schema == xlsx_schema), ) } guard @office.find_capability_command("query") is Some(query) else { @@ -186,6 +193,22 @@ test "DOCX read capabilities publish every enforced bound and result schema" { assert_true(query.inputs.any(field => field.name == "property")) assert_true(query.usage.contains("--ignore-case")) assert_false(query.usage.to_lower().contains("regex")) + guard @office.find_capability_command("outline") is Some(outline) else { + fail("missing outline command") + } + for name in ["active_sheet", "sheets", "defined_names", "limits"] { + assert_true(outline.outputs.any(field => field.name == name)) + } + guard @office.find_capability_command("get") is Some(get) else { + fail("missing get command") + } + for + name in [ + "sheet_count", "sheets", "defined_names", "sheet", "cell", "cells", "reference", + "styles", "scanned_cells", "returned", + ] { + assert_true(get.outputs.any(field => field.name == name)) + } } ///| @@ -196,13 +219,14 @@ test "capability registry format filters remain truthful" { inspect( records, content=( - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"outline","summary":"Summarize bounded DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"projection scan ceiling; default 50000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":true,"description":"number of elements in the bounded projection"},{"name":"counts","type":"object","required":true,"description":"deterministic structural counts across every DOCX story"},{"name":"stories","type":"array","required":true,"description":"body, header, footer, note, endnote, and comment story roots"},{"name":"headings","type":"array","required":true,"description":"bounded heading previews with canonical paths"},{"name":"styles_in_use","type":"array","required":true,"description":"deduplicated styles referenced by projected elements"},{"name":"images","type":"array","required":true,"description":"image metadata with canonical paths and byte counts"},{"name":"sections","type":"array","required":true,"description":"section boundaries and effective header/footer references"},{"name":"diagnostics","type":"array","required":true,"description":"reader and annotation source diagnostics"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"get","summary":"Resolve one canonical DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical /docx selector"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"projection scan ceiling; default 50000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":true,"description":"projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":true,"description":"physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":true,"description":"addressable direct children with canonical paths"},{"name":"properties","type":"object","required":true,"description":"declared formatting and element summary"},{"name":"metadata","type":"object","required":true,"description":"role-specific annotation or collection metadata"},{"name":"text","type":"string","required":true,"description":"bounded raw text projection"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"text","summary":"Extract bounded path-tagged DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical subtree root"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based paragraph offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"projection scan ceiling; default 50000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in deterministic document order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan paragraph count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":true,"description":"number of elements in the bounded projection"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved subtree root when one was requested"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"query","summary":"Run bounded deterministic predicates over DOCX elements","usage":"office query FILE [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical subtree root"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"exact kind; documented long-name aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"exact annotation id predicate, including ids outside selector path limits"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"up to 16 exact predicates with values capped at 1048576 characters"},{"name":"ignore-case","type":"boolean","required":false,"description":"locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"projection scan ceiling; default 50000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":true,"description":"normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":true,"description":"number of elements in the bounded projection"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved subtree root when one was requested"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"outline","summary":"Summarize bounded XLSX or DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.outline/1|office.xlsx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"},{"name":"path","type":"literal(/xlsx/workbook)","required":false,"description":"XLSX only: canonical workbook selector"},{"name":"sheet_count","type":"integer","required":false,"description":"XLSX only: workbook tab count"},{"name":"active_sheet","type":"object","required":false,"description":"XLSX only: canonical active-sheet summary when the workbook has tabs"},{"name":"sheets","type":"array","required":false,"description":"XLSX only: bounded tab-order sheet summaries with canonical paths and used ranges"},{"name":"defined_names","type":"array","required":false,"description":"XLSX only: bounded workbook defined-name inventory"},{"name":"limits","type":"object","required":false,"description":"XLSX only: effective scan and metadata limits"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office outline FILE","result_schema":"office.docx.outline/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office outline FILE","result_schema":"office.xlsx.outline/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"get","summary":"Resolve one canonical XLSX or DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.element/1|office.xlsx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"},{"name":"sheet_count","type":"integer","required":false,"description":"XLSX workbook selectors only: workbook tab count"},{"name":"sheets","type":"array","required":false,"description":"XLSX workbook selectors only: bounded tab-order sheet summaries"},{"name":"defined_names","type":"array","required":false,"description":"XLSX workbook selectors only: bounded defined-name inventory"},{"name":"sheet","type":"object","required":false,"description":"XLSX sheet selectors only: bounded sheet summary"},{"name":"cell","type":"object","required":false,"description":"XLSX cell selectors only: typed value, formula, style, and canonical path"},{"name":"cells","type":"array","required":false,"description":"XLSX range selectors only: populated cells in row-major order"},{"name":"reference","type":"a1-range","required":false,"description":"XLSX range selectors only: normalized A1 rectangle"},{"name":"styles","type":"object","required":false,"description":"XLSX coordinate selectors only: deduplicated referenced style definitions"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX coordinate selectors only: exact rectangle scan count"},{"name":"returned","type":"integer","required":false,"description":"XLSX range selectors only: populated cell count"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office get FILE /docx/...","result_schema":"office.docx.element/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office get FILE /xlsx/...","result_schema":"office.xlsx.element/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.text/1|office.xlsx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office text FILE [--under /xlsx/...]","result_schema":"office.xlsx.text/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: cell followed by up to 16 bounded type, value, formula, or text predicates; default cell"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.query/1|office.xlsx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: supplied bounded cell selector"},{"name":"styles","type":"object","required":false,"description":"XLSX only: deduplicated styles referenced by returned matches"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office query FILE [cell[predicate]...]","result_schema":"office.xlsx.query/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} + ), ) } @@ -212,7 +236,8 @@ test "capability fingerprint and inventory ordering are deterministic" { inspect( @office.capability_fingerprint(), content=( - #|crc32:31aeb371 + #|crc32:e9c5b34f + ), ) let first = @office.capabilities_data().stringify() @@ -228,7 +253,8 @@ test "capability operation queries do not leak unrelated records" { inspect( data.stringify(), content=( - #|{"schema":"office.capabilities/2","fingerprint":"crc32:31aeb371","records":[{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} + #|{"schema":"office.capabilities/2","fingerprint":"crc32:e9c5b34f","records":[{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} + ), ) } diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index 52ad9882..f0b34400 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -82,7 +82,7 @@ fn docx_common_options() -> Array[@argparse.OptionArg] { OptionArg( "max-elements", long="max-elements", - about="maximum canonical projection nodes to scan (default 50000, hard limit 200000)", + about="maximum DOCX projection nodes or XLSX cells to scan (default 50000; XLSX hard ceiling 100000)", ), OptionArg( "max-output-chars", @@ -116,25 +116,63 @@ fn resolve_optional_under( } } +///| +fn resolve_optional_xlsx_under( + projection : XlsxProjection, + matches : @argparse.Matches, +) -> XlsxResolvedSelector? raise CliFailure { + match optional_value(matches, "under") { + Some(path) => Some(resolve_xlsx_selector(projection, path)) + None => None + } +} + ///| async fn run_docx_outline(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let max_elements = cli_max_elements(matches) let max_output = cli_max_output_chars(matches) - let projection = read_docx_projection(file, max_elements) - if matches.flags.get_or_default("json", false) { - emit_docx_success( - checked_docx_json_output( - docx_outline_payload(projection, max_output_chars=max_output), - ), - ) - } else { - emit_docx_success( - checked_docx_human_output( - docx_outline_human(projection, maximum=max_output), - max_output, - ), - ) + let source = read_office_package(file) + match source.format { + Docx => { + let projection = open_docx_projection_archive( + file, + source.archive, + max_elements, + ) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_docx_json_output( + docx_outline_payload(projection, max_output_chars=max_output), + ), + ) + } else { + emit_docx_success( + checked_docx_human_output( + docx_outline_human(projection, maximum=max_output), + max_output, + ), + ) + } + } + Xlsx => { + let projection = open_xlsx_projection(source, max_elements) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_office_json_output( + xlsx_outline_payload(projection, max_output), + ), + ) + } else { + emit_docx_success( + checked_office_human_output( + xlsx_outline_human(projection, max_output), + max_output, + "xlsx", + ), + ) + } + } } } @@ -144,29 +182,59 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { let selector = required_value(matches, "selector") let max_elements = cli_max_elements(matches) let max_output = cli_max_output_chars(matches) - let projection = read_docx_projection(file, max_elements) - let entry = resolve_projection_selector(projection, selector) - if matches.flags.get_or_default("json", false) { - emit_docx_success( - checked_docx_json_output(get_docx_payload(projection, entry, max_output)), - ) - } else { - let text = match entry.element { - Some(_) => { - let available = (max_output - docx_cli_output_framing_chars).max(0) - let budget = DocxTextBudget::new_reported( - "successful command output characters", available, max_output, docx_cli_output_framing_chars, + let source = read_office_package(file) + match source.format { + Docx => { + let projection = open_docx_projection_archive( + file, + source.archive, + max_elements, + ) + let entry = resolve_projection_selector(projection, selector) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_docx_json_output( + get_docx_payload(projection, entry, max_output), + ), + ) + } else { + let text = match entry.element { + Some(_) => { + let available = (max_output - docx_cli_output_framing_chars).max(0) + let budget = DocxTextBudget::new_reported( + "successful command output characters", available, max_output, docx_cli_output_framing_chars, + ) + entry_text_with_budget(entry, budget) + } + None => "" + } + emit_docx_success( + checked_docx_human_output( + docx_get_human(entry, text, projection.warnings, maximum=max_output), + max_output, + ), + ) + } + } + Xlsx => { + let projection = open_xlsx_projection(source, max_elements) + let resolved = resolve_xlsx_selector(projection, selector) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_office_json_output( + xlsx_get_payload(projection, resolved, max_output), + ), + ) + } else { + emit_docx_success( + checked_office_human_output( + xlsx_get_human(projection, resolved, max_output), + max_output, + "xlsx", + ), ) - entry_text_with_budget(entry, budget) } - None => "" } - emit_docx_success( - checked_docx_human_output( - docx_get_human(entry, text, projection.warnings, maximum=max_output), - max_output, - ), - ) } } @@ -181,30 +249,58 @@ async fn run_docx_text(matches : @argparse.Matches) -> Unit { let limit = bounded_decimal_argument( matches, "limit", docx_cli_default_text_limit, 0, docx_cli_hard_text_limit, ) - let projection = read_docx_projection(file, max_elements) - let (under, under_path) = resolve_optional_under(projection, matches) - if matches.flags.get_or_default("json", false) { - emit_docx_success( - checked_docx_json_output( - docx_text_payload( - projection, under, under_path, offset, limit, max_output, - ), - ), - ) - } else { - emit_docx_success( - checked_docx_human_output( - docx_text_human( - projection, - under, - offset, - limit, - projection.warnings, - maximum=max_output, - ), - max_output, - ), - ) + let source = read_office_package(file) + match source.format { + Docx => { + let projection = open_docx_projection_archive( + file, + source.archive, + max_elements, + ) + let (under, under_path) = resolve_optional_under(projection, matches) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_docx_json_output( + docx_text_payload( + projection, under, under_path, offset, limit, max_output, + ), + ), + ) + } else { + emit_docx_success( + checked_docx_human_output( + docx_text_human( + projection, + under, + offset, + limit, + projection.warnings, + maximum=max_output, + ), + max_output, + ), + ) + } + } + Xlsx => { + let projection = open_xlsx_projection(source, max_elements) + let under = resolve_optional_xlsx_under(projection, matches) + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_office_json_output( + xlsx_text_payload(projection, under, offset, limit, max_output), + ), + ) + } else { + emit_docx_success( + checked_office_human_output( + xlsx_text_human(projection, under, offset, limit, max_output), + max_output, + "xlsx", + ), + ) + } + } } } @@ -362,6 +458,50 @@ fn query_properties( predicates } +///| +fn reject_xlsx_query_options( + matches : @argparse.Matches, +) -> Unit raise CliFailure { + let incompatible : Array[String] = [] + for name in ["kind", "text", "id"] { + if optional_value(matches, name) is Some(_) { + incompatible.push("--" + name) + } + } + if !repeated_values(matches, "property").is_empty() { + incompatible.push("--property") + } + if matches.flags.get_or_default("ignore-case", false) { + incompatible.push("--ignore-case") + } + if !incompatible.is_empty() { + raise xlsx_cli_failure( + "office.xlsx.unsupported_query_options", + "DOCX query options cannot be used with an XLSX workbook; use the cell selector predicates", + details=Json::object({ + "options": Json::array(incompatible.map(Json::string)), + }), + ) + } +} + +///| +fn reject_docx_content_selector( + matches : @argparse.Matches, +) -> Unit raise CliFailure { + match optional_value(matches, "selector") { + Some(value) => + raise docx_cli_failure( + "office.docx.unsupported_query_selector", + "the positional cell selector is available only for XLSX queries", + details=Json::object({ + "selector": Json::string(bounded_text(value, 240)), + }), + ) + None => () + } +} + ///| async fn run_docx_query(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") @@ -373,47 +513,84 @@ async fn run_docx_query(matches : @argparse.Matches) -> Unit { let limit = bounded_decimal_argument( matches, "limit", docx_cli_default_query_limit, 0, docx_cli_hard_query_limit, ) - let text = optional_value(matches, "text") - match text { - Some(value) => validate_query_value(value, "text", false) - None => () - } - let stable_id = optional_value(matches, "id") - match stable_id { - Some(value) => validate_query_value(value, "id", false) - None => () - } - let projection = read_docx_projection(file, max_elements) - let (under, under_path) = resolve_optional_under(projection, matches) - let spec : DocxQuerySpec = { - under: under_path, - kind: query_kind(matches), - text, - stable_id, - properties: query_properties(matches), - ignore_case: matches.flags.get_or_default("ignore-case", false), - offset, - limit, - } - if matches.flags.get_or_default("json", false) { - emit_docx_success( - checked_docx_json_output( - docx_query_payload(projection, under, spec, max_output), - ), - ) - } else { - emit_docx_success( - checked_docx_human_output( - docx_query_human( - projection, - under, - spec, - projection.warnings, - maximum=max_output, - ), - max_output, - ), - ) + let source = read_office_package(file) + match source.format { + Docx => { + reject_docx_content_selector(matches) + let text = optional_value(matches, "text") + match text { + Some(value) => validate_query_value(value, "text", false) + None => () + } + let stable_id = optional_value(matches, "id") + match stable_id { + Some(value) => validate_query_value(value, "id", false) + None => () + } + let projection = open_docx_projection_archive( + file, + source.archive, + max_elements, + ) + let (under, under_path) = resolve_optional_under(projection, matches) + let spec : DocxQuerySpec = { + under: under_path, + kind: query_kind(matches), + text, + stable_id, + properties: query_properties(matches), + ignore_case: matches.flags.get_or_default("ignore-case", false), + offset, + limit, + } + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_docx_json_output( + docx_query_payload(projection, under, spec, max_output), + ), + ) + } else { + emit_docx_success( + checked_docx_human_output( + docx_query_human( + projection, + under, + spec, + projection.warnings, + maximum=max_output, + ), + max_output, + ), + ) + } + } + Xlsx => { + reject_xlsx_query_options(matches) + let selector = optional_value(matches, "selector").unwrap_or("cell") + let projection = open_xlsx_projection(source, max_elements) + let under = resolve_optional_xlsx_under(projection, matches) + let spec : XlsxQuerySpec = { + selector, + predicates: parse_xlsx_query_selector(selector), + offset, + limit, + } + if matches.flags.get_or_default("json", false) { + emit_docx_success( + checked_office_json_output( + xlsx_query_payload(projection, under, spec, max_output), + ), + ) + } else { + emit_docx_success( + checked_office_human_output( + xlsx_query_human(projection, under, spec, max_output), + max_output, + "xlsx", + ), + ) + } + } } } @@ -421,7 +598,7 @@ async fn run_docx_query(matches : @argparse.Matches) -> Unit { fn outline_command() -> @argparse.Command { Command( "outline", - about="Summarize bounded DOCX structure using canonical selectors", + about="Summarize bounded XLSX or DOCX structure using canonical selectors", positionals=[PositionArg("file", num_args=@argparse.ValueRange::single())], options=docx_common_options(), flags=[docx_json_flag()], @@ -432,12 +609,12 @@ fn outline_command() -> @argparse.Command { fn get_command() -> @argparse.Command { Command( "get", - about="Resolve one canonical DOCX selector", + about="Resolve one canonical XLSX or DOCX selector", positionals=[ PositionArg("file", num_args=@argparse.ValueRange::single()), PositionArg( "selector", - about="canonical path such as /docx/body/p[1]", + about="canonical path such as /docx/body/p[1] or /xlsx/sheet[name=\"Data\"]/cell[A1]", num_args=@argparse.ValueRange::single(), ), ], @@ -453,18 +630,18 @@ fn text_command() -> @argparse.Command { OptionArg( "under", long="under", - about="restrict paragraphs to a canonical DOCX selector subtree", + about="restrict results to a canonical DOCX subtree or XLSX sheet/range/cell", ), OptionArg("offset", long="offset", about="zero-based result offset"), OptionArg( "limit", long="limit", - about="maximum returned paragraphs (default 2000, hard limit 10000)", + about="maximum returned paragraphs or cells (default 2000, hard limit 10000)", ), ]) Command( "text", - about="Extract bounded path-tagged DOCX paragraph text", + about="Extract bounded path-tagged XLSX cell or DOCX paragraph text", positionals=[PositionArg("file", num_args=@argparse.ValueRange::single())], options~, flags=[docx_json_flag()], @@ -478,24 +655,24 @@ fn query_command() -> @argparse.Command { OptionArg( "under", long="under", - about="restrict matches to a canonical DOCX selector subtree", + about="restrict matches to a canonical DOCX subtree or XLSX sheet/range/cell", ), OptionArg( "kind", long="kind", - about="exact element kind (aliases: paragraph, run, table, row, cell, link, picture)", + about="DOCX only: exact element kind (aliases: paragraph, run, table, row, cell, link, picture)", ), OptionArg( "text", long="text", - about="literal text substring (no regular expressions)", + about="DOCX only: literal text substring (no regular expressions)", ), - OptionArg("id", long="id", about="exact annotation id"), + OptionArg("id", long="id", about="DOCX only: exact annotation id"), OptionArg( "property", long="property", action=Append, - about="repeatable declared property predicate NAME=VALUE", + about="DOCX only: repeatable declared property predicate NAME=VALUE", ), OptionArg("offset", long="offset", about="zero-based match offset"), OptionArg( @@ -506,8 +683,15 @@ fn query_command() -> @argparse.Command { ]) Command( "query", - about="Run bounded deterministic predicates over DOCX elements", - positionals=[PositionArg("file", num_args=@argparse.ValueRange::single())], + about="Run bounded deterministic predicates over XLSX cells or DOCX elements", + positionals=[ + PositionArg("file", num_args=@argparse.ValueRange::single()), + PositionArg( + "selector", + about="XLSX only: cell[predicate] selector (default cell)", + num_args=ValueRange(lower=0, upper=1), + ), + ], options~, flags=[ docx_json_flag(), diff --git a/office/cmd/office/docx_projection.mbt b/office/cmd/office/docx_projection.mbt index 0d1d4e7b..b019fce3 100644 --- a/office/cmd/office/docx_projection.mbt +++ b/office/cmd/office/docx_projection.mbt @@ -1011,6 +1011,25 @@ fn open_docx_projection( file : String, data : BytesView, max_elements : Int, +) -> DocxProjection raise CliFailure { + let archive = @zip.read_limited( + data, + max_package_bytes=docx_cli_max_package_bytes, + max_entries=docx_cli_max_zip_entries, + max_entry_uncompressed_bytes=docx_cli_max_zip_entry_bytes, + max_total_uncompressed_bytes=docx_cli_max_zip_total_bytes, + max_total_preserved_source_bytes=docx_cli_max_package_bytes, + ) catch { + error => raise docx_zip_failure(error, file) + } + open_docx_projection_archive(file, archive, max_elements) +} + +///| +fn open_docx_projection_archive( + file : String, + archive : @zip.Archive, + max_elements : Int, ) -> DocxProjection raise CliFailure { if max_elements < 1 || max_elements > docx_cli_hard_max_elements { raise docx_cli_failure( @@ -1023,16 +1042,6 @@ fn open_docx_projection( }), ) } - let archive = @zip.read_limited( - data, - max_package_bytes=docx_cli_max_package_bytes, - max_entries=docx_cli_max_zip_entries, - max_entry_uncompressed_bytes=docx_cli_max_zip_entry_bytes, - max_total_uncompressed_bytes=docx_cli_max_zip_total_bytes, - max_total_preserved_source_bytes=docx_cli_max_package_bytes, - ) catch { - error => raise docx_zip_failure(error, file) - } let main_document_part = validate_docx_read_archive(file, archive) let xml_budget = @xml.xml_read_budget( max_source_units=docx_cli_max_xml_source_units, @@ -1051,18 +1060,3 @@ fn open_docx_projection( } projection_from_annotated(file, annotated, max_elements) } - -///| -async fn read_docx_projection( - file : String, - max_elements : Int, -) -> DocxProjection { - let data = read_bounded_file( - file, - docx_cli_max_package_bytes, - "office.file_read_failed", - "input package", - limit_code="office.docx.resource_limit", - ) - open_docx_projection(file, data, max_elements) -} diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index c83d4e6d..241eef54 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -25,12 +25,6 @@ let docx_cli_query_preview_chars : Int = 160 ///| let docx_cli_heading_preview_chars : Int = 512 -///| -/// A linear-work ceiling independent of retained output. UTF-16 length is used -/// as a conservative upper bound for Unicode-scalar iteration, so the limit -/// also covers supplementary characters without under-counting. -let docx_cli_max_query_predicate_work_units : Int = 128 * 1024 * 1024 - ///| struct DocxPropertyPredicate { name : String @@ -49,121 +43,19 @@ struct DocxQuerySpec { limit : Int } -///| -struct DocxQueryWorkBudget { - maximum : Int - mut used : Int -} - -///| -fn DocxQueryWorkBudget::new(maximum : Int) -> DocxQueryWorkBudget { - { maximum, used: 0 } -} - -///| -fn DocxQueryWorkBudget::charge( - self : DocxQueryWorkBudget, - count : Int, -) -> Unit raise CliFailure { - if count < 0 || count > self.maximum - self.used { - let actual = if count < 0 { self.used } else { self.used + count } - raise docx_resource_failure( - "query predicate work units", - self.maximum, - actual~, - ) - } - self.used += count -} - -///| -/// A Knuth-Morris-Pratt literal pattern. The failure table is compiled once -/// per command, making every document-text match linear in the scanned text. -struct DocxLinearPattern { - characters : Array[Char] - failure : Array[Int] - ignore_case : Bool -} - -///| -fn query_fold_character(character : Char, ignore_case : Bool) -> Char { - if ignore_case { - // Locale-independent Unicode simple lowercase mapping is one-to-one, so - // it preserves substring boundaries and has a fixed bound per scalar. - @unicode.to_simple_lowercase(character) - } else { - character - } -} - -///| -fn DocxLinearPattern::compile( - needle : String, - ignore_case : Bool, - work : DocxQueryWorkBudget, -) -> DocxLinearPattern raise CliFailure { - // Four code-unit work units conservatively cover scalar iteration, simple - // folding, and the amortized failure-table comparisons. - work.charge(needle.length() * 4 + 1) - let characters : Array[Char] = [] - for character in needle { - characters.push(query_fold_character(character, ignore_case)) - } - let failure = Array::make(characters.length(), 0) - let mut matched = 0 - for at in 1.. 0 && characters[at] != characters[matched] { - matched = failure[matched - 1] - } - if characters[at] == characters[matched] { - matched += 1 - } - failure[at] = matched - } - { characters, failure, ignore_case } -} - -///| -fn DocxLinearPattern::is_in( - self : DocxLinearPattern, - value : String, - work : DocxQueryWorkBudget, -) -> Bool raise CliFailure { - if self.characters.is_empty() { - return true - } - // KMP performs at most two pattern comparisons per scalar. The third unit - // covers iteration/folding; UTF-16 length safely over-counts scalars. - work.charge(value.length() * 3 + 1) - let mut matched = 0 - for raw_character in value { - let character = query_fold_character(raw_character, self.ignore_case) - while matched > 0 && character != self.characters[matched] { - matched = self.failure[matched - 1] - } - if character == self.characters[matched] { - matched += 1 - if matched == self.characters.length() { - return true - } - } - } - false -} - ///| struct PreparedDocxQuery { spec : DocxQuerySpec scope_path : String? scope_prefix : String? - text_pattern : DocxLinearPattern? + text_pattern : OfficeLinearPattern? } ///| fn PreparedDocxQuery::build( spec : DocxQuerySpec, under : DocxProjectionEntry?, - work : DocxQueryWorkBudget, + work : OfficeQueryWorkBudget, ) -> PreparedDocxQuery raise CliFailure { let (scope_path, scope_prefix) = match under { Some(root) => (Some(root.path), Some(root.path + "/")) @@ -171,7 +63,7 @@ fn PreparedDocxQuery::build( } let text_pattern = match spec.text { Some(needle) => - Some(DocxLinearPattern::compile(needle, spec.ignore_case, work)) + Some(OfficeLinearPattern::compile(needle, spec.ignore_case, work)) None => None } { spec, scope_path, scope_prefix, text_pattern } @@ -686,7 +578,7 @@ fn query_record_json( fn entry_matches_properties( entry : DocxProjectionEntry, predicates : Array[DocxPropertyPredicate], - work : DocxQueryWorkBudget, + work : OfficeQueryWorkBudget, ) -> Bool raise CliFailure { for predicate in predicates { match entry.properties.get(predicate.name) { @@ -707,7 +599,7 @@ fn entry_matches_properties( fn query_strings_equal( value : String, expected : String, - work : DocxQueryWorkBudget, + work : OfficeQueryWorkBudget, ) -> Bool raise CliFailure { work.charge(value.length().min(expected.length()) + 1) value == expected @@ -717,7 +609,7 @@ fn query_strings_equal( fn query_entry_is_in_scope( entry : DocxProjectionEntry, query : PreparedDocxQuery, - work : DocxQueryWorkBudget, + work : OfficeQueryWorkBudget, ) -> Bool raise CliFailure { match (query.scope_path, query.scope_prefix) { (Some(root), Some(prefix)) => @@ -736,7 +628,7 @@ fn entry_matches_query( entry : DocxProjectionEntry, query : PreparedDocxQuery, scan_budget : DocxTextBudget, - work : DocxQueryWorkBudget, + work : OfficeQueryWorkBudget, ) -> Bool raise CliFailure { if !query_entry_is_in_scope(entry, query, work) { return false @@ -837,7 +729,9 @@ fn docx_query_payload( let scan_budget = DocxTextBudget::new( "query text scan characters", docx_cli_max_query_text_scan_chars, ) - let work = DocxQueryWorkBudget::new(docx_cli_max_query_predicate_work_units) + let work = OfficeQueryWorkBudget::new( + office_cli_max_query_predicate_work_units, + ) let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 for entry in projection.entries { @@ -1295,7 +1189,9 @@ fn docx_query_human( let scan_budget = DocxTextBudget::new( "query text scan characters", docx_cli_max_query_text_scan_chars, ) - let work = DocxQueryWorkBudget::new(docx_cli_max_query_predicate_work_units) + let work = OfficeQueryWorkBudget::new( + office_cli_max_query_predicate_work_units, + ) let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 let mut returned = 0 diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index 80234037..b908371e 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -319,7 +319,7 @@ test "DOCX exact queries cover the full supported XML token length" { offset: 0, limit: 1, } - let work = DocxQueryWorkBudget::new(4 * 1024 * 1024) + let work = OfficeQueryWorkBudget::new(4 * 1024 * 1024) assert_true( entry_matches_query( item, @@ -352,23 +352,23 @@ test "DOCX exact queries cover the full supported XML token length" { ///| test "DOCX query matching is linear-work and Unicode simple-case aware" { - let work = DocxQueryWorkBudget::new(100_000) - let adversarial = DocxLinearPattern::compile( + let work = OfficeQueryWorkBudget::new(100_000) + let adversarial = OfficeLinearPattern::compile( "a".repeat(127) + "b", false, work, ) assert_false(adversarial.is_in("a".repeat(4096), work)) assert_true(adversarial.is_in("a".repeat(127) + "b", work)) - let unicode = DocxLinearPattern::compile("école", true, work) + let unicode = OfficeLinearPattern::compile("école", true, work) assert_true(unicode.is_in("L'ÉCOLE MoonBit", work)) - let supplementary = DocxLinearPattern::compile("𐐨", true, work) + let supplementary = OfficeLinearPattern::compile("𐐨", true, work) assert_true(supplementary.is_in("𐐀", work)) } ///| test "DOCX query predicate work has an independent hard budget" { - let equality_work = DocxQueryWorkBudget::new(16) + let equality_work = OfficeQueryWorkBudget::new(16) try query_strings_equal("x".repeat(32), "x".repeat(32), equality_work) catch { CliFailure(error) => { assert_eq(error.code, "office.docx.resource_limit") @@ -383,8 +383,8 @@ test "DOCX query predicate work has an independent hard budget" { } noraise { _ => fail("expected exact comparison work to be bounded") } - let pattern_work = DocxQueryWorkBudget::new(32) - let pattern = DocxLinearPattern::compile("a", false, pattern_work) + let pattern_work = OfficeQueryWorkBudget::new(32) + let pattern = OfficeLinearPattern::compile("a", false, pattern_work) try pattern.is_in("a".repeat(32), pattern_work) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") } noraise { diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index e3d0a6d8..5901a4c0 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -25,8 +25,6 @@ type DocxCollectedText type DocxHumanWriter -type DocxLinearPattern - type DocxProjection type DocxProjectionBuilder @@ -39,22 +37,26 @@ type DocxPropertyPredicate type DocxQuerySpec -type DocxQueryWorkBudget - type DocxTextBudget type DocxTextCollector type HelpQuery +type OfficeLinearPattern + type OfficeOutputBudget +type OfficeQueryWorkBudget + type OfficeReadPackage type OutputMode type PreparedDocxQuery +type PreparedXlsxCellPredicate + type XlsxCellFilter type XlsxCellKind diff --git a/office/cmd/office/query_work.mbt b/office/cmd/office/query_work.mbt new file mode 100644 index 00000000..faec6437 --- /dev/null +++ b/office/cmd/office/query_work.mbt @@ -0,0 +1,112 @@ +///| +/// A command-wide linear-work ceiling independent of retained output. UTF-16 +/// length is a conservative upper bound for Unicode-scalar iteration, so the +/// limit also covers supplementary characters without under-counting. +let office_cli_max_query_predicate_work_units : Int = 128 * 1024 * 1024 + +///| +struct OfficeQueryWorkBudget { + format : String + maximum : Int + mut used : Int +} + +///| +fn OfficeQueryWorkBudget::new( + maximum : Int, + format? : String = "docx", +) -> OfficeQueryWorkBudget { + { format, maximum, used: 0 } +} + +///| +fn OfficeQueryWorkBudget::charge( + self : OfficeQueryWorkBudget, + count : Int, +) -> Unit raise CliFailure { + if count < 0 || count > self.maximum - self.used { + let actual = if count < 0 { self.used } else { self.used + count } + raise format_resource_failure( + self.format, + "query predicate work units", + self.maximum, + actual~, + ) + } + self.used += count +} + +///| +/// A Knuth-Morris-Pratt literal pattern. The failure table is compiled once +/// per command, making every value match linear in the scanned text. +struct OfficeLinearPattern { + characters : Array[Char] + failure : Array[Int] + ignore_case : Bool +} + +///| +fn query_fold_character(character : Char, ignore_case : Bool) -> Char { + if ignore_case { + // Locale-independent Unicode simple lowercase mapping is one-to-one, so + // it preserves substring boundaries and has a fixed bound per scalar. + @unicode.to_simple_lowercase(character) + } else { + character + } +} + +///| +fn OfficeLinearPattern::compile( + needle : String, + ignore_case : Bool, + work : OfficeQueryWorkBudget, +) -> OfficeLinearPattern raise CliFailure { + // Four code-unit work units conservatively cover scalar iteration, simple + // folding, and the amortized failure-table comparisons. + work.charge(needle.length() * 4 + 1) + let characters : Array[Char] = [] + for character in needle { + characters.push(query_fold_character(character, ignore_case)) + } + let failure = Array::make(characters.length(), 0) + let mut matched = 0 + for at in 1.. 0 && characters[at] != characters[matched] { + matched = failure[matched - 1] + } + if characters[at] == characters[matched] { + matched += 1 + } + failure[at] = matched + } + { characters, failure, ignore_case } +} + +///| +fn OfficeLinearPattern::is_in( + self : OfficeLinearPattern, + value : String, + work : OfficeQueryWorkBudget, +) -> Bool raise CliFailure { + if self.characters.is_empty() { + return true + } + // KMP performs at most two pattern comparisons per scalar. The third unit + // covers iteration/folding; UTF-16 length safely over-counts scalars. + work.charge(value.length() * 3 + 1) + let mut matched = 0 + for raw_character in value { + let character = query_fold_character(raw_character, self.ignore_case) + while matched > 0 && character != self.characters[matched] { + matched = self.failure[matched - 1] + } + if character == self.characters[matched] { + matched += 1 + if matched == self.characters.length() { + return true + } + } + } + false +} diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 9cb11736..27b78677 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -33,7 +33,7 @@ fn office_read_format_hint( } else if lower.has_suffix(".docx") { Docx } else { - raise CliFailure(identify_error(@lib.UnsupportedFileExtension(file), file)) + raise CliFailure(identify_error(UnsupportedFileExtension(file), file)) } } @@ -44,7 +44,7 @@ fn office_zip_read_failure( expected : @lib.DocumentFormat, ) -> CliFailure { match error { - @zip.ResourceLimitExceeded(kind~, limit~, actual~) => + ResourceLimitExceeded(kind~, limit~, actual~) => CliFailure( @lib.protocol_error( "office.\{expected.name()}.resource_limit", @@ -57,7 +57,7 @@ fn office_zip_read_failure( }), ), ) - @zip.OutputLimitExceeded(limit~) => + OutputLimitExceeded(limit~) => format_resource_failure(expected.name(), "ZIP output", limit) _ => CliFailure( @@ -111,7 +111,7 @@ async fn read_office_package(file : String) -> OfficeReadPackage { ///| fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { match error { - @xlsx.ResourceLimitExceeded(kind~, limit~, actual~) => + ResourceLimitExceeded(kind~, limit~, actual~) => CliFailure( @lib.protocol_error( "office.xlsx.resource_limit", @@ -124,7 +124,7 @@ fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { }), ), ) - @xlsx.InvalidPackage(msg~) | @xlsx.InvalidXml(msg~) => + InvalidPackage(msg~) | InvalidXml(msg~) => CliFailure( @lib.protocol_error( "office.invalid_package", diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt index dd688a15..3c654323 100644 --- a/office/cmd/office/xlsx_query.mbt +++ b/office/cmd/office/xlsx_query.mbt @@ -36,6 +36,16 @@ enum XlsxCellPredicate { TextContains(String) } +///| +enum PreparedXlsxCellPredicate { + PreparedHasType(XlsxCellKind) + PreparedValueCompare(XlsxNumberOperator, Double) + PreparedHasFormula + PreparedFormulaContains(OfficeLinearPattern) + PreparedTextEquals(String) + PreparedTextContains(OfficeLinearPattern) +} + ///| struct XlsxQuerySpec { selector : String @@ -213,12 +223,39 @@ fn XlsxNumberOperator::matches( } ///| -fn XlsxCellPredicate::matches( +fn XlsxCellPredicate::prepare( self : XlsxCellPredicate, + work : OfficeQueryWorkBudget, +) -> PreparedXlsxCellPredicate raise CliFailure { + match self { + HasType(kind) => PreparedHasType(kind) + ValueCompare(operator, bound) => PreparedValueCompare(operator, bound) + HasFormula => PreparedHasFormula + FormulaContains(needle) => + PreparedFormulaContains(OfficeLinearPattern::compile(needle, false, work)) + TextEquals(expected) => PreparedTextEquals(expected) + TextContains(needle) => + PreparedTextContains(OfficeLinearPattern::compile(needle, false, work)) + } +} + +///| +fn prepare_xlsx_query_predicates( + predicates : Array[XlsxCellPredicate], + work : OfficeQueryWorkBudget, +) -> Array[PreparedXlsxCellPredicate] raise CliFailure { + predicates.map(predicate => predicate.prepare(work)) +} + +///| +fn PreparedXlsxCellPredicate::matches( + self : PreparedXlsxCellPredicate, cell : XlsxCellSnapshot, -) -> Bool { + work : OfficeQueryWorkBudget, +) -> Bool raise CliFailure { match self { - HasType(kind) => + PreparedHasType(kind) => { + work.charge(1) match kind { Formula => cell.formula is Some(_) Number => cell.raw is Some(Numeric(_)) @@ -226,26 +263,44 @@ fn XlsxCellPredicate::matches( Boolean => cell.raw is Some(Bool(_)) ErrorValue => cell.raw is Some(Error(_)) } - ValueCompare(operator, bound) => + } + PreparedValueCompare(operator, bound) => { + work.charge(1) match cell.raw { Some(Numeric(value)) => operator.matches(value, bound) _ => false } - HasFormula => cell.formula is Some(_) - FormulaContains(needle) => + } + PreparedHasFormula => { + work.charge(1) + cell.formula is Some(_) + } + PreparedFormulaContains(pattern) => match cell.formula { - Some(formula) => formula.contains(needle) - None => false + Some(formula) => pattern.is_in(formula, work) + None => { + work.charge(1) + false + } } - TextEquals(expected) => + PreparedTextEquals(expected) => match cell.raw { - Some(String(value)) => value == expected - _ => false + Some(String(value)) => { + work.charge(value.length().min(expected.length()) + 1) + value == expected + } + _ => { + work.charge(1) + false + } } - TextContains(needle) => + PreparedTextContains(pattern) => match cell.raw { - Some(String(value)) => value.contains(needle) - _ => false + Some(String(value)) => pattern.is_in(value, work) + _ => { + work.charge(1) + false + } } } } @@ -253,13 +308,15 @@ fn XlsxCellPredicate::matches( ///| fn xlsx_cell_matches( cell : XlsxCellSnapshot, - predicates : Array[XlsxCellPredicate], -) -> Bool { + predicates : Array[PreparedXlsxCellPredicate], + work : OfficeQueryWorkBudget, +) -> Bool raise CliFailure { + work.charge(1) if !cell.is_present() { return false } for predicate in predicates { - if !predicate.matches(cell) { + if !predicate.matches(cell, work) { return false } } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 56dcc1c5..95bc4442 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -296,6 +296,22 @@ fn XlsxProjection::find_sheet_by_name( } } +///| +fn XlsxProjection::require_sheet_by_name( + self : XlsxProjection, + name : String, +) -> XlsxSheetTarget raise CliFailure { + match self.find_sheet_by_name(name) { + Some(sheet) => sheet + None => + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook tab order contains an unresolved sheet", + details=Json::object({ "sheet": Json::string(bounded_text(name, 160)) }), + ) + } +} + ///| fn XlsxProjection::resolve_sheet_segment( self : XlsxProjection, @@ -425,17 +441,14 @@ fn xlsx_scan_regions( } Some(Workbook(..)) | None => for name in projection.sheet_names { - match projection.find_sheet_by_name(name) { - Some(sheet) => - match sheet.content { - Worksheet(worksheet) => - match xlsx_used_rect(worksheet) { - Some(rect) => regions.push({ sheet, rect }) - None => () - } - ChartSheet(_) => () + let sheet = projection.require_sheet_by_name(name) + match sheet.content { + Worksheet(worksheet) => + match xlsx_used_rect(worksheet) { + Some(rect) => regions.push({ sheet, rect }) + None => () } - None => () + ChartSheet(_) => () } } } diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 4a33d6d6..e1a5fae5 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -2,7 +2,7 @@ enum XlsxCellFilter { PresentCells TextCells - QueryCells(Array[XlsxCellPredicate]) + QueryCells(Array[PreparedXlsxCellPredicate], OfficeQueryWorkBudget) } ///| @@ -19,7 +19,7 @@ fn xlsx_sheet_state_name(state : @xlsx.SheetState) -> String { match state { Visible => "visible" Hidden => "hidden" - VeryHidden => "very-hidden" + VeryHidden => "very_hidden" } } @@ -125,18 +125,8 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { let metadata = XlsxMetadataBudget::new() let sheets : Array[Json] = [] for name in projection.sheet_names { - match projection.find_sheet_by_name(name) { - Some(sheet) => - sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) - None => - raise xlsx_cli_failure( - "office.xlsx.read_failed", - "workbook tab order contains an unresolved sheet", - details=Json::object({ - "sheet": Json::string(bounded_text(name, 160)), - }), - ) - } + let sheet = projection.require_sheet_by_name(name) + sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) } let defined_names : Array[Json] = [] for defined in projection.workbook.get_defined_names() { @@ -176,19 +166,24 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { } if !projection.sheet_names.is_empty() { let active_index = projection.workbook.get_active_sheet_index() - match projection.sheet_names.get(active_index) { - Some(name) => - match projection.find_sheet_by_name(name) { - Some(sheet) => - fields["active_sheet"] = Json::object({ - "path": Json::string(sheet.path), - "name": Json::string(sheet.name), - "index": Json::number((sheet.index + 1).to_double()), - }) - None => () - } - None => () + guard projection.sheet_names.get(active_index) is Some(name) else { + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook active-sheet index is outside the tab order", + details=Json::object({ + "active_sheet_index": Json::number(active_index.to_double()), + "sheet_count": Json::number( + projection.sheet_names.length().to_double(), + ), + }), + ) } + let sheet = projection.require_sheet_by_name(name) + fields["active_sheet"] = Json::object({ + "path": Json::string(sheet.path), + "name": Json::string(sheet.name), + "index": Json::number((sheet.index + 1).to_double()), + }) } Json::object(fields) } @@ -294,14 +289,25 @@ fn xlsx_cell_text(cell : XlsxCellSnapshot) -> String? { fn XlsxCellFilter::matches( self : XlsxCellFilter, cell : XlsxCellSnapshot, -) -> Bool { +) -> Bool raise CliFailure { match self { PresentCells => cell.is_present() TextCells => xlsx_cell_text(cell) is Some(_) - QueryCells(predicates) => xlsx_cell_matches(cell, predicates) + QueryCells(predicates, work) => xlsx_cell_matches(cell, predicates, work) } } +///| +fn xlsx_query_filter( + predicates : Array[XlsxCellPredicate], +) -> XlsxCellFilter raise CliFailure { + let work = OfficeQueryWorkBudget::new( + office_cli_max_query_predicate_work_units, + format="xlsx", + ) + QueryCells(prepare_xlsx_query_predicates(predicates, work), work) +} + ///| fn collect_xlsx_cell_page( projection : XlsxProjection, @@ -357,7 +363,13 @@ fn xlsx_get_data( match resolved { Workbook(path~) => { let outline = xlsx_outline_data(projection) - guard outline is Object(outline_fields) else { abort("outline object") } + let outline_fields = match outline { + Object(fields) => fields + _ => + raise xlsx_cli_failure( + "office.xlsx.read_failed", "workbook outline projection did not produce an object", + ) + } Json::object({ "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), "file": Json::string(projection.file), @@ -518,7 +530,7 @@ fn xlsx_query_data( let page = collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), - QueryCells(spec.predicates), + xlsx_query_filter(spec.predicates), spec.offset, spec.limit, ) @@ -564,7 +576,7 @@ fn xlsx_outline_human( output.write_plain("workbook\t/xlsx/workbook\t") output.write_plain("\{projection.sheet_names.length()} sheets") for name in projection.sheet_names { - guard projection.find_sheet_by_name(name) is Some(sheet) else { continue } + let sheet = projection.require_sheet_by_name(name) output.begin_line() output.write_terminal_safe(sheet.path) output.write_char('\t') @@ -694,7 +706,7 @@ fn xlsx_query_human( collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), - QueryCells(spec.predicates), + xlsx_query_filter(spec.predicates), spec.offset, spec.limit, ), diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 2c1de5fa..647b2ef4 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -133,9 +133,24 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { "cell[type=formula][formula~=SUM][text=3]", ) assert_eq(predicates.length(), 3) - assert_true(xlsx_cell_matches(snapshot, predicates)) + let work = OfficeQueryWorkBudget::new(1024, format="xlsx") + assert_true( + xlsx_cell_matches( + snapshot, + prepare_xlsx_query_predicates(predicates, work), + work, + ), + ) + let comparison_work = OfficeQueryWorkBudget::new(1024, format="xlsx") assert_false( - xlsx_cell_matches(snapshot, parse_xlsx_query_selector("cell[value>3]")), + xlsx_cell_matches( + snapshot, + prepare_xlsx_query_predicates( + parse_xlsx_query_selector("cell[value>3]"), + comparison_work, + ), + comparison_work, + ), ) let malformed = xlsx_cli_error(() => { ignore(parse_xlsx_query_selector("row[type=number]")) @@ -146,6 +161,22 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { ignore(parse_xlsx_query_selector(too_many)) }) assert_eq(predicate_limit.code, "office.xlsx.query_predicate_limit") + let work_limit = xlsx_cli_error(() => { + let limited = OfficeQueryWorkBudget::new(24, format="xlsx") + let prepared = prepare_xlsx_query_predicates( + parse_xlsx_query_selector("cell[text~=alpha]"), + limited, + ) + ignore(xlsx_cell_matches(snapshot, prepared, limited)) + }) + assert_eq(work_limit.code, "office.xlsx.resource_limit") + guard work_limit.details is Some(Object(fields)) else { + fail("expected query-work details") + } + assert_eq( + fields.get("resource"), + Some(Json::string("query predicate work units")), + ) } ///| From ac6802fda8974a31cef8a42ff82a4d233fee5f63 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:36:16 +0800 Subject: [PATCH 006/150] test(office): cover public xlsx read contract --- .github/workflows/ci.yml | 24 +++++ README.mbt.md | 14 ++- docs/agent-json-schemas.md | 83 +++++++++++++++- docs/office-docx-read.md | 8 +- docs/office-major-parity.md | 41 +++++++- docs/office-selectors.md | 17 +++- docs/office-xlsx-read.md | 185 +++++++++++++++++++++++++++++++++++ office/cmd/office/cram/cli.t | 41 ++++++-- 8 files changed, 388 insertions(+), 25 deletions(-) create mode 100644 docs/office-xlsx-read.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdbd6770..c3fc9baa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,6 +281,30 @@ jobs: and .success == false and .error.code == "office.unknown_format" and .error.details.suggestions == ["xlsx"]' office-wasm-error.json + # XLSX structured reads share the validated bounded archive and use + # canonical name-keyed paths even when the input selector is positional. + xlsx_fixture=fixtures/excelize/test/Book1.xlsx + xo=$(moon run --target wasm office/cmd/office -- outline "$xlsx_fixture" --json 2>office-wasm.err) || { echo "office XLSX outline failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xo" | jq -e '.success == true + and .data.schema == "office.xlsx.outline/1" + and .data.path == "/xlsx/workbook" + and [.data.sheets[].path] == ["/xlsx/sheet[name=\"Sheet1\"]", "/xlsx/sheet[name=\"Sheet2\"]"]' + xg=$(moon run --target wasm office/cmd/office -- get "$xlsx_fixture" '/xlsx/sheet[1]/range[A19:B19]' --json 2>office-wasm.err) || { echo "office XLSX get failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xg" | jq -e '.success == true + and .data.schema == "office.xlsx.element/1" + and .data.path == "/xlsx/sheet[name=\"Sheet1\"]/range[A19:B19]" + and [.data.cells[].reference] == ["A19", "B19"] + and .data.cells[1].formula == "SUM(Sheet2!D2,Sheet2!D11)"' + xt=$(moon run --target wasm office/cmd/office -- text "$xlsx_fixture" --under '/xlsx/sheet[name="Sheet1"]' --offset 1 --limit 2 --json 2>office-wasm.err) || { echo "office XLSX text failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xt" | jq -e '.success == true + and .data.schema == "office.xlsx.text/1" + and .data.matched_total == 5 + and [.data.entries[].text] == ["237", "Column1"]' + xq=$(moon run --target wasm office/cmd/office -- query "$xlsx_fixture" 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json 2>office-wasm.err) || { echo "office XLSX query failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xq" | jq -e '.success == true + and .data.schema == "office.xlsx.query/1" + and .data.matched_total == 2 + and [.data.matches[].reference] == ["F11", "G11"]' # DOCX structured reads share one canonical bounded projection. # Drive every command through the wasm async filesystem path. o=$(moon run --target wasm office/cmd/office -- outline docx2html/tests/cram/fixtures/single-paragraph.docx --json 2>office-wasm.err) || { echo "office DOCX outline failed; stderr:"; cat office-wasm.err; exit 1; } diff --git a/README.mbt.md b/README.mbt.md index d52b671f..ce7f6a48 100644 --- a/README.mbt.md +++ b/README.mbt.md @@ -311,8 +311,7 @@ test "validate output" { `office/cmd/office` is the agent-oriented facade for DOCX and XLSX packages. Its registry-driven help, format detection, validated raw OOXML fallback, and -DOCX structured reads use versioned JSON envelopes and explicit resource -limits: +structured reads use versioned JSON envelopes and explicit resource limits: ```sh moon run office/cmd/office -- help docx @@ -321,12 +320,17 @@ moon run office/cmd/office -- outline report.docx --json moon run office/cmd/office -- get report.docx '/docx/body/p[1]' --json moon run office/cmd/office -- text report.docx --under '/docx/comments/comment[id="0"]' moon run office/cmd/office -- query report.docx --kind paragraph --text revenue --ignore-case --json +moon run office/cmd/office -- outline book.xlsx --json +moon run office/cmd/office -- get book.xlsx '/xlsx/sheet[name="Data"]/range[A1:C12]' --json +moon run office/cmd/office -- text book.xlsx --under '/xlsx/sheet[name="Data"]' --json +moon run office/cmd/office -- query book.xlsx 'cell[type=formula]' --under '/xlsx/sheet[name="Data"]' --json ``` -The four DOCX read schemas are `office.docx.outline/1`, -`office.docx.element/1`, `office.docx.text/1`, and `office.docx.query/1`, all +DOCX results use the `office.docx.{outline,element,text,query}/1` family and +XLSX results use `office.xlsx.{outline,element,text,query}/1`; every result is inside `office.output/1`. See -[Unified Office DOCX reads](docs/office-docx-read.md) and +[Unified Office DOCX reads](docs/office-docx-read.md), +[Unified Office XLSX reads](docs/office-xlsx-read.md), and [Canonical Office selectors](docs/office-selectors.md). ## XLSX command-line tool diff --git a/docs/agent-json-schemas.md b/docs/agent-json-schemas.md index b0703b8b..53fc7427 100644 --- a/docs/agent-json-schemas.md +++ b/docs/agent-json-schemas.md @@ -429,9 +429,10 @@ document. Success is `{schema, success: true, data, warnings?}`; failure is `code`, bounded `message`, and optional bounded `details`. The optional warning array contains `{code, message}` records. -The DOCX read commands below share canonical `office.selector/1` paths and one -bounded projection. Full command, selector, limit, and error-code details are -in [office-docx-read.md](office-docx-read.md). +The structured read commands dispatch by validated package content and share +canonical `office.selector/1` paths. Full command, selector, limit, and +error-code details are in [office-docx-read.md](office-docx-read.md) and +[office-xlsx-read.md](office-xlsx-read.md). ### `office.docx.outline/1` (`office outline FILE --json`) @@ -500,6 +501,82 @@ evaluated. `ignore_case` applies locale-independent Unicode simple lowercase mapping to the text predicate (one scalar to one scalar, without locale tailoring or multi-character expansion). +### `office.xlsx.outline/1` (`office outline FILE --json`) + +| key | type | notes | +| --- | --- | --- | +| `schema` | string | `"office.xlsx.outline/1"` | +| `file`, `format` | string | input path and literal `"xlsx"` | +| `path` | string | stable `/xlsx/workbook` | +| `sheet_count` | number | exact tab-order sheet count | +| `active_sheet` | object? | `{path, name, index}` when the workbook has sheets | +| `sheets` | array | tab-order worksheet/chart-sheet summaries | +| `defined_names` | array | `{name, refers_to, scope?, comment?}` | +| `limits` | object | effective cell-scan and metadata ceilings | + +Every sheet summary has canonical `path`, `name`, 1-based `index`, `kind`, +`state`, and `counts`. Worksheets additionally expose `max_row`, `max_column`, +and optional `used_range: {path, reference, cell_count}`. Worksheet counts +cover merges, tables, charts, images, pivots, comments, hyperlinks, data +validations, conditional-format ranges, and slicers. Chart sheets report their +chart count. State is `visible`, `hidden`, or `very_hidden`. + +### `office.xlsx.element/1` (`office get FILE SELECTOR --json`) + +Common keys are `schema`, `file`, literal `format: "xlsx"`, canonical `path`, +`kind`, and `stability`. Kind-specific members are: + +| kind | members | +| --- | --- | +| `workbook` | `sheet_count`, `sheets`, and `defined_names` | +| `sheet` | one `sheet` summary | +| `cell` | stable sheet `parent`, `cell`, `styles`, and `scanned_cells` | +| `range` | sheet `parent`, normalized `reference`, populated `cells`, `styles`, `scanned_cells`, and `returned` | + +Cell records contain canonical `path`, A1 `reference`, 1-based `row` and +`column`, optional displayed `value`, optional typed `raw: {type, value}`, +optional formula text without `=`, and optional nonzero `style_id`. Raw types +are `string`, `number`, `bool`, and `error`. `styles` is an object keyed by +referenced style ids. Range cells are row-major and omit completely blank, +unstyled coordinates. + +### `office.xlsx.text/1` (`office text FILE ... --json`) + +| key | type | notes | +| --- | --- | --- | +| `schema` | string | `"office.xlsx.text/1"` | +| `file`, `format` | string | input path and literal `"xlsx"` | +| `under` | string? | resolved canonical workbook/sheet/cell/range scope | +| `entries` | array | `{path, stability, text}` in tab/row/column order | +| `matched_total` | number | exact count after the completed bounded scan | +| `offset`, `limit`, `returned` | number | explicit pagination | +| `truncated` | boolean | later matches were omitted | +| `scanned_cells` | number | coordinates inspected | + +`text` is the displayed value. A formula without a cached display value falls +back to its formula text prefixed with `=`. + +### `office.xlsx.query/1` (`office query FILE [CELL_SELECTOR] ... --json`) + +| key | type | notes | +| --- | --- | --- | +| `schema` | string | `"office.xlsx.query/1"` | +| `file`, `format` | string | input path and literal `"xlsx"` | +| `under` | string? | resolved canonical workbook/sheet/cell/range scope | +| `selector` | string | the supplied `cell[predicate]...` selector, or default `cell` | +| `matches` | array | row-major cell records in the same shape as `get` | +| `styles` | object | effective styles referenced by returned matches | +| `matched_total` | number | exact count after the completed bounded scan | +| `offset`, `limit`, `returned` | number | explicit pagination | +| `truncated` | boolean | later matches were omitted | +| `scanned_cells` | number | coordinates inspected | + +Predicates are literal and ANDed: `type=formula|number|string|bool|error`, +`formula`, `formula~=TEXT`, `text=TEXT`, `text~=TEXT`, and numeric +`value>NUMBER` comparisons (`>=`, `<`, `<=`, `=`, and `!=` are also +supported). Substring predicates are guaranteed-linear and command-work +bounded. No regular expression or arbitrary expression is evaluated. + ## Standalone `docx` CLI schemas ## `docx.outline/1` — document structure map (`docx outline `) diff --git a/docs/office-docx-read.md b/docs/office-docx-read.md index 90b26192..4475f7e6 100644 --- a/docs/office-docx-read.md +++ b/docs/office-docx-read.md @@ -12,9 +12,11 @@ moon run office/cmd/office -- query report.docx --kind paragraph --text revenue Run `office help docx` for the installed command catalog and `office help --json` for the declared inputs, outputs, and bounds. -These commands accept DOCX packages only. Passing an XLSX package fails with -`office.unsupported_operation`; XLSX selector resolution remains a later -milestone. +The file's validated package format selects the result contract. This document +covers the DOCX branch; passing an XLSX package invokes the corresponding XLSX +read contract in [office-xlsx-read.md](office-xlsx-read.md). Passing an +`/xlsx/...` selector to a DOCX package fails with +`office.docx.selector_format_mismatch`. ## Output envelope diff --git a/docs/office-major-parity.md b/docs/office-major-parity.md index 6c218686..c9ba0b45 100644 --- a/docs/office-major-parity.md +++ b/docs/office-major-parity.md @@ -33,8 +33,8 @@ Codex review at least at `xhigh` effort. | D1: preservation-safe DOCX edit sessions | [#146](https://github.com/moonbitlang/office.mbt/issues/146) | Complete | | D2: bounded DOCX outline, get, text, and query | [#147](https://github.com/moonbitlang/office.mbt/issues/147) | Complete | | X1: provenance-checked bounded XLSX archive reads | [#160](https://github.com/moonbitlang/office.mbt/issues/160) | Complete | -| Coordinate validation hardening | [#138](https://github.com/moonbitlang/office.mbt/issues/138) | Planned | -| X2: unified bounded XLSX outline, get, text, and query | [#161](https://github.com/moonbitlang/office.mbt/issues/161) | Planned | +| Coordinate validation hardening | [#138](https://github.com/moonbitlang/office.mbt/issues/138) | Complete | +| X2: unified bounded XLSX outline, get, text, and query | [#161](https://github.com/moonbitlang/office.mbt/issues/161) | Complete | | X3: transactional XLSX create and batch | [#162](https://github.com/moonbitlang/office.mbt/issues/162) | Planned | | D3: fresh DOCX create and batch | [#163](https://github.com/moonbitlang/office.mbt/issues/163) | Planned | | D4: preservation-safe DOCX annotation mutations | [#164](https://github.com/moonbitlang/office.mbt/issues/164) | Planned | @@ -193,8 +193,41 @@ The XLSX SDK now has one fail-closed policy for every package ingestion path: relationship-relocated XML, CRC corruption, CFB cycles, KDF work, provenance, mutation, async-reader, and encrypted-package cases. -X1 changes the SDK boundary only. Unified XLSX command exposure remains scoped -to X2 after coordinate validation hardening, and no C stubs are introduced. +X1 changes the SDK boundary only and introduces no C stubs. + +## X2 XLSX read contract + +The unified CLI now advertises XLSX support for `outline`, `get`, `text`, and +`query` only after their end-to-end implementations are present: + +- one `moonbitlang/async` bounded file read, validated format dispatch, and one + limited ZIP snapshot feed the provenance-checked archive-backed XLSX parser; +- workbook, name-keyed sheet, cell, and normalized range selectors resolve to + canonical `/xlsx/...` paths; positional sheet input is canonicalized to a + stable name path, while coordinate paths remain explicitly + snapshot-relative; +- outline reports tab order, active sheet, worksheet/chart-sheet state, used + ranges, feature counts, defined names, and effective limits; `get` returns + typed raw/formatted/formula values plus effective styles; +- `text` and `query` scan in tab/row/column order and expose exact bounded-scan + totals plus deterministic pagination; formulas without cached display values + remain visible through an `=FORMULA` text fallback; +- query accepts only declared `cell[...]` predicates for type, formula, + literal string, and finite numeric comparisons; substring predicates compile + once to linear KMP matchers under an explicit command-wide work ceiling; +- package bytes, ZIP entries and expansion, XML parts, scan rectangles and + visited cells, per-value and aggregate strings, metadata, predicates, + predicate work, retained results, and successful output all have explicit + ceilings with stable XLSX resource failures; and +- every machine result is wrapped in `office.output/1` and carries one of + `office.xlsx.outline/1`, `office.xlsx.element/1`, `office.xlsx.text/1`, or + `office.xlsx.query/1`; capability records expose separate DOCX and XLSX + variants with the exact result schemas. + +Native and Wasm tests exercise the same async filesystem and package dispatch +paths, with Cram coverage for the public schema, canonical selectors, +pagination, query predicates, and cross-format mismatch failures. See +[office-xlsx-read.md](office-xlsx-read.md) for the complete contract. ## Deferred beyond major parity diff --git a/docs/office-selectors.md b/docs/office-selectors.md index 80fcf685..ea5cf95f 100644 --- a/docs/office-selectors.md +++ b/docs/office-selectors.md @@ -93,9 +93,11 @@ adapt syntax only: they do not open a package or establish that an addressed object exists. Format records in `office help --json` expose the selector schema, root, -examples, and the resolver status for that format. DOCX is `read-resolved`: -`office outline`, `get`, `text`, and `query` all resolve the same bounded -projection. XLSX remains `syntax-only` until its structured resolver lands. +examples, and the resolver status for that format. Both DOCX and XLSX are +`read-resolved`: `office outline`, `get`, `text`, and `query` dispatch from the +validated package format and resolve the applicable bounded projection. A +selector with the other format's root fails with a format-specific mismatch; +it never falls through to the other resolver. The DOCX resolver covers body, header, footer, footnote, endnote, and comment stories plus paragraphs, runs, tables, rows, cells, hyperlinks, and images. @@ -105,3 +107,12 @@ unrepresentable ids emit positional paths and a bounded warning instead; asking for a duplicated stable id fails as ambiguous. Every descendant of an annotation item remains snapshot-relative because it contains positional segments. + +The XLSX resolver covers the workbook singleton, worksheets, chart sheets, and +worksheet cell/range coordinates. A positional sheet selector such as +`/xlsx/sheet[2]` is accepted as snapshot-relative input, but successful output +uses the stable name-keyed sheet path. Coordinate endpoints are normalized to +canonical uppercase A1 form; their paths remain snapshot-relative. Chart +sheets can be resolved and inspected as sheets, but reject cell/range +descendants. See [office-xlsx-read.md](office-xlsx-read.md) for scan ordering, +query predicates, and resource limits. diff --git a/docs/office-xlsx-read.md b/docs/office-xlsx-read.md new file mode 100644 index 00000000..efec8d42 --- /dev/null +++ b/docs/office-xlsx-read.md @@ -0,0 +1,185 @@ +# Unified Office XLSX reads + +The `office` executable exposes four read-only XLSX commands over one bounded +workbook projection: + +```sh +moon run office/cmd/office -- outline book.xlsx --json +moon run office/cmd/office -- get book.xlsx '/xlsx/sheet[name="Data"]/range[A1:C12]' --json +moon run office/cmd/office -- text book.xlsx --under '/xlsx/sheet[name="Data"]' --json +moon run office/cmd/office -- query book.xlsx 'cell[type=formula]' --under '/xlsx/sheet[name="Data"]' --json +``` + +The command reads the file once with `moonbitlang/async`, validates the package +format from its OOXML relationships, and passes the same provenance-checked, +bounded archive to the XLSX parser. It does not use C stubs. A `.docx` package +selects the DOCX result contract instead; an extension/content mismatch fails +before either projection is opened. + +Run `office help xlsx` for the installed command catalog and +`office help --json` for the declared format variants, inputs, +outputs, and limits. + +## Output envelope + +`--json` emits exactly one `office.output/1` document. Successful XLSX reads +carry one of these data schemas: + +| command | data schema | purpose | +| --- | --- | --- | +| `outline` | `office.xlsx.outline/1` | workbook, worksheet, chart-sheet, used-range, feature-count, and defined-name metadata | +| `get` | `office.xlsx.element/1` | one workbook, sheet, cell, or populated-cell range projection | +| `text` | `office.xlsx.text/1` | path-tagged displayed cell text with pagination | +| `query` | `office.xlsx.query/1` | row-major cell matches for bounded declared predicates | + +Human output is terminal-safe and intended for inspection. JSON is the +automation contract. + +## Canonical selectors and ordering + +The resolver accepts the canonical `office.selector/1` XLSX shapes: + +```text +/xlsx/workbook +/xlsx/sheet[name="Data"] +/xlsx/sheet[2] +/xlsx/sheet[name="Data"]/cell[B2] +/xlsx/sheet[name="Data"]/range[A1:C12] +``` + +Positional sheet input resolves to a stable, name-keyed output path. Cell and +range endpoints are normalized and emitted in uppercase A1 form. Workbook and +named-sheet selectors are reported as stable; cell/range selectors are +snapshot-relative because row or column edits can move their content. A cell +or range cannot descend from a chart sheet. + +Whole-workbook scans visit sheets in tab order, then cells in row-major order. +`--under` accepts a workbook, worksheet, cell, or range selector. Chart sheets +remain visible in metadata but contribute no cells to text or query scans. + +## `outline` + +```text +office outline FILE [--max-elements N] [--max-output-chars N] [--json] +``` + +The outline includes `/xlsx/workbook`, the active sheet when present, tab-order +sheet summaries, defined names, and the effective limits. Worksheet summaries +report state (`visible`, `hidden`, or `very_hidden`), maximum parsed row and +column, a canonical used-range selector, and counts for merges, tables, charts, +images, pivots, comments, hyperlinks, validations, conditional-format ranges, +and slicers. Chart sheets report their kind, state, and chart count. + +## `get` + +```text +office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json] +``` + +Workbook and sheet reads return bounded orientation metadata. Cell records can +contain: + +- canonical `path`, A1 `reference`, and 1-based `row` and `column`; +- displayed `value`; +- typed `raw` value (`string`, `number`, `bool`, or `error`); +- formula text without a leading `=`; and +- nonzero effective `style_id`. + +Cell and range results include a `styles` object keyed by the referenced style +ids. Range reads omit completely blank, unstyled cells while preserving +row-major order. Selecting one blank, unstyled coordinate fails with +`office.xlsx.selector_not_found`; it does not fabricate a cell record. + +## `text` + +```text +office text FILE [--under SELECTOR] [--offset N] [--limit N] + [--max-elements N] [--max-output-chars N] [--json] +``` + +Each entry contains `{path, stability, text}`. Text is the workbook's displayed +cell value. If a formula has no cached displayed value, text falls back to the +formula prefixed with `=` so an uncached formula is not silently omitted. +`matched_total`, `returned`, `offset`, `limit`, `truncated`, and +`scanned_cells` describe the completed bounded scan exactly. + +## `query` + +```text +office query FILE [CELL_SELECTOR] [--under SELECTOR] + [--offset N] [--limit N] + [--max-elements N] [--max-output-chars N] [--json] +``` + +`CELL_SELECTOR` defaults to `cell`. It is `cell` followed by zero or more +bracketed predicates; all predicates are ANDed: + +| predicate | meaning | +| --- | --- | +| `type=formula` | cell has a formula | +| `type=number|string|bool|error` | typed raw value has that kind | +| `formula` | cell has a formula | +| `formula~=TEXT` | formula contains the literal text | +| `text=TEXT` | raw string value equals the literal text | +| `text~=TEXT` | raw string value contains the literal text | +| `value>NUMBER` | numeric raw value comparison; also supports `>=`, `<`, `<=`, `=`, and `!=` | + +Examples: + +```text +cell[type=formula][formula~=SUM] +cell[type=number][value>=0] +cell[text~=revenue] +``` + +Regular expressions, arbitrary expressions, locale-sensitive matching, and +the DOCX-only `--kind`, `--text`, `--id`, `--property`, and `--ignore-case` +options are rejected. Literal substring predicates are compiled once and use +guaranteed-linear KMP matching under a command-wide work budget. + +## Limits + +The principal read limits are: + +| resource | default | hard limit | +| --- | ---: | ---: | +| scanned cells (`--max-elements`) | 50,000 | 100,000 effective XLSX ceiling | +| successful stdout characters | 1,048,576 | 4,194,304 | +| text rows per page | 2,000 | 10,000 | +| query matches per page | 100 | 1,000 | +| one cell string/formula | — | 1,048,576 characters | +| aggregate scanned cell strings | — | 16,777,216 characters | +| metadata items | — | 10,000 | +| aggregate metadata strings | — | 8,388,608 characters | +| query selector | — | 4,096 characters | +| query predicates | — | 16 | +| one query predicate value | — | 1,024 characters | +| query predicate work | — | 134,217,728 units | + +The shared package boundary additionally caps the input file at 64 MiB, ZIP +entries at 4,096, one inflated entry at 32 MiB, total inflation at 128 MiB, +preserved source bytes at 64 MiB plus the maximum ZIP comment (65,535 bytes), +and one XML part at 16 MiB. Scan rectangles +are preflighted before iteration, and every visited coordinate is charged +again while its snapshot is produced. `--max-output-chars` includes the +successful command's trailing line feed; a failure envelope is independent so +resource exhaustion remains machine-readable. + +## Correctable failures + +Important stable codes include: + +| code | meaning | +| --- | --- | +| `office.xlsx.selector_not_found` | sheet or cell selector does not exist in this snapshot | +| `office.xlsx.selector_format_mismatch` | a `/docx/...` selector was passed to an XLSX read | +| `office.xlsx.unsupported_sheet_kind` | a cell/range selector targeted a chart sheet | +| `office.xlsx.invalid_query` | the cell selector or predicate is invalid | +| `office.xlsx.query_predicate_limit` | more than 16 predicates were supplied | +| `office.xlsx.unsupported_query_options` | DOCX-only query options were supplied for XLSX | +| `office.xlsx.resource_limit` | an explicit package, scan, string, query-work, metadata, or output ceiling was reached | +| `office.xlsx.read_failed` | workbook structure could not be projected consistently | +| `office.selector.*` | canonical selector syntax or shape is invalid | + +Package corruption and extension/content mismatch use the shared +`office.invalid_package` and `office.format_mismatch` families. diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index 466a4f18..11a032a3 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -17,7 +17,7 @@ and JSONL inventories without deferred PowerPoint or MCP entries. $ office.exe help | sed -n '1,8p' Office capability registry Schema: office.capabilities/2 - Fingerprint: crc32:31aeb371 + Fingerprint: crc32:e9c5b34f Formats: docx (aliases: word) — WordprocessingML documents xlsx (aliases: excel) — SpreadsheetML workbooks @@ -36,11 +36,14 @@ and JSONL inventories without deferred PowerPoint or MCP entries. $ office.exe help docx --json | jq -c '.data.records[0] | {kind,name,selector}' {"kind":"format","name":"docx","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} + $ office.exe help xlsx query --json | jq -c '.data.records[0] | {formats,variants:[.variants[]|{name,result_schema,constraints}]}' + {"formats":["docx","xlsx"],"variants":[{"name":"docx","result_schema":"office.docx.query/1","constraints":["format=docx"]},{"name":"xlsx","result_schema":"office.xlsx.query/1","constraints":["format=xlsx"]}]} + $ office.exe help all --json | jq -c '{schema,success,capability_schema:.data.schema,fingerprint:.data.fingerprint,names:[.data.records[].name]}' - {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:31aeb371","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} + {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:e9c5b34f","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} $ office.exe help all --jsonl | jq -s -c 'map({schema,fingerprint,kind,name})' - [{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:31aeb371","kind":"command","name":"raw"}] + [{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"raw"}] The raw command publishes explicit subcommand schemas, including every edit input and its conditional constraints. @@ -92,7 +95,7 @@ valid internal-anchor hyperlink fixture without relying on an external file. {"paths":["/docx/body/p[1]/hyperlink[1]"],"matched_total":1} Pagination and all user-controlled scan/output ceilings are explicit. Selector -syntax, missing paths, and XLSX routing retain stable machine-readable codes. +syntax and missing paths retain stable machine-readable codes. $ office.exe text "$TESTDIR/../../../../docx2html/tests/cram/fixtures/single-paragraph.docx" --limit 0 --json | jq -c '{matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated}' {"matched_total":1,"returned":0,"truncated":true} @@ -117,10 +120,34 @@ syntax, missing paths, and XLSX routing retain stable machine-readable codes. $ jq -c '{success,code:.error.code,resource:.error.details.resource,limit:.error.details.limit}' output-limit.json {"success":false,"code":"office.docx.resource_limit","resource":"successful command output characters","limit":40} - $ office.exe outline "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --json > xlsx-structured.json 2>&1; echo $? +Structured XLSX reads use the same commands and envelope. Positional sheet +input canonicalizes to stable name paths; ranges, text, and query scan in +tab/row/column order with exact totals. + + $ office.exe outline "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --json | jq -c '{success,schema:.data.schema,path:.data.path,sheet_count:.data.sheet_count,active:.data.active_sheet.path,sheets:[.data.sheets[]|{path,kind,state,used:.used_range.reference}]}' + {"success":true,"schema":"office.xlsx.outline/1","path":"/xlsx/workbook","sheet_count":2,"active":"/xlsx/sheet[name=\"Sheet1\"]","sheets":[{"path":"/xlsx/sheet[name=\"Sheet1\"]","kind":"worksheet","state":"visible","used":"A1:D22"},{"path":"/xlsx/sheet[name=\"Sheet2\"]","kind":"worksheet","state":"visible","used":"A1:I11"}]} + + $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" '/xlsx/sheet[1]/range[A19:B19]' --json | jq -c '{schema:.data.schema,path:.data.path,kind:.data.kind,refs:[.data.cells[].reference],raw:[.data.cells[].raw],formulas:[.data.cells[]|(.formula // null)],returned:.data.returned}' + {"schema":"office.xlsx.element/1","path":"/xlsx/sheet[name=\"Sheet1\"]/range[A19:B19]","kind":"range","refs":["A19","B19"],"raw":[{"type":"string","value":"Total:"},{"type":"number","value":237}],"formulas":[null,"SUM(Sheet2!D2,Sheet2!D11)"],"returned":2} + + $ office.exe text "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --under '/xlsx/sheet[name="Sheet1"]' --offset 1 --limit 2 --json | jq -c '{schema:.data.schema,under:.data.under,paths:[.data.entries[].path],texts:[.data.entries[].text],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' + {"schema":"office.xlsx.text/1","under":"/xlsx/sheet[name=\"Sheet1\"]","paths":["/xlsx/sheet[name=\"Sheet1\"]/cell[B19]","/xlsx/sheet[name=\"Sheet1\"]/cell[C21]"],"texts":["237","Column1"],"matched_total":5,"returned":2,"truncated":true,"scanned":88} + + $ office.exe query "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json | jq -c '{schema:.data.schema,selector:.data.selector,under:.data.under,paths:[.data.matches[].path],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' + {"schema":"office.xlsx.query/1","selector":"cell[type=formula][formula~=IF]","under":"/xlsx/sheet[name=\"Sheet2\"]","paths":["/xlsx/sheet[name=\"Sheet2\"]/cell[F11]","/xlsx/sheet[name=\"Sheet2\"]/cell[G11]"],"matched_total":2,"returned":2,"truncated":false,"scanned":99} + +Cross-format selectors and DOCX-only XLSX query flags fail with XLSX-specific, +machine-correctable codes. + + $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" '/docx/body' --json > xlsx-selector-mismatch.json 2>&1; echo $? + 1 + $ jq -c '{success,code:.error.code,expected:.error.details.expected_format,actual:.error.details.actual_format}' xlsx-selector-mismatch.json + {"success":false,"code":"office.xlsx.selector_format_mismatch","expected":"xlsx","actual":"docx"} + + $ office.exe query "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --kind cell --json > xlsx-query-options.json 2>&1; echo $? 1 - $ jq -c '{success,code:.error.code,format:.error.details.format}' xlsx-structured.json - {"success":false,"code":"office.unsupported_operation","format":"xlsx"} + $ jq -c '{success,code:.error.code,options:.error.details.options}' xlsx-query-options.json + {"success":false,"code":"office.xlsx.unsupported_query_options","options":["--kind"]} Extension/content mismatches and malformed input fail non-zero. From 1dec18cbc0ee998853748ca740cd944e8f15ff98 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:37:01 +0800 Subject: [PATCH 007/150] test(office): keep byte projection helper test-only --- office/cmd/office/docx_projection.mbt | 19 ------------------- office/cmd/office/docx_read_wbtest.mbt | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/office/cmd/office/docx_projection.mbt b/office/cmd/office/docx_projection.mbt index b019fce3..3e6e8da8 100644 --- a/office/cmd/office/docx_projection.mbt +++ b/office/cmd/office/docx_projection.mbt @@ -1006,25 +1006,6 @@ fn validate_docx_read_archive( } } -///| -fn open_docx_projection( - file : String, - data : BytesView, - max_elements : Int, -) -> DocxProjection raise CliFailure { - let archive = @zip.read_limited( - data, - max_package_bytes=docx_cli_max_package_bytes, - max_entries=docx_cli_max_zip_entries, - max_entry_uncompressed_bytes=docx_cli_max_zip_entry_bytes, - max_total_uncompressed_bytes=docx_cli_max_zip_total_bytes, - max_total_preserved_source_bytes=docx_cli_max_package_bytes, - ) catch { - error => raise docx_zip_failure(error, file) - } - open_docx_projection_archive(file, archive, max_elements) -} - ///| fn open_docx_projection_archive( file : String, diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index b908371e..694a5774 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -51,6 +51,25 @@ fn docx_read_test_bytes() -> Bytes raise { } } +///| +fn open_docx_projection( + file : String, + data : BytesView, + max_elements : Int, +) -> DocxProjection raise CliFailure { + let archive = @zip.read_limited( + data, + max_package_bytes=docx_cli_max_package_bytes, + max_entries=docx_cli_max_zip_entries, + max_entry_uncompressed_bytes=docx_cli_max_zip_entry_bytes, + max_total_uncompressed_bytes=docx_cli_max_zip_total_bytes, + max_total_preserved_source_bytes=docx_cli_max_package_bytes, + ) catch { + error => raise docx_zip_failure(error, file) + } + open_docx_projection_archive(file, archive, max_elements) +} + ///| fn docx_read_test_projection() -> DocxProjection raise { open_docx_projection("fixture.docx", docx_read_test_bytes(), 1000) From f056d4c80da277b67e3e17d142d7034ad0800d99 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:37:44 +0800 Subject: [PATCH 008/150] refactor(office): retire duplicate docx zip limits --- office/cmd/office/docx_projection.mbt | 36 -------------------------- office/cmd/office/docx_read_wbtest.mbt | 12 ++++----- 2 files changed, 6 insertions(+), 42 deletions(-) diff --git a/office/cmd/office/docx_projection.mbt b/office/cmd/office/docx_projection.mbt index 3e6e8da8..72bc027a 100644 --- a/office/cmd/office/docx_projection.mbt +++ b/office/cmd/office/docx_projection.mbt @@ -1,15 +1,3 @@ -///| -let docx_cli_max_package_bytes : Int = 64 * 1024 * 1024 - -///| -let docx_cli_max_zip_entries : Int = 4096 - -///| -let docx_cli_max_zip_entry_bytes : Int = 32 * 1024 * 1024 - -///| -let docx_cli_max_zip_total_bytes : Int = 128 * 1024 * 1024 - ///| let docx_cli_max_xml_source_units : Int = 64 * 1024 * 1024 @@ -116,30 +104,6 @@ fn docx_resource_failure( format_resource_failure("docx", resource, limit, actual?) } -///| -fn docx_zip_failure(error : @zip.ZipError, file : String) -> CliFailure { - match error { - ResourceLimitExceeded(kind~, limit~, actual~) => - docx_cli_failure( - "office.docx.resource_limit", - "DOCX ZIP \{bounded_text(kind, 80)} exceeds the configured limit", - details=Json::object({ - "file": Json::string(bounded_text(file, 160)), - "resource": Json::string(bounded_text(kind, 80)), - "limit": Json::number(limit.to_double()), - "actual": Json::number(actual.to_double()), - }), - ) - OutputLimitExceeded(limit~) => docx_resource_failure("ZIP output", limit) - _ => - docx_cli_failure( - "office.invalid_package", - "invalid DOCX package: archive is not a readable bounded ZIP", - details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), - ) - } -} - ///| fn docx_reader_failure( error : @docx_core.DocxError, diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index 694a5774..cc1e26a3 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -59,13 +59,13 @@ fn open_docx_projection( ) -> DocxProjection raise CliFailure { let archive = @zip.read_limited( data, - max_package_bytes=docx_cli_max_package_bytes, - max_entries=docx_cli_max_zip_entries, - max_entry_uncompressed_bytes=docx_cli_max_zip_entry_bytes, - max_total_uncompressed_bytes=docx_cli_max_zip_total_bytes, - max_total_preserved_source_bytes=docx_cli_max_package_bytes, + max_package_bytes=office_read_max_package_bytes, + max_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, ) catch { - error => raise docx_zip_failure(error, file) + error => raise office_zip_read_failure(error, file, Docx) } open_docx_projection_archive(file, archive, max_elements) } From bbfb430f0c6ab720245f6fd5f15907723d9ae80c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 04:59:52 +0800 Subject: [PATCH 009/150] style(office): format capability snapshots --- office/capabilities_test.mbt | 3 --- 1 file changed, 3 deletions(-) diff --git a/office/capabilities_test.mbt b/office/capabilities_test.mbt index cb2f93b9..6f0c7701 100644 --- a/office/capabilities_test.mbt +++ b/office/capabilities_test.mbt @@ -226,7 +226,6 @@ test "capability registry format filters remain truthful" { #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.text/1|office.xlsx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office text FILE [--under /xlsx/...]","result_schema":"office.xlsx.text/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: cell followed by up to 16 bounded type, value, formula, or text predicates; default cell"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.query/1|office.xlsx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: supplied bounded cell selector"},{"name":"styles","type":"object","required":false,"description":"XLSX only: deduplicated styles referenced by returned matches"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office query FILE [cell[predicate]...]","result_schema":"office.xlsx.query/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} - ), ) } @@ -237,7 +236,6 @@ test "capability fingerprint and inventory ordering are deterministic" { @office.capability_fingerprint(), content=( #|crc32:e9c5b34f - ), ) let first = @office.capabilities_data().stringify() @@ -254,7 +252,6 @@ test "capability operation queries do not leak unrelated records" { data.stringify(), content=( #|{"schema":"office.capabilities/2","fingerprint":"crc32:e9c5b34f","records":[{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} - ), ) } From 0bcb5346c726c63adbaef0a7aa2b06a181f019d8 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 05:40:18 +0800 Subject: [PATCH 010/150] fix(office): close xlsx read review gaps --- .github/workflows/ci.yml | 9 ++ office/cmd/office/cram/cli.t | 8 ++ office/cmd/office/xlsx_read_model.mbt | 150 ++++++++++++------------ office/cmd/office/xlsx_read_output.mbt | 61 ++++------ office/cmd/office/xlsx_read_wbtest.mbt | 92 +++++++++++++++ xlsx/pkg.generated.mbti | 3 + xlsx/read_coordinate_hardening_test.mbt | 13 ++ xlsx/read_worksheet_xml.mbt | 11 +- xlsx/workbook.mbt | 37 ++++++ xlsx/worksheet.mbt | 25 ++++ xlsx/worksheet_cell_index.mbt | 22 ++++ xlsx/worksheet_types.mbt | 3 +- 12 files changed, 315 insertions(+), 119 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3fc9baa..eaf82d62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,15 @@ jobs: and .data.schema == "office.xlsx.outline/1" and .data.path == "/xlsx/workbook" and [.data.sheets[].path] == ["/xlsx/sheet[name=\"Sheet1\"]", "/xlsx/sheet[name=\"Sheet2\"]"]' + singleton_fixture=fixtures/excelize/test/OverflowNumericCell.xlsx + xs=$(moon run --target wasm office/cmd/office -- outline "$singleton_fixture" --json 2>office-wasm.err) || { echo "office XLSX singleton outline failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xs" | jq -e '.success == true + and .data.sheets[0].used_range.reference == "A1:A1" + and .data.sheets[0].used_range.path == "/xlsx/sheet[name=\"Sheet1\"]/range[A1:A1]"' + xsg=$(moon run --target wasm office/cmd/office -- get "$singleton_fixture" '/xlsx/sheet[1]/range[A1:A1]' --json 2>office-wasm.err) || { echo "office XLSX singleton get failed; stderr:"; cat office-wasm.err; exit 1; } + echo "$xsg" | jq -e '.success == true + and .data.reference == "A1:A1" + and [.data.cells[].reference] == ["A1"]' xg=$(moon run --target wasm office/cmd/office -- get "$xlsx_fixture" '/xlsx/sheet[1]/range[A19:B19]' --json 2>office-wasm.err) || { echo "office XLSX get failed; stderr:"; cat office-wasm.err; exit 1; } echo "$xg" | jq -e '.success == true and .data.schema == "office.xlsx.element/1" diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index 11a032a3..d4bcca25 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -127,6 +127,14 @@ tab/row/column order with exact totals. $ office.exe outline "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --json | jq -c '{success,schema:.data.schema,path:.data.path,sheet_count:.data.sheet_count,active:.data.active_sheet.path,sheets:[.data.sheets[]|{path,kind,state,used:.used_range.reference}]}' {"success":true,"schema":"office.xlsx.outline/1","path":"/xlsx/workbook","sheet_count":2,"active":"/xlsx/sheet[name=\"Sheet1\"]","sheets":[{"path":"/xlsx/sheet[name=\"Sheet1\"]","kind":"worksheet","state":"visible","used":"A1:D22"},{"path":"/xlsx/sheet[name=\"Sheet2\"]","kind":"worksheet","state":"visible","used":"A1:I11"}]} +Singleton extents retain two endpoints, so every emitted range path parses and +round-trips through the public selector grammar. + + $ office.exe outline "$TESTDIR/../../../../fixtures/excelize/test/OverflowNumericCell.xlsx" --json | jq -c '{reference:.data.sheets[0].used_range.reference,path:.data.sheets[0].used_range.path}' + {"reference":"A1:A1","path":"/xlsx/sheet[name=\"Sheet1\"]/range[A1:A1]"} + $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/OverflowNumericCell.xlsx" '/xlsx/sheet[1]/range[A1:A1]' --json | jq -c '{path:.data.path,reference:.data.reference,refs:[.data.cells[].reference]}' + {"path":"/xlsx/sheet[name=\"Sheet1\"]/range[A1:A1]","reference":"A1:A1","refs":["A1"]} + $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" '/xlsx/sheet[1]/range[A19:B19]' --json | jq -c '{schema:.data.schema,path:.data.path,kind:.data.kind,refs:[.data.cells[].reference],raw:[.data.cells[].raw],formulas:[.data.cells[]|(.formula // null)],returned:.data.returned}' {"schema":"office.xlsx.element/1","path":"/xlsx/sheet[name=\"Sheet1\"]/range[A19:B19]","kind":"range","refs":["A19","B19"],"raw":[{"type":"string","value":"Total:"},{"type":"number","value":237}],"formulas":[null,"SUM(Sheet2!D2,Sheet2!D11)"],"returned":2} diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 95bc4442..28d2df1a 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -31,7 +31,8 @@ struct XlsxSheetTarget { struct XlsxProjection { file : String workbook : @xlsx.Workbook - sheet_names : Array[String] + sheets : Array[XlsxSheetTarget] + sheet_index : Map[String, Int] max_scan_cells : Int } @@ -181,10 +182,53 @@ fn make_xlsx_projection( }), ) } + let worksheets : Map[String, @xlsx.Worksheet] = Map([]) + for worksheet in workbook.sheets() { + worksheets[worksheet.name()] = worksheet + } + let chart_sheets : Map[String, @xlsx.ChartSheet] = Map([]) + for chart_sheet in workbook.chart_sheets() { + chart_sheets[chart_sheet.name()] = chart_sheet + } + let sheets : Array[XlsxSheetTarget] = [] + let sheet_index : Map[String, Int] = Map([]) + for index, name in workbook.get_sheet_list() { + if sheet_index.contains(name) { + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook tab order contains a duplicate sheet name", + details=Json::object({ "sheet": Json::string(bounded_text(name, 160)) }), + ) + } + let content = match worksheets.get(name) { + Some(worksheet) => Worksheet(worksheet) + None => + match chart_sheets.get(name) { + Some(chart_sheet) => ChartSheet(chart_sheet) + None => + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook tab order contains an unresolved sheet", + details=Json::object({ + "sheet": Json::string(bounded_text(name, 160)), + }), + ) + } + } + let target : XlsxSheetTarget = { + name, + index, + path: canonical_xlsx_sheet_path(name), + content, + } + sheet_index[name] = index + sheets.push(target) + } { file, workbook, - sheet_names: workbook.get_sheet_list(), + sheets, + sheet_index, max_scan_cells: max_elements.min(xlsx_cli_hard_max_scan_cells), } } @@ -205,21 +249,23 @@ fn XlsxRect::cell_count(self : XlsxRect) -> Int64 { } ///| -fn XlsxRect::reference( +fn XlsxRect::cell_reference( self : XlsxRect, file : String, ) -> String raise CliFailure { - let first = xlsx_call(file, () => { - @xlsx.coordinates_to_cell_name(self.col_lo, self.row_lo) + xlsx_call(file, () => @xlsx.coordinates_to_cell_name(self.col_lo, self.row_lo)) +} + +///| +fn XlsxRect::range_reference( + self : XlsxRect, + file : String, +) -> String raise CliFailure { + let first = self.cell_reference(file) + let last = xlsx_call(file, () => { + @xlsx.coordinates_to_cell_name(self.col_hi, self.row_hi) }) - if self.col_lo == self.col_hi && self.row_lo == self.row_hi { - first - } else { - let last = xlsx_call(file, () => { - @xlsx.coordinates_to_cell_name(self.col_hi, self.row_hi) - }) - first + ":" + last - } + first + ":" + last } ///| @@ -272,43 +318,10 @@ fn XlsxProjection::check_scan_rect( fn XlsxProjection::find_sheet_by_name( self : XlsxProjection, name : String, -) -> XlsxSheetTarget? raise CliFailure { - let mut found_index = -1 - for index, candidate in self.sheet_names { - if candidate == name { - found_index = index - break - } - } - if found_index < 0 { - return None - } - let path = canonical_xlsx_sheet_path(name) - match self.workbook.sheet(name) { - Some(sheet) => - Some({ name, index: found_index, path, content: Worksheet(sheet) }) - None => - match self.workbook.chart_sheet(name) { - Some(sheet) => - Some({ name, index: found_index, path, content: ChartSheet(sheet) }) - None => None - } - } -} - -///| -fn XlsxProjection::require_sheet_by_name( - self : XlsxProjection, - name : String, -) -> XlsxSheetTarget raise CliFailure { - match self.find_sheet_by_name(name) { - Some(sheet) => sheet - None => - raise xlsx_cli_failure( - "office.xlsx.read_failed", - "workbook tab order contains an unresolved sheet", - details=Json::object({ "sheet": Json::string(bounded_text(name, 160)) }), - ) +) -> XlsxSheetTarget? { + match self.sheet_index.get(name) { + Some(index) => self.sheets.get(index) + None => None } } @@ -322,11 +335,7 @@ fn XlsxProjection::resolve_sheet_segment( Some(name) => self.find_sheet_by_name(name) None => match segment.position() { - Some(position) => - match self.sheet_names.get(position - 1) { - Some(name) => self.find_sheet_by_name(name) - None => None - } + Some(position) => self.sheets.get(position - 1) None => None } } @@ -393,20 +402,23 @@ fn resolve_xlsx_selector( ignore(sheet.worksheet(selector.render())) let rect = xlsx_rect_from_coordinate(coordinate) projection.check_scan_rect(rect) - let reference = rect.reference(projection.file) match coordinate { - Cell(_) => + Cell(_) => { + let reference = rect.cell_reference(projection.file) Cell( sheet, rect, path=canonical_xlsx_cell_path(sheet.name, reference), ) - Range(_, _) => + } + Range(_, _) => { + let reference = rect.range_reference(projection.file) Range( sheet, rect, path=canonical_xlsx_range_path(sheet.name, reference), ) + } } } } @@ -440,8 +452,7 @@ fn xlsx_scan_regions( ChartSheet(_) => () } Some(Workbook(..)) | None => - for name in projection.sheet_names { - let sheet = projection.require_sheet_by_name(name) + for sheet in projection.sheets { match sheet.content { Worksheet(worksheet) => match xlsx_used_rect(worksheet) { @@ -544,37 +555,32 @@ fn xlsx_cell_snapshot( budget : XlsxScanBudget, ) -> XlsxCellSnapshot raise CliFailure { budget.charge_cell() + let worksheet = sheet.worksheet(sheet.path) let reference = xlsx_call(projection.file, () => { @xlsx.coordinates_to_cell_name(column, row) }) let raw = xlsx_call(projection.file, () => { - projection.workbook.get_cell_value_raw(sheet.name, reference) + worksheet.get_cell_value_raw(reference) }) let normalized_raw = match raw { Some(String("")) => None value => value } let formula = match - xlsx_call(projection.file, () => { - projection.workbook.get_cell_formula(sheet.name, reference) - }) { + xlsx_call(projection.file, () => worksheet.get_cell_formula(reference)) { Some("") => None value => value } let style_id = xlsx_call(projection.file, () => { - @inspect.effective_style_id( - projection.workbook, - sheet.name, - reference, - row~, - col=column, - ) + worksheet.effective_style_id_rc(row, column) }) let formatted = match normalized_raw { Some(_) => Some( xlsx_call(projection.file, () => { - projection.workbook.get_cell_value(sheet.name, reference) + projection.workbook.get_cell_value_from_worksheet_rc( + worksheet, row, column, + ) }).unwrap_or(""), ) None => None diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index e1a5fae5..8f1c28bc 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -25,12 +25,10 @@ fn xlsx_sheet_state_name(state : @xlsx.SheetState) -> String { ///| fn xlsx_conditional_format_range_count( - projection : XlsxProjection, - sheet : String, + file : String, + worksheet : @xlsx.Worksheet, ) -> Int raise CliFailure { - let formats = xlsx_call(projection.file, () => { - projection.workbook.get_conditional_formats(sheet) - }) + let formats = xlsx_call(file, () => worksheet.get_conditional_formats()) let mut count = 0 for ranges, _ in formats { for part in ranges.split(" ") { @@ -69,7 +67,7 @@ fn xlsx_sheet_summary_json( fields["max_column"] = Json::number(max_col.to_double()) match xlsx_used_rect(worksheet) { Some(rect) => { - let reference = rect.reference(projection.file) + let reference = rect.range_reference(projection.file) let path = canonical_xlsx_range_path(sheet.name, reference) metadata.charge_item([reference, path]) fields["used_range"] = Json::object({ @@ -80,27 +78,13 @@ fn xlsx_sheet_summary_json( } None => () } - let merges = xlsx_call(projection.file, () => { - projection.workbook.get_merge_cells(sheet.name) - }) - let tables = xlsx_call(projection.file, () => { - projection.workbook.get_tables(sheet.name) - }) - let comments = xlsx_call(projection.file, () => { - projection.workbook.get_comments(sheet.name) - }) - let hyperlinks = xlsx_call(projection.file, () => { - projection.workbook.get_hyperlink_cells(sheet.name) - }) - let validations = xlsx_call(projection.file, () => { - projection.workbook.get_data_validations(sheet.name) - }) - let pivots = xlsx_call(projection.file, () => { - projection.workbook.get_pivot_tables(sheet.name) - }) - let slicers = xlsx_call(projection.file, () => { - projection.workbook.get_slicers(sheet.name) - }) + let merges = worksheet.merged_cells() + let tables = worksheet.tables() + let comments = worksheet.comments() + let hyperlinks = worksheet.get_hyperlink_cells() + let validations = worksheet.data_validations() + let pivots = worksheet.pivot_tables() + let slicers = worksheet.slicers() fields["counts"] = Json::object({ "merges": Json::number(merges.length().to_double()), "tables": Json::number(tables.length().to_double()), @@ -111,7 +95,7 @@ fn xlsx_sheet_summary_json( "hyperlinks": Json::number(hyperlinks.length().to_double()), "data_validations": Json::number(validations.length().to_double()), "conditional_format_ranges": Json::number( - xlsx_conditional_format_range_count(projection, sheet.name).to_double(), + xlsx_conditional_format_range_count(projection.file, worksheet).to_double(), ), "slicers": Json::number(slicers.length().to_double()), }) @@ -124,8 +108,7 @@ fn xlsx_sheet_summary_json( fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { let metadata = XlsxMetadataBudget::new() let sheets : Array[Json] = [] - for name in projection.sheet_names { - let sheet = projection.require_sheet_by_name(name) + for sheet in projection.sheets { sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) } let defined_names : Array[Json] = [] @@ -164,21 +147,18 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { ), }), } - if !projection.sheet_names.is_empty() { + if !projection.sheets.is_empty() { let active_index = projection.workbook.get_active_sheet_index() - guard projection.sheet_names.get(active_index) is Some(name) else { + guard projection.sheets.get(active_index) is Some(sheet) else { raise xlsx_cli_failure( "office.xlsx.read_failed", "workbook active-sheet index is outside the tab order", details=Json::object({ "active_sheet_index": Json::number(active_index.to_double()), - "sheet_count": Json::number( - projection.sheet_names.length().to_double(), - ), + "sheet_count": Json::number(projection.sheets.length().to_double()), }), ) } - let sheet = projection.require_sheet_by_name(name) fields["active_sheet"] = Json::object({ "path": Json::string(sheet.path), "name": Json::string(sheet.name), @@ -444,7 +424,7 @@ fn xlsx_get_data( "kind": Json::string("range"), "stability": Json::string("snapshot-relative"), "parent": Json::string(sheet.path), - "reference": Json::string(rect.reference(projection.file)), + "reference": Json::string(rect.range_reference(projection.file)), "cells": Json::array(page.cells.map(xlsx_cell_json)), "styles": xlsx_styles_json(projection, page.cells), "scanned_cells": Json::number(page.scanned_cells.to_double()), @@ -574,9 +554,8 @@ fn xlsx_outline_human( output.write_terminal_safe(projection.file) output.begin_line() output.write_plain("workbook\t/xlsx/workbook\t") - output.write_plain("\{projection.sheet_names.length()} sheets") - for name in projection.sheet_names { - let sheet = projection.require_sheet_by_name(name) + output.write_plain("\{projection.sheets.length()} sheets") + for sheet in projection.sheets { output.begin_line() output.write_terminal_safe(sheet.path) output.write_char('\t') @@ -588,7 +567,7 @@ fn xlsx_outline_human( match xlsx_used_rect(worksheet) { Some(rect) => { output.write_char('\t') - output.write_terminal_safe(rect.reference(projection.file)) + output.write_terminal_safe(rect.range_reference(projection.file)) } None => output.write_plain("\tempty") } diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 647b2ef4..ed33affb 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -40,6 +40,15 @@ test "XLSX selectors resolve to stable named canonical paths" { assert_eq(rect.cell_count(), 4L) assert_eq(path, "/xlsx/sheet[name=\"Data\"]/range[A1:B2]") assert_eq(@lib.parse_selector(path).render(), path) + guard resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/range[B2:B2]", + ) + is Range(_, singleton, path~) else { + fail("expected singleton range selector") + } + assert_eq(singleton.range_reference(projection.file), "B2:B2") + assert_eq(path, "/xlsx/sheet[name=\"Data\"]/range[B2:B2]") + assert_eq(@lib.parse_selector(path).render(), path) } ///| @@ -95,6 +104,28 @@ test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" assert_eq(budget.scanned, 1) } +///| +test "XLSX cell scans keep the resolved worksheet handle" { + let projection = xlsx_read_test_projection() + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]") + is Cell(sheet, rect, ..) else { + fail("expected cell selector") + } + projection.workbook.set_sheet_name("Data", "Renamed") + let snapshot = xlsx_cell_snapshot( + projection, + sheet, + rect.row_lo, + rect.col_lo, + XlsxScanBudget::new(1), + ) + guard snapshot.raw is Some(String(raw)) else { + fail("expected resolved raw string") + } + assert_eq(raw, "alpha") + assert_eq(snapshot.formatted, Some("alpha")) +} + ///| test "XLSX string and metadata accounting fail before unbounded retention" { let per_value = XlsxStringBudget::new(maximum=20, per_value_maximum=3) @@ -216,6 +247,67 @@ test "XLSX outline and get use office schemas with canonical cell records" { assert_eq(missing_error.code, "office.xlsx.selector_not_found") } +///| +test "XLSX singleton used ranges remain canonical two-endpoint ranges" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Only")) + workbook.set_cell_str("Only", "A1", "one") + let projection = make_xlsx_projection("one.xlsx", workbook, 10) + let outline = xlsx_outline_data(projection) + guard outline is Object(fields) && + fields.get("sheets") is Some(Array([Object(sheet), ..])) && + sheet.get("used_range") is Some(Object(used)) else { + fail("expected singleton used range") + } + assert_eq(used.get("reference"), Some(Json::string("A1:A1"))) + assert_eq( + used.get("path"), + Some(Json::string("/xlsx/sheet[name=\"Only\"]/range[A1:A1]")), + ) + let resolved = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Only\"]/range[A1:A1]", + ) + guard xlsx_get_data(projection, resolved) is Object(get) else { + fail("expected singleton range get") + } + assert_eq(get.get("reference"), Some(Json::string("A1:A1"))) +} + +///| +test "XLSX structured reads reject non-finite package numbers" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_int("Data", "A1", 1) + let archive = try! @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/worksheets/sheet1.xml") is Some(sheet_bytes) else { + fail("missing worksheet part") + } + let sheet_xml = try! @utf8.decode(sheet_bytes, ignore_bom=true) + assert_true(sheet_xml.contains("1")) + assert_true( + archive.replace( + "xl/worksheets/sheet1.xml", + @utf8.encode(sheet_xml.replace(old="1", new="NaN")), + ), + ) + let bytes = @zip.write(archive) + let bounded = try! @zip.read_limited( + bytes, + max_package_bytes=office_read_max_package_bytes, + max_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + ) + let source : OfficeReadPackage = { + file: "non-finite.xlsx", + format: Xlsx, + archive: bounded, + } + let error = xlsx_cli_error(() => ignore(open_xlsx_projection(source, 10))) + assert_eq(error.code, "office.invalid_package") +} + ///| test "XLSX text and query scan fully while retaining only the requested page" { let projection = xlsx_read_test_projection() diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 872634a5..4c5bddaa 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1827,6 +1827,7 @@ pub fn Workbook::get_cell_style(Self, StringView, StringView) -> Int? raise Xlsx pub fn Workbook::get_cell_style_rc(Self, StringView, Int, Int) -> Int? raise XlsxError pub fn Workbook::get_cell_type(Self, StringView, StringView) -> CellValueType? raise XlsxError pub fn Workbook::get_cell_value(Self, StringView, StringView, raw? : Bool, options? : Options) -> String? raise XlsxError +pub fn Workbook::get_cell_value_from_worksheet_rc(Self, Worksheet, Int, Int, raw? : Bool, options? : Options) -> String? raise XlsxError pub fn Workbook::get_cell_value_raw(Self, StringView, StringView) -> CellValue? raise XlsxError pub fn Workbook::get_cell_value_raw_rc(Self, StringView, Int, Int) -> CellValue? raise XlsxError pub fn Workbook::get_cell_value_rc(Self, StringView, Int, Int, raw? : Bool, options? : Options) -> String? raise XlsxError @@ -2092,6 +2093,7 @@ pub fn Worksheet::delete_comment(Self, StringView) -> Unit raise XlsxError pub fn Worksheet::delete_data_validation(Self, sqrefs? : Array[String]) -> Unit raise XlsxError pub fn Worksheet::delete_pivot_table(Self, StringView) -> Unit raise XlsxError pub fn Worksheet::delete_table(Self, StringView) -> Unit raise XlsxError +pub fn Worksheet::effective_style_id_rc(Self, Int, Int) -> Int raise XlsxError pub fn Worksheet::formula_refs(Self) -> Array[String] pub fn Worksheet::get_cell(Self, StringView) -> String? raise XlsxError pub fn Worksheet::get_cell_formula(Self, StringView) -> String? raise XlsxError @@ -2159,6 +2161,7 @@ pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxEr pub fn Worksheet::shapes(Self) -> ArrayView[Shape] pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? +pub fn Worksheet::slicers(Self) -> ArrayView[Slicer] pub fn Worksheet::sparkline_groups(Self) -> ArrayView[SparklineGroup] pub fn Worksheet::state(Self) -> SheetState pub fn Worksheet::tables(Self) -> ArrayView[Table] diff --git a/xlsx/read_coordinate_hardening_test.mbt b/xlsx/read_coordinate_hardening_test.mbt index dc160ce6..35fa7816 100644 --- a/xlsx/read_coordinate_hardening_test.mbt +++ b/xlsx/read_coordinate_hardening_test.mbt @@ -105,6 +105,19 @@ fn coordinate_hardening_bounded_archive( ) } +///| +test "read rejects non-finite numeric cell values before typed projection" { + let base = coordinate_hardening_base_bytes() + for value in ["NaN", "Infinity", "1e999999999"] { + let bytes = coordinate_hardening_insert_sheet_xml( + base, + " ", + "\{value}", + ) + coordinate_hardening_assert_invalid_xml("non-finite number \{value}", bytes) + } +} + ///| test "read rejects every out-of-grid worksheet coordinate ingress" { let base = coordinate_hardening_base_bytes() diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index f6e537aa..43226e24 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -193,11 +193,12 @@ fn parse_worksheet( value = "" value_type = String } else { - ignore( - @string.parse_double(raw_value) catch { - _ => raise InvalidXml(msg="cell number invalid") - }, - ) + let number = @string.parse_double(raw_value) catch { + _ => raise InvalidXml(msg="cell number invalid") + } + if number.is_nan() || number.is_inf() { + raise InvalidXml(msg="cell number must be finite") + } value = raw_value value_type = Number } diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index e504c1a3..8462088c 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1816,6 +1816,43 @@ pub fn Workbook::get_cell_value( } } +///| +/// Formats one cell from a worksheet that has already been resolved from this +/// workbook. This is the bulk-read counterpart to `get_cell_value`: callers +/// can resolve a worksheet once, then read many coordinates without repeating +/// the workbook's name lookup for every cell. The cell's own style and the +/// workbook's formatting options have the same semantics as +/// `get_cell_value`. +/// +/// `worksheet` must be an entry from this workbook's `sheets()` view. The row +/// and column are 1-based and must fit the XLSX grid. +pub fn Workbook::get_cell_value_from_worksheet_rc( + self : Workbook, + worksheet : Worksheet, + row : Int, + col : Int, + raw? : Bool = false, + options? : Options, +) -> String? raise XlsxError { + ignore(cell_ref_from(row, col)) + let resolved_options = match options { + Some(value) => value + None => self.options + } + match worksheet.cell_index_of(row, col) { + Some(index) => + Some( + format_cell_value_for_workbook( + self.styles, + worksheet.cells[index], + raw, + resolved_options, + ), + ) + None => None + } +} + ///| pub fn Workbook::calc_cell_value( self : Workbook, diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 6dc67f9a..e50a9474 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -935,6 +935,31 @@ pub fn Worksheet::get_cell_style_rc( } } +///| +/// Returns the style that formats a resolved coordinate without performing a +/// workbook sheet-name lookup. Precedence matches the write path: a non-zero +/// cell style wins, then the row style, then the column style, then style 0. +/// The row and column are 1-based and must fit the XLSX grid. +pub fn Worksheet::effective_style_id_rc( + self : Worksheet, + row : Int, + col : Int, +) -> Int raise XlsxError { + ignore(cell_ref_from(row, col)) + match self.get_cell_style_rc(row, col) { + Some(style_id) if style_id != 0 => style_id + _ => + match self.get_row_style(row) { + Some(style_id) if style_id != 0 => style_id + _ => + match self.get_col_style(col) { + Some(style_id) if style_id != 0 => style_id + _ => 0 + } + } + } +} + ///| /// Merges the rectangular range `range_ref` (e.g. `"A1:C3"`) into a single /// region; the top-left cell's value is displayed across the whole range. The diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index 897abffc..cb31911d 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -79,3 +79,25 @@ test "duplicate coordinates resolve to the first stored cell" { other => fail("unexpected raw value: \{to_repr(other)}") } } + +///| +test "resolved worksheet reads avoid repeated names and preserve style precedence" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("Original") + let cell_style = workbook.add_style(Style::builtin_number_format(2)) + let row_style = workbook.add_style(Style::new()) + let column_style = workbook.add_style(Style::new()) + sheet.set_cell_value_rc(1, 1, Numeric(1.5)) + sheet.set_cell_style_rc(1, 1, cell_style) + workbook.set_row_style("Original", 2, row_style) + workbook.set_col_style("Original", 1, column_style) + workbook.set_sheet_name("Original", "Renamed") + assert_eq( + workbook.get_cell_value_from_worksheet_rc(sheet, 1, 1), + Some("1.50"), + ) + assert_eq(sheet.effective_style_id_rc(1, 1), cell_style) + assert_eq(sheet.effective_style_id_rc(2, 2), row_style) + assert_eq(sheet.effective_style_id_rc(3, 1), column_style) + assert_eq(sheet.effective_style_id_rc(3, 2), 0) +} diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 7aca229e..d149575b 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -227,7 +227,8 @@ fn Worksheet::form_controls(self : Worksheet) -> ArrayView[FormControl] { } ///| -fn Worksheet::slicers(self : Worksheet) -> ArrayView[Slicer] { +/// Returns the slicers attached to this worksheet in stored order. +pub fn Worksheet::slicers(self : Worksheet) -> ArrayView[Slicer] { self.slicers } From 23db2bf0ef85ab385c5519097516713ccde27a2e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 07:13:07 +0800 Subject: [PATCH 011/150] fix(office): harden xlsx read projections --- crypto/moon.pkg | 1 - inspect/style_json.mbt | 7 +- inspect/style_json_test.mbt | 12 ++ office/cmd/office/cram/cli.t | 2 +- office/cmd/office/pkg.generated.mbti | 2 + office/cmd/office/xlsx_read_model.mbt | 180 +++++++++++++++++++++---- office/cmd/office/xlsx_read_output.mbt | 26 +++- office/cmd/office/xlsx_read_wbtest.mbt | 168 +++++++++++++++++++++++ ooxml/moon.pkg | 1 - xlsx/formula_opts_test.mbt | 18 +++ xlsx/pkg.generated.mbti | 10 ++ xlsx/read.mbt | 87 ++++++------ xlsx/workbook.mbt | 41 ++++++ xlsx/worksheet.mbt | 53 ++++++++ xlsx/worksheet_types.mbt | 13 ++ 15 files changed, 542 insertions(+), 79 deletions(-) create mode 100644 inspect/style_json_test.mbt diff --git a/crypto/moon.pkg b/crypto/moon.pkg index 2c8c5336..221b993e 100644 --- a/crypto/moon.pkg +++ b/crypto/moon.pkg @@ -1,4 +1,3 @@ import { "moonbitlang/core/debug", } - diff --git a/inspect/style_json.mbt b/inspect/style_json.mbt index 1331f3a1..591475ee 100644 --- a/inspect/style_json.mbt +++ b/inspect/style_json.mbt @@ -30,8 +30,13 @@ fn set_int(out : Map[String, Json], key : String, value : Int?) -> Unit { ///| fn set_double(out : Map[String, Json], key : String, value : Double?) -> Unit { match value { - Some(value) => out[key] = Json::number(value) + // JSON numbers are finite. Persisted XLSX input is rejected earlier, but + // this public converter also accepts programmatically constructed styles; + // omit an invalid optional value instead of emitting bare NaN/Infinity. + Some(value) if !value.is_nan() && !value.is_inf() => + out[key] = Json::number(value) None => () + _ => () } } diff --git a/inspect/style_json_test.mbt b/inspect/style_json_test.mbt new file mode 100644 index 00000000..b4c9f1a9 --- /dev/null +++ b/inspect/style_json_test.mbt @@ -0,0 +1,12 @@ +///| +test "style JSON omits programmatic non-finite optional numbers" { + let nan = @string.parse_double("NaN") + let infinity = @string.parse_double("Infinity") + let style = @xlsx.Style::font( + @xlsx.Font::with_values(size=nan, color_tint=infinity), + ) + let encoded = @inspect.style_to_json(style).stringify() + assert_false(encoded.contains("NaN")) + assert_false(encoded.contains("Infinity")) + assert_eq(encoded, "{}") +} diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index d4bcca25..d2afc580 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -142,7 +142,7 @@ round-trips through the public selector grammar. {"schema":"office.xlsx.text/1","under":"/xlsx/sheet[name=\"Sheet1\"]","paths":["/xlsx/sheet[name=\"Sheet1\"]/cell[B19]","/xlsx/sheet[name=\"Sheet1\"]/cell[C21]"],"texts":["237","Column1"],"matched_total":5,"returned":2,"truncated":true,"scanned":88} $ office.exe query "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json | jq -c '{schema:.data.schema,selector:.data.selector,under:.data.under,paths:[.data.matches[].path],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' - {"schema":"office.xlsx.query/1","selector":"cell[type=formula][formula~=IF]","under":"/xlsx/sheet[name=\"Sheet2\"]","paths":["/xlsx/sheet[name=\"Sheet2\"]/cell[F11]","/xlsx/sheet[name=\"Sheet2\"]/cell[G11]"],"matched_total":2,"returned":2,"truncated":false,"scanned":99} + {"schema":"office.xlsx.query/1","selector":"cell[type=formula][formula~=IF]","under":"/xlsx/sheet[name=\"Sheet2\"]","paths":["/xlsx/sheet[name=\"Sheet2\"]/cell[F11]","/xlsx/sheet[name=\"Sheet2\"]/cell[G11]","/xlsx/sheet[name=\"Sheet2\"]/cell[H11]","/xlsx/sheet[name=\"Sheet2\"]/cell[I11]"],"matched_total":4,"returned":4,"truncated":false,"scanned":99} Cross-format selectors and DOCX-only XLSX query flags fail with XLSX-specific, machine-correctable codes. diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 5901a4c0..77ffbd5d 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -67,6 +67,8 @@ type XlsxCellPredicate type XlsxCellSnapshot +type XlsxFormatBudget + type XlsxMetadataBudget type XlsxNumberOperator diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 28d2df1a..a3b0bb64 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -7,6 +7,9 @@ let xlsx_cli_max_cell_string_chars : Int = 1024 * 1024 ///| let xlsx_cli_max_scanned_string_chars : Int = 16 * 1024 * 1024 +///| +let xlsx_cli_max_format_work_units : Int = 16 * 1024 * 1024 + ///| let xlsx_cli_max_metadata_items : Int = 10_000 @@ -25,6 +28,8 @@ struct XlsxSheetTarget { index : Int path : String content : XlsxSheetContent + shared_formula_masters : Map[Int, String] + mut shared_formulas_indexed : Bool } ///| @@ -59,9 +64,11 @@ struct XlsxCellSnapshot { row : Int column : Int raw : @xlsx.CellValue? - formatted : String? + mut formatted : String? + mut formatted_ready : Bool formula : String? style_id : Int + worksheet : @xlsx.Worksheet } ///| @@ -81,9 +88,16 @@ struct XlsxStringBudget { struct XlsxScanBudget { maximum : Int strings : XlsxStringBudget + formatting : XlsxFormatBudget mut scanned : Int } +///| +struct XlsxFormatBudget { + maximum : Int + mut used : Int +} + ///| struct XlsxMetadataBudget { maximum_items : Int @@ -220,6 +234,8 @@ fn make_xlsx_projection( index, path: canonical_xlsx_sheet_path(name), content, + shared_formula_masters: Map([]), + shared_formulas_indexed: content is ChartSheet(_), } sheet_index[name] = index sheets.push(target) @@ -372,6 +388,28 @@ fn XlsxSheetTarget::worksheet( } } +///| +/// Resolves shared-formula masters on demand. Most reads never encounter a +/// shared follower, so they avoid a worksheet-wide formula pass; the first +/// follower builds every master in one pass and later lookups are O(1). +fn XlsxSheetTarget::shared_formula_master( + self : XlsxSheetTarget, + projection : XlsxProjection, + worksheet : @xlsx.Worksheet, + shared_index : Int, +) -> String? raise CliFailure { + if !self.shared_formulas_indexed { + let masters = xlsx_call(projection.file, () => { + worksheet.shared_formula_masters() + }) + for index, formula in masters { + self.shared_formula_masters[index] = formula + } + self.shared_formulas_indexed = true + } + self.shared_formula_masters.get(shared_index) +} + ///| fn resolve_xlsx_selector( projection : XlsxProjection, @@ -510,8 +548,49 @@ fn XlsxStringBudget::charge( } ///| -fn XlsxScanBudget::new(maximum : Int) -> XlsxScanBudget { - { maximum, strings: XlsxStringBudget::new(), scanned: 0 } +fn XlsxFormatBudget::new( + maximum? : Int = xlsx_cli_max_format_work_units, +) -> XlsxFormatBudget { + { maximum, used: 0 } +} + +///| +/// Charges the linear parse cost of a custom number format before invoking +/// the formatter. Built-in and absent formats have bounded constant work and +/// are already bounded by the scanned-cell ceiling. +fn XlsxFormatBudget::charge_style( + self : XlsxFormatBudget, + projection : XlsxProjection, + style_id : Int, +) -> Unit raise CliFailure { + let style = xlsx_call(projection.file, () => { + projection.workbook.get_style(style_id) + }) + let cost = match style.number_format { + Some(Custom(code)) => code.length().max(1) + _ => 0 + } + if cost > self.maximum - self.used { + raise xlsx_resource_failure( + "cell formatting work units", + self.maximum, + actual=self.used + cost, + ) + } + self.used += cost +} + +///| +fn XlsxScanBudget::new( + maximum : Int, + format_work_maximum? : Int = xlsx_cli_max_format_work_units, +) -> XlsxScanBudget { + { + maximum, + strings: XlsxStringBudget::new(), + formatting: XlsxFormatBudget::new(maximum=format_work_maximum), + scanned: 0, + } } ///| @@ -536,16 +615,42 @@ fn XlsxScanBudget::charge_snapshot_strings( Some(Error(value)) => self.strings.charge("cell error", value) _ => () } - match snapshot.formatted { - Some(value) => self.strings.charge("formatted cell value", value) - None => () - } match snapshot.formula { Some(value) => self.strings.charge("cell formula", value) None => () } } +///| +fn xlsx_cell_formula( + projection : XlsxProjection, + sheet : XlsxSheetTarget, + worksheet : @xlsx.Worksheet, + row : Int, + column : Int, +) -> String? raise CliFailure { + match + xlsx_call(projection.file, () => { + worksheet.get_cell_formula_info_rc(row, column) + }) { + None => None + Some(info) if info.formula != "" => Some(info.formula) + Some(info) => + match info.shared_index { + Some(shared_index) => + // `Some("")` is intentional when a malformed package has a shared + // follower but no resolvable master: preserve formula presence even + // when no formula text can be supplied. + Some( + sheet + .shared_formula_master(projection, worksheet, shared_index) + .unwrap_or(""), + ) + None => Some("") + } + } +} + ///| fn xlsx_cell_snapshot( projection : XlsxProjection, @@ -566,39 +671,66 @@ fn xlsx_cell_snapshot( Some(String("")) => None value => value } - let formula = match - xlsx_call(projection.file, () => worksheet.get_cell_formula(reference)) { - Some("") => None - value => value - } + let formula = xlsx_cell_formula(projection, sheet, worksheet, row, column) let style_id = xlsx_call(projection.file, () => { worksheet.effective_style_id_rc(row, column) }) - let formatted = match normalized_raw { - Some(_) => - Some( - xlsx_call(projection.file, () => { - projection.workbook.get_cell_value_from_worksheet_rc( - worksheet, row, column, - ) - }).unwrap_or(""), - ) - None => None - } let snapshot : XlsxCellSnapshot = { path: canonical_xlsx_cell_path(sheet.name, reference), reference, row, column, raw: normalized_raw, - formatted, + formatted: None, + formatted_ready: false, formula, style_id, + worksheet, } budget.charge_snapshot_strings(snapshot) snapshot } +///| +/// Materializes display text at most once. Custom format parse work is charged +/// before formatting and the produced string joins the aggregate scan-string +/// budget, so neither CPU nor retained text can grow only as a side effect of +/// a large scan. +fn XlsxCellSnapshot::ensure_formatted( + self : XlsxCellSnapshot, + projection : XlsxProjection, + budget : XlsxScanBudget, +) -> Unit raise CliFailure { + if self.formatted_ready { + return + } + match self.raw { + Some(String(_)) | Some(Numeric(_)) => + budget.formatting.charge_style(projection, self.style_id) + _ => () + } + let formatted = match self.raw { + Some(_) => + Some( + xlsx_call(projection.file, () => { + projection.workbook.get_cell_value_styled_from_worksheet_rc( + self.worksheet, + self.row, + self.column, + self.style_id, + ) + }).unwrap_or(""), + ) + None => None + } + match formatted { + Some(value) => budget.strings.charge("formatted cell value", value) + None => () + } + self.formatted = formatted + self.formatted_ready = true +} + ///| fn XlsxCellSnapshot::is_present(self : XlsxCellSnapshot) -> Bool { self.raw is Some(_) || self.formula is Some(_) || self.style_id != 0 diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 8f1c28bc..0d7de313 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -257,14 +257,28 @@ fn xlsx_styles_json( fn xlsx_cell_text(cell : XlsxCellSnapshot) -> String? { match cell.formatted { Some(value) if value != "" => Some(value) + // A cached result was present and intentionally formatted to blank (for + // example with `;;;`). Do not replace Excel's blank display with formula + // source text. `None` can also mean lazy formatting has not run yet, so + // raw presence remains the authoritative distinction. + _ if cell.raw is Some(_) => None _ => match cell.formula { - Some(formula) => Some("=" + formula) + Some(formula) if formula != "" => Some("=" + formula) None => None + _ => None } } } +///| +fn XlsxCellFilter::needs_formatted(self : XlsxCellFilter) -> Bool { + match self { + TextCells => true + PresentCells | QueryCells(_, _) => false + } +} + ///| fn XlsxCellFilter::matches( self : XlsxCellFilter, @@ -295,8 +309,12 @@ fn collect_xlsx_cell_page( filter : XlsxCellFilter, offset : Int, limit : Int, + format_work_maximum? : Int = xlsx_cli_max_format_work_units, ) -> XlsxCellPage raise CliFailure { - let scan = XlsxScanBudget::new(projection.max_scan_cells) + let scan = XlsxScanBudget::new( + projection.max_scan_cells, + format_work_maximum~, + ) let cells : Array[XlsxCellSnapshot] = [] let mut matched = 0 for region in regions { @@ -309,8 +327,12 @@ fn collect_xlsx_cell_page( column, scan, ) + if filter.needs_formatted() { + cell.ensure_formatted(projection, scan) + } if filter.matches(cell) { if matched >= offset && cells.length() < limit { + cell.ensure_formatted(projection, scan) cells.push(cell) } matched += 1 diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index ed33affb..7d0595e5 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -99,6 +99,8 @@ test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" assert_eq(snapshot.row, 2) assert_eq(snapshot.column, 2) assert_eq(snapshot.formula, Some("SUM(1,2)")) + assert_eq(snapshot.formatted, None) + snapshot.ensure_formatted(projection, budget) assert_eq(snapshot.formatted, Some("3")) assert_true(snapshot.raw is Some(String("3"))) assert_eq(budget.scanned, 1) @@ -123,6 +125,7 @@ test "XLSX cell scans keep the resolved worksheet handle" { fail("expected resolved raw string") } assert_eq(raw, "alpha") + snapshot.ensure_formatted(projection, XlsxScanBudget::new(1)) assert_eq(snapshot.formatted, Some("alpha")) } @@ -210,6 +213,95 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { ) } +///| +test "XLSX snapshots preserve and resolve shared-formula followers" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_formula_opts( + "Data", + "A1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("A1:C1"), + ) + let projection = make_xlsx_projection("shared.xlsx", workbook, 10) + assert_false(projection.sheets[0].shared_formulas_indexed) + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), + ) + let spec : XlsxQuerySpec = { + selector: "cell[type=formula][formula~=D1]", + predicates: parse_xlsx_query_selector("cell[type=formula][formula~=D1]"), + offset: 0, + limit: 10, + } + guard xlsx_query_data(projection, under, spec) is Object(fields) && + fields.get("matches") is Some(Array(matches)) else { + fail("expected shared-formula matches") + } + assert_eq(fields.get("matched_total"), Some(Json::number(3))) + assert_eq(matches.length(), 3) + assert_true(projection.sheets[0].shared_formulas_indexed) + for match_record in matches { + assert_true(match_record.stringify().contains("\"formula\":\"D1+1\"")) + } +} + +///| +test "XLSX formatting is deferred and custom-format work is charged" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + let style_id = workbook.new_style(@xlsx.Style::number_format("[Red]0.00")) + for column in 1..<=2 { + workbook.set_cell_int( + "Data", + @xlsx.coordinates_to_cell_name(column, 1), + column, + ) + workbook.set_cell_style( + "Data", + @xlsx.coordinates_to_cell_name(column, 1), + style_id, + ) + } + let projection = make_xlsx_projection("format-work.xlsx", workbook, 10) + let regions = xlsx_scan_regions( + projection, + Some(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]")), + ) + // A raw predicate with an empty page never formats matching cells. + let page = collect_xlsx_cell_page( + projection, + regions, + xlsx_query_filter(parse_xlsx_query_selector("cell[type=number]")), + 0, + 0, + format_work_maximum=1, + ) + assert_eq(page.matched_total, 2) + assert_eq(page.cells.length(), 0) + assert_false(projection.sheets[0].shared_formulas_indexed) + let limited = xlsx_cli_error(() => { + ignore( + collect_xlsx_cell_page( + projection, + regions, + TextCells, + 0, + 10, + format_work_maximum=1, + ), + ) + }) + assert_eq(limited.code, "office.xlsx.resource_limit") + guard limited.details is Some(Object(details)) else { + fail("expected format-work limit details") + } + assert_eq( + details.get("resource"), + Some(Json::string("cell formatting work units")), + ) +} + ///| test "XLSX outline and get use office schemas with canonical cell records" { let projection = xlsx_read_test_projection() @@ -308,6 +400,82 @@ test "XLSX structured reads reject non-finite package numbers" { assert_eq(error.code, "office.invalid_package") } +///| +test "XLSX style parsing rejects non-finite JSON-facing numbers" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_int("Data", "A1", 1) + let style_id = workbook.new_style( + @xlsx.Style::font(@xlsx.Font::with_values(size=12.0)), + ) + workbook.set_cell_style("Data", "A1", style_id) + let archive = try! @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/styles.xml") is Some(style_bytes) else { + fail("missing styles part") + } + let styles_xml = try! @utf8.decode(style_bytes, ignore_bom=true) + assert_true(styles_xml.contains("")) + assert_true( + archive.replace( + "xl/styles.xml", + @utf8.encode( + styles_xml.replace(old="", new=""), + ), + ), + ) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(@zip.write(archive)), + ) catch { + error => Err(error) + } + assert_true(result is Err(@xlsx.XlsxError::InvalidXml(_))) +} + +///| +test "XLSX displayed text preserves formula results formatted to blank" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + let style_id = workbook.new_style(@xlsx.Style::number_format(";;;")) + workbook.set_cell_formula("Data", "A1", "1+0", value="1") + workbook.set_cell_style("Data", "A1", style_id) + let archive = try! @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/worksheets/sheet1.xml") is Some(sheet_bytes) else { + fail("missing worksheet part") + } + let sheet_xml = try! @utf8.decode(sheet_bytes, ignore_bom=true) + assert_true(sheet_xml.contains(" t=\"str\"")) + // Formula setters retain string compatibility. Remove t="str" to model a + // normal OOXML cached numeric formula result. + assert_true( + archive.replace( + "xl/worksheets/sheet1.xml", + @utf8.encode(sheet_xml.replace(old=" t=\"str\"", new="")), + ), + ) + let projection = make_xlsx_projection( + "blank-display.xlsx", + try! @xlsx.read(@zip.write(archive)), + 10, + ) + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), + ) + guard xlsx_text_data(projection, under, 0, 10) is Object(text) else { + fail("expected text result") + } + assert_eq(text.get("matched_total"), Some(Json::number(0))) + guard xlsx_get_data( + projection, + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]"), + ) + is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected formula cell") + } + assert_eq(cell.get("value"), Some(Json::string(""))) + assert_eq(cell.get("formula"), Some(Json::string("1+0"))) +} + ///| test "XLSX text and query scan fully while retaining only the requested page" { let projection = xlsx_read_test_projection() diff --git a/ooxml/moon.pkg b/ooxml/moon.pkg index 2c8c5336..221b993e 100644 --- a/ooxml/moon.pkg +++ b/ooxml/moon.pkg @@ -1,4 +1,3 @@ import { "moonbitlang/core/debug", } - diff --git a/xlsx/formula_opts_test.mbt b/xlsx/formula_opts_test.mbt index f985e544..2aa7b9dc 100644 --- a/xlsx/formula_opts_test.mbt +++ b/xlsx/formula_opts_test.mbt @@ -98,6 +98,24 @@ test "formula opts shared writes typed formula" { content="Some(\"A1+B1\")", ) debug_inspect(parsed.get_cell_formula("Sheet1", "C2"), content="None") + guard parsed.sheet("Sheet1") is Some(parsed_sheet) else { + fail("missing parsed worksheet") + } + guard parsed_sheet.get_cell_formula_info_rc(1, 3) is Some(master) else { + fail("missing shared-formula master metadata") + } + assert_eq(master.formula, "A1+B1") + assert_true(master.formula_type is Some(Shared)) + assert_eq(master.range_ref, Some("C1:C3")) + assert_eq(master.shared_index, Some(0)) + guard parsed_sheet.get_cell_formula_info_rc(2, 3) is Some(follower) else { + fail("missing shared-formula follower metadata") + } + assert_eq(follower.formula, "") + assert_true(follower.formula_type is Some(Shared)) + assert_eq(follower.range_ref, None) + assert_eq(follower.shared_index, Some(0)) + assert_eq(parsed_sheet.shared_formula_masters().get(0), Some("A1+B1")) } ///| diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 4c5bddaa..3566403a 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -232,6 +232,13 @@ pub struct Cell { style_id : Int } +pub struct CellFormulaInfo { + formula : String + formula_type : FormulaType? + range_ref : String? + shared_index : Int? +} + type CellImage pub(all) enum CellValue { @@ -1832,6 +1839,7 @@ pub fn Workbook::get_cell_value_raw(Self, StringView, StringView) -> CellValue? pub fn Workbook::get_cell_value_raw_rc(Self, StringView, Int, Int) -> CellValue? raise XlsxError pub fn Workbook::get_cell_value_rc(Self, StringView, Int, Int, raw? : Bool, options? : Options) -> String? raise XlsxError pub fn Workbook::get_cell_value_styled(Self, StringView, StringView, Int, options? : Options) -> String? raise XlsxError +pub fn Workbook::get_cell_value_styled_from_worksheet_rc(Self, Worksheet, Int, Int, Int, options? : Options) -> String? raise XlsxError pub fn Workbook::get_col(Self, StringView, Int) -> Array[String] raise XlsxError pub fn Workbook::get_col_outline_level(Self, StringView, Int) -> Int raise XlsxError pub fn Workbook::get_col_style(Self, StringView, Int) -> Int? raise XlsxError @@ -2097,6 +2105,7 @@ pub fn Worksheet::effective_style_id_rc(Self, Int, Int) -> Int raise XlsxError pub fn Worksheet::formula_refs(Self) -> Array[String] pub fn Worksheet::get_cell(Self, StringView) -> String? raise XlsxError pub fn Worksheet::get_cell_formula(Self, StringView) -> String? raise XlsxError +pub fn Worksheet::get_cell_formula_info_rc(Self, Int, Int) -> CellFormulaInfo? raise XlsxError pub fn Worksheet::get_cell_hyperlink(Self, StringView) -> Hyperlink? raise XlsxError pub fn Worksheet::get_cell_rc(Self, Int, Int) -> String? raise XlsxError pub fn Worksheet::get_cell_rich_text(Self, StringView) -> Array[RichTextRun]? raise XlsxError @@ -2159,6 +2168,7 @@ pub fn Worksheet::set_row_style(Self, Int, Int) -> Unit raise XlsxError pub fn Worksheet::set_row_visible(Self, Int, Bool) -> Unit raise XlsxError pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxError pub fn Worksheet::shapes(Self) -> ArrayView[Shape] +pub fn Worksheet::shared_formula_masters(Self) -> Map[Int, String] raise XlsxError pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? pub fn Worksheet::slicers(Self) -> ArrayView[Slicer] diff --git a/xlsx/read.mbt b/xlsx/read.mbt index c611a059..c603be8f 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -96,6 +96,24 @@ fn decode_utf8( } } +///| +/// OOXML numeric attributes must be finite before they enter the workbook +/// model. MoonBit's parser accepts IEEE NaN and infinities, but those values +/// cannot be represented by JSON and make comparisons in layout/style logic +/// non-deterministic. +fn parse_finite_double_for_read( + value : StringView, + message : String, +) -> Double raise XlsxError { + let parsed = @string.parse_double(value) catch { + _ => raise InvalidXml(msg=message) + } + if parsed.is_nan() || parsed.is_inf() { + raise InvalidXml(msg=message) + } + parsed +} + ///| fn parse_default_font(xml : StringView) -> String { let xml_str = xml.to_owned() @@ -205,9 +223,7 @@ fn parse_font_entry(xml : StringView) -> Font raise XlsxError { match attr_value(tag, "val") { Some(value) => font.size = Some( - @string.parse_double(value) catch { - _ => raise InvalidXml(msg="font sz invalid") - }, + parse_finite_double_for_read(value, "font sz invalid"), ) None => () } @@ -292,11 +308,9 @@ fn parse_font_entry(xml : StringView) -> Font raise XlsxError { } match attr_value(tag, "tint") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="font color tint invalid") - } noraise { - val => font.color_tint = Some(val) - } + font.color_tint = Some( + parse_finite_double_for_read(value, "font color tint invalid"), + ) None => () } } @@ -378,47 +392,27 @@ fn parse_fill_entry(xml : StringView) -> Fill raise XlsxError { fill.typ = Some("gradient") let bottom = match attr_value(tag, "bottom") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="gradientFill bottom invalid") - } noraise { - v => v - } + parse_finite_double_for_read(value, "gradientFill bottom invalid") None => 0.0 } let degree = match attr_value(tag, "degree") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="gradientFill degree invalid") - } noraise { - v => v - } + parse_finite_double_for_read(value, "gradientFill degree invalid") None => 0.0 } let left = match attr_value(tag, "left") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="gradientFill left invalid") - } noraise { - v => v - } + parse_finite_double_for_read(value, "gradientFill left invalid") None => 0.0 } let right = match attr_value(tag, "right") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="gradientFill right invalid") - } noraise { - v => v - } + parse_finite_double_for_read(value, "gradientFill right invalid") None => 0.0 } let top = match attr_value(tag, "top") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="gradientFill top invalid") - } noraise { - v => v - } + parse_finite_double_for_read(value, "gradientFill top invalid") None => 0.0 } let typ = match attr_value(tag, "type") { @@ -444,12 +438,11 @@ fn parse_fill_entry(xml : StringView) -> Fill raise XlsxError { let open_tag = text[:open_end] match attr_value(open_tag, "position") { Some(value) => - try @string.parse_double(value) catch { - _ => - raise InvalidXml(msg="gradientFill stop position invalid") - } noraise { - v => stop_positions.push(v) - } + stop_positions.push( + parse_finite_double_for_read( + value, "gradientFill stop position invalid", + ), + ) None => stop_positions.push(0.0) } let rest = text[open_end + 1:] @@ -549,11 +542,9 @@ fn parse_fill_entry(xml : StringView) -> Fill raise XlsxError { } match attr_value(color_tag, "tint") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="fill fgColor tint invalid") - } noraise { - val => fill.fg_tint = Some(val) - } + fill.fg_tint = Some( + parse_finite_double_for_read(value, "fill fgColor tint invalid"), + ) None => () } match attr_value(color_tag, "rgb") { @@ -593,11 +584,9 @@ fn parse_fill_entry(xml : StringView) -> Fill raise XlsxError { } match attr_value(color_tag, "tint") { Some(value) => - try @string.parse_double(value) catch { - _ => raise InvalidXml(msg="fill bgColor tint invalid") - } noraise { - val => fill.bg_tint = Some(val) - } + fill.bg_tint = Some( + parse_finite_double_for_read(value, "fill bgColor tint invalid"), + ) None => () } match attr_value(color_tag, "rgb") { diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 8462088c..85c65148 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1853,6 +1853,47 @@ pub fn Workbook::get_cell_value_from_worksheet_rc( } } +///| +/// Formats one cell from an already-resolved worksheet using an explicit +/// style id. This is the bulk-read counterpart to `get_cell_value_styled` and +/// lets callers apply row/column-inherited styles without repeating a sheet +/// name lookup for every coordinate. Returns `None` for an absent cell. +/// +/// `worksheet` must be an entry from this workbook's `sheets()` view. The row +/// and column are 1-based and `style_id` must belong to this workbook. +pub fn Workbook::get_cell_value_styled_from_worksheet_rc( + self : Workbook, + worksheet : Worksheet, + row : Int, + col : Int, + style_id : Int, + options? : Options, +) -> String? raise XlsxError { + ignore(cell_ref_from(row, col)) + self.check_style_id(style_id) + let resolved_options = match options { + Some(value) => value + None => self.options + } + match worksheet.cell_index_of(row, col) { + Some(index) => { + let cell = worksheet.cells[index] + if resolved_options.raw_cell_value { + return Some(cell.value) + } + Some( + format_cell_value( + cell.value_type, + cell.value, + Some(self.styles[style_id]), + resolved_options, + ), + ) + } + None => None + } +} + ///| pub fn Workbook::calc_cell_value( self : Workbook, diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index e50a9474..6047bd6a 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -378,6 +378,29 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { refs } +///| +/// Returns non-empty shared-formula masters keyed by OOXML shared index. +/// Followers are deliberately omitted, so master formula text is retained +/// only once even for large shared ranges. Conflicting masters are rejected +/// rather than resolved by iteration order. +pub fn Worksheet::shared_formula_masters( + self : Worksheet, +) -> Map[Int, String] raise XlsxError { + let masters : Map[Int, String] = Map([]) + for cell in self.cells { + match (cell.formula_type, cell.formula, cell.formula_shared_index) { + (Some(Shared), Some(formula), Some(shared_index)) if formula != "" => + match masters.get(shared_index) { + Some(existing) if existing != formula => + raise InvalidXml(msg="shared formula index has conflicting masters") + _ => masters[shared_index] = formula + } + _ => () + } + } + masters +} + ///| pub fn Worksheet::set_cell_formula( self : Worksheet, @@ -441,6 +464,36 @@ pub fn Worksheet::get_cell_formula( } } +///| +/// Returns the stored formula metadata for a 1-based coordinate without +/// collapsing an empty shared-formula follower to `None`. +/// +/// This complements `get_cell_formula`, whose compatibility contract returns +/// only independently stored, non-empty formula text. The returned formula is +/// not translated from a shared master; `shared_index` identifies that master +/// when a consumer needs its text. +pub fn Worksheet::get_cell_formula_info_rc( + self : Worksheet, + row : Int, + col : Int, +) -> CellFormulaInfo? raise XlsxError { + ignore(cell_ref_from(row, col)) + match self.cell_index_of(row, col) { + Some(i) => + match self.cells[i].formula { + Some(formula) => + Some({ + formula, + formula_type: self.cells[i].formula_type, + range_ref: self.cells[i].formula_ref, + shared_index: self.cells[i].formula_shared_index, + }) + None => None + } + None => None + } +} + ///| pub fn Worksheet::set_cell_formula_opts( self : Worksheet, diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index d149575b..78b8be76 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -13,6 +13,19 @@ pub struct Cell { style_id : Int } +///| +/// Read-only formula metadata for one worksheet coordinate. `formula` is the +/// exact text stored in that cell; OOXML shared-formula followers therefore +/// carry an empty string while `formula_type` and `shared_index` preserve +/// their formula-bearing state. Callers that need display/query text can use +/// the shared index to resolve the corresponding non-empty master formula. +pub struct CellFormulaInfo { + formula : String + formula_type : FormulaType? + range_ref : String? + shared_index : Int? +} + ///| pub struct Image { reference : String From 66cab47c8ce5a9b726544b8310f353bfe829dd91 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 08:11:18 +0800 Subject: [PATCH 012/150] fix(office): close final xlsx read review gaps --- docs/office-xlsx-read.md | 11 ++ office/cmd/office/read_package.mbt | 23 ++-- office/cmd/office/xlsx_query.mbt | 37 ++++- office/cmd/office/xlsx_read_model.mbt | 39 +++--- office/cmd/office/xlsx_read_wbtest.mbt | 182 +++++++++++++++++++++++++ xlsx/ooxml_rels.mbt | 52 ++++++- xlsx/pkg.generated.mbti | 2 + xlsx/read_worksheet_xml.mbt | 1 + xlsx/stream.mbt | 1 + xlsx/workbook_compat.mbt | 1 + xlsx/worksheet.mbt | 28 ++++ xlsx/worksheet_cell_index.mbt | 1 + xlsx/worksheet_types.mbt | 5 + xlsx/write.mbt | 11 +- 14 files changed, 360 insertions(+), 34 deletions(-) diff --git a/docs/office-xlsx-read.md b/docs/office-xlsx-read.md index efec8d42..f5ce976f 100644 --- a/docs/office-xlsx-read.md +++ b/docs/office-xlsx-read.md @@ -16,6 +16,10 @@ bounded archive to the XLSX parser. It does not use C stubs. A `.docx` package selects the DOCX result contract instead; an extension/content mismatch fails before either projection is opened. +Both Transitional and ISO Strict Office relationship families are accepted. +The detector and the structured XLSX parser therefore agree on every package +dialect they admit. + Run `office help xlsx` for the installed command catalog and `office help --json` for the declared format variants, inputs, outputs, and limits. @@ -89,6 +93,8 @@ Cell and range results include a `styles` object keyed by the referenced style ids. Range reads omit completely blank, unstyled cells while preserving row-major order. Selecting one blank, unstyled coordinate fails with `office.xlsx.selector_not_found`; it does not fabricate a cell record. +For formulas, an explicit empty cached `` remains an empty `raw`/displayed +value and stays distinct from a formula whose cache is absent. ## `text` @@ -100,6 +106,7 @@ office text FILE [--under SELECTOR] [--offset N] [--limit N] Each entry contains `{path, stability, text}`. Text is the workbook's displayed cell value. If a formula has no cached displayed value, text falls back to the formula prefixed with `=` so an uncached formula is not silently omitted. +An explicitly cached blank result remains blank and does not use that fallback. `matched_total`, `returned`, `offset`, `limit`, `truncated`, and `scanned_cells` describe the completed bounded scan exactly. @@ -128,10 +135,14 @@ Examples: ```text cell[type=formula][formula~=SUM] +cell[formula~=SUM(Table1[[#Headers],[Amount]])] cell[type=number][value>=0] cell[text~=revenue] ``` +Balanced brackets inside literal predicate values are part of the surrounding +predicate, which permits Excel structured references without quoting. + Regular expressions, arbitrary expressions, locale-sensitive matching, and the DOCX-only `--kind`, `--text`, `--id`, `--property`, and `--ignore-case` options are rejected. Literal substring predicates are compiled once and use diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 27b78677..551e1cd6 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -110,6 +110,15 @@ async fn read_office_package(file : String) -> OfficeReadPackage { ///| fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { + fn invalid_package(message : String) -> CliFailure { + CliFailure( + @lib.protocol_error( + "office.invalid_package", + "invalid XLSX package: " + bounded_text(message, 320), + details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), + ), + ) + } match error { ResourceLimitExceeded(kind~, limit~, actual~) => CliFailure( @@ -124,14 +133,12 @@ fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { }), ), ) - InvalidPackage(msg~) | InvalidXml(msg~) => - CliFailure( - @lib.protocol_error( - "office.invalid_package", - "invalid XLSX package: " + bounded_text(msg, 320), - details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), - ), - ) + InvalidPackage(msg~) | InvalidXml(msg~) => invalid_package(msg) + MissingPart(path~) => invalid_package("missing required part: \{path}") + InvalidSharedString(index~) => + invalid_package("shared string index is out of range: \{index}") + InvalidStyleId(index~) => + invalid_package("cell style index is out of range: \{index}") _ => CliFailure( @lib.protocol_error( diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt index 3c654323..a4515d25 100644 --- a/office/cmd/office/xlsx_query.mbt +++ b/office/cmd/office/xlsx_query.mbt @@ -162,6 +162,35 @@ fn parse_xlsx_predicate( } } +///| +/// Finds the terminator for one outer predicate group while permitting +/// balanced brackets inside its literal value. This is required for normal +/// Excel structured references such as `Table1[Amount]` and +/// `Table1[[#Headers],[Amount]]`. +fn xlsx_query_predicate_close( + rest : String, + selector : String, +) -> Int raise CliFailure { + let mut depth = 0 + let mut index = 0 + while index < rest.length() { + let character = rest[index] + if character == ('[' : UInt16) { + depth += 1 + } else if character == (']' : UInt16) { + depth -= 1 + if depth == 0 { + return index + } + if depth < 0 { + break + } + } + index += 1 + } + raise xlsx_query_failure("query selector is missing a closing ']'", selector) +} + ///| fn parse_xlsx_query_selector( source : String, @@ -193,13 +222,7 @@ fn parse_xlsx_query_selector( if !rest.has_prefix("[") { raise xlsx_query_failure("expected '[' in query selector", selector) } - let close = match rest.find("]") { - Some(index) => index - None => - raise xlsx_query_failure( - "query selector is missing a closing ']'", selector, - ) - } + let close = xlsx_query_predicate_close(rest, selector) predicates.push(parse_xlsx_predicate(rest[1:close].to_owned(), selector)) rest = rest[close + 1:].to_owned() } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index a3b0bb64..4232741a 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -628,26 +628,30 @@ fn xlsx_cell_formula( worksheet : @xlsx.Worksheet, row : Int, column : Int, -) -> String? raise CliFailure { +) -> (String?, Bool) raise CliFailure { match xlsx_call(projection.file, () => { worksheet.get_cell_formula_info_rc(row, column) }) { - None => None - Some(info) if info.formula != "" => Some(info.formula) + None => (None, false) + Some(info) if info.formula != "" => + (Some(info.formula), info.cached_value_present) Some(info) => - match info.shared_index { - Some(shared_index) => - // `Some("")` is intentional when a malformed package has a shared - // follower but no resolvable master: preserve formula presence even - // when no formula text can be supplied. - Some( - sheet - .shared_formula_master(projection, worksheet, shared_index) - .unwrap_or(""), - ) - None => Some("") - } + ( + match info.shared_index { + Some(shared_index) => + // `Some("")` is intentional when a malformed package has a shared + // follower but no resolvable master: preserve formula presence even + // when no formula text can be supplied. + Some( + sheet + .shared_formula_master(projection, worksheet, shared_index) + .unwrap_or(""), + ) + None => Some("") + }, + info.cached_value_present, + ) } } @@ -667,11 +671,14 @@ fn xlsx_cell_snapshot( let raw = xlsx_call(projection.file, () => { worksheet.get_cell_value_raw(reference) }) + let (formula, cached_value_present) = xlsx_cell_formula( + projection, sheet, worksheet, row, column, + ) let normalized_raw = match raw { + Some(String("")) if formula is Some(_) && cached_value_present => raw Some(String("")) => None value => value } - let formula = xlsx_cell_formula(projection, sheet, worksheet, row, column) let style_id = xlsx_call(projection.file, () => { worksheet.effective_style_id_rc(row, column) }) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 7d0595e5..88179d79 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -20,6 +20,46 @@ fn xlsx_cli_error( } } +///| +fn xlsx_bounded_test_source( + file : String, + bytes : BytesView, +) -> OfficeReadPackage raise { + let archive = @zip.read_limited( + bytes, + max_package_bytes=office_read_max_package_bytes, + max_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + ) + { file, format: Xlsx, archive } +} + +///| +fn xlsx_strict_test_bytes() -> Bytes raise { + let archive = @zip.Archive::new() + let content_types = + #| + let root_relationships = + #| + let workbook = + #| + let workbook_relationships = + #| + let worksheet = + #|strict-value + archive.add("[Content_Types].xml", @utf8.encode(content_types)) + archive.add("_rels/.rels", @utf8.encode(root_relationships)) + archive.add("xl/workbook.xml", @utf8.encode(workbook)) + archive.add( + "xl/_rels/workbook.xml.rels", + @utf8.encode(workbook_relationships), + ) + archive.add("xl/worksheets/sheet.xml", @utf8.encode(worksheet)) + @zip.write(archive) +} + ///| test "XLSX selectors resolve to stable named canonical paths" { let projection = xlsx_read_test_projection() @@ -51,6 +91,22 @@ test "XLSX selectors resolve to stable named canonical paths" { assert_eq(@lib.parse_selector(path).render(), path) } +///| +test "XLSX structured reads accept detector-valid Strict OOXML relationships" { + let source = xlsx_bounded_test_source("strict.xlsx", xlsx_strict_test_bytes()) + assert_true(@lib.detect_archive_format(source.file, source.archive) is Xlsx) + let projection = open_xlsx_projection(source, 10) + guard xlsx_get_data( + projection, + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Strict\"]/cell[A1]"), + ) + is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected Strict XLSX cell") + } + assert_eq(cell.get("value"), Some(Json::string("strict-value"))) +} + ///| test "XLSX selector failures distinguish format, absence, and scan limits" { let projection = xlsx_read_test_projection() @@ -167,6 +223,14 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { "cell[type=formula][formula~=SUM][text=3]", ) assert_eq(predicates.length(), 3) + let structured = parse_xlsx_query_selector( + "cell[formula~=SUM(Table1[[#Headers],[Amount]])][text~=total]", + ) + assert_eq(structured.length(), 2) + guard structured[0] is FormulaContains(literal) else { + fail("expected structured-reference formula predicate") + } + assert_eq(literal, "SUM(Table1[[#Headers],[Amount]])") let work = OfficeQueryWorkBudget::new(1024, format="xlsx") assert_true( xlsx_cell_matches( @@ -400,6 +464,59 @@ test "XLSX structured reads reject non-finite package numbers" { assert_eq(error.code, "office.invalid_package") } +///| +test "XLSX parse-time index corruption is classified as an invalid package" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_str("Data", "A1", "alpha") + let shared_archive = @zip.read(@xlsx.write(workbook)) + guard shared_archive.get("xl/worksheets/sheet1.xml") is Some(shared_bytes) else { + fail("missing shared-string worksheet") + } + let shared_xml = @utf8.decode(shared_bytes, ignore_bom=true) + assert_true(shared_xml.contains("0")) + assert_true( + shared_archive.replace( + "xl/worksheets/sheet1.xml", + @utf8.encode(shared_xml.replace(old="0", new="7")), + ), + ) + let shared_source = xlsx_bounded_test_source( + "broken-shared.xlsx", + @zip.write(shared_archive), + ) + let shared = xlsx_cli_error(() => { + ignore(open_xlsx_projection(shared_source, 10)) + }) + assert_eq(shared.code, "office.invalid_package") + + let styled_workbook = @xlsx.Workbook::new() + ignore(styled_workbook.new_sheet("Data")) + styled_workbook.set_cell_int("Data", "A1", 1) + let style_archive = @zip.read(@xlsx.write(styled_workbook)) + guard style_archive.get("xl/worksheets/sheet1.xml") is Some(style_bytes) else { + fail("missing styled worksheet") + } + let style_xml = @utf8.decode(style_bytes, ignore_bom=true) + assert_true(style_xml.contains(" { + ignore(open_xlsx_projection(style_source, 10)) + }) + assert_eq(style.code, "office.invalid_package") +} + ///| test "XLSX style parsing rejects non-finite JSON-facing numbers" { let workbook = @xlsx.Workbook::new() @@ -476,6 +593,71 @@ test "XLSX displayed text preserves formula results formatted to blank" { assert_eq(cell.get("formula"), Some(Json::string("1+0"))) } +///| +test "XLSX cached empty formula results remain distinct from a missing cache" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_formula("Data", "A1", "1+0") + let archive = @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/worksheets/sheet1.xml") is Some(sheet_bytes) else { + fail("missing formula worksheet") + } + let sheet_xml = @utf8.decode(sheet_bytes, ignore_bom=true) + assert_true(sheet_xml.contains("1+0")) + let cached_xml = sheet_xml + .replace(old="`. + let roundtrip = @zip.read(@xlsx.write(parsed)) + guard roundtrip.get("xl/worksheets/sheet1.xml") is Some(roundtrip_bytes) else { + fail("missing round-trip formula worksheet") + } + let roundtrip_xml = @utf8.decode(roundtrip_bytes, ignore_bom=true) + assert_true(roundtrip_xml.contains("1+0")) + + let projection = make_xlsx_projection("cached-empty.xlsx", parsed, 10) + guard xlsx_get_data( + projection, + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]"), + ) + is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected cached-empty formula cell") + } + assert_eq(cell.get("value"), Some(Json::string(""))) + assert_eq( + cell.get("raw"), + Some( + Json::object({ "type": Json::string("string"), "value": Json::string("") }), + ), + ) + guard xlsx_text_data( + projection, + Some(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]")), + 0, + 10, + ) + is Object(text) else { + fail("expected cached-empty text result") + } + assert_eq(text.get("matched_total"), Some(Json::number(0))) +} + ///| test "XLSX text and query scan fully while retaining only the requested page" { let projection = xlsx_read_test_projection() diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 8d541ad2..96ea6b1c 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -1,5 +1,11 @@ ///| -fn parse_relationship_targets( +let transitional_office_relationship_prefix = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" + +///| +let strict_office_relationship_prefix = "http://purl.oclc.org/ooxml/officeDocument/relationships/" + +///| +fn parse_relationship_targets_exact( xml : StringView, rel_type : StringView, ) -> Map[String, String] raise XlsxError { @@ -8,6 +14,31 @@ fn parse_relationship_targets( } } +///| +/// Reads both ISO Strict and ECMA Transitional aliases for relationship types +/// from the Office relationship family. Microsoft extension relationships use +/// their own namespaces and therefore remain exact matches. +fn parse_relationship_targets( + xml : StringView, + rel_type : StringView, +) -> Map[String, String] raise XlsxError { + let requested = rel_type.to_owned() + let targets = parse_relationship_targets_exact(xml, requested) + guard requested.strip_prefix(transitional_office_relationship_prefix) + is Some(suffix) else { + return targets + } + let strict_type = strict_office_relationship_prefix + suffix.to_owned() + let strict_targets = parse_relationship_targets_exact(xml, strict_type) + for id, target in strict_targets { + if targets.contains(id) { + raise InvalidXml(msg="relationship id has conflicting dialect aliases") + } + targets[id] = target + } + targets +} + ///| fn first_relationship_target(targets : Map[String, String]) -> String? { for _, target in targets { @@ -140,6 +171,25 @@ test "ooxml_rels: parse_relationship_targets filters by Type and maps Id->Target ) } +///| +test "ooxml_rels: Office relationship types accept Strict aliases" { + let transitional = transitional_office_relationship_prefix + "worksheet" + let strict = strict_office_relationship_prefix + "worksheet" + let xml = + #| + #| + #| + #| + let targets = parse_relationship_targets(xml, transitional) + assert_eq(targets.get("strict"), Some("worksheets/strict.xml")) + assert_eq(targets.get("transitional"), Some("worksheets/transitional.xml")) + // Asking for an already-Strict URI remains exact and does not broaden to + // the Transitional family in the opposite direction. + let strict_only = parse_relationship_targets(xml, strict) + assert_eq(strict_only.length(), 1) + assert_eq(strict_only.get("strict"), Some("worksheets/strict.xml")) +} + ///| test "ooxml_rels: first_relationship_target returns first map entry" { let targets : Map[String, String] = { "rId9": "x", "rId10": "y" } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 3566403a..e0b0d535 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -229,6 +229,7 @@ pub struct Cell { formula_type : FormulaType? formula_ref : String? formula_shared_index : Int? + formula_value_present : Bool style_id : Int } @@ -237,6 +238,7 @@ pub struct CellFormulaInfo { formula_type : FormulaType? range_ref : String? shared_index : Int? + cached_value_present : Bool } type CellImage diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index 43226e24..e3e538b4 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -222,6 +222,7 @@ fn parse_worksheet( formula_type, formula_ref, formula_shared_index, + formula_value_present: formula is Some(_) && raw_value_opt is Some(_), style_id, }) } diff --git a/xlsx/stream.mbt b/xlsx/stream.mbt index aa35d118..264d674e 100644 --- a/xlsx/stream.mbt +++ b/xlsx/stream.mbt @@ -446,6 +446,7 @@ pub fn StreamWriter::set_row_cells( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: cell.formula is Some(_) && cell.value != "", style_id, }) col = col + 1 diff --git a/xlsx/workbook_compat.mbt b/xlsx/workbook_compat.mbt index 3efec14d..48240411 100644 --- a/xlsx/workbook_compat.mbt +++ b/xlsx/workbook_compat.mbt @@ -271,6 +271,7 @@ pub fn Workbook::update_linked_value(self : Workbook) -> Unit raise XlsxError { formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, } } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 6047bd6a..e3011465 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -128,6 +128,7 @@ pub fn Worksheet::set_cell( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, } } @@ -144,6 +145,7 @@ pub fn Worksheet::set_cell( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -202,6 +204,7 @@ pub fn Worksheet::set_cell_value( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, } } @@ -218,6 +221,7 @@ pub fn Worksheet::set_cell_value( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -303,6 +307,7 @@ pub fn Worksheet::set_cell_rich_text( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, } } @@ -319,6 +324,7 @@ pub fn Worksheet::set_cell_rich_text( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -425,6 +431,7 @@ pub fn Worksheet::set_cell_formula( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: value != "", style_id, } } @@ -441,6 +448,7 @@ pub fn Worksheet::set_cell_formula( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: value != "", style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -487,6 +495,7 @@ pub fn Worksheet::get_cell_formula_info_rc( formula_type: self.cells[i].formula_type, range_ref: self.cells[i].formula_ref, shared_index: self.cells[i].formula_shared_index, + cached_value_present: self.cells[i].formula_value_present, }) None => None } @@ -544,6 +553,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_type: Some(Array), formula_ref: Some(formula_ref), formula_shared_index: None, + formula_value_present: value != "", style_id, } } @@ -560,6 +570,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_type: Some(Array), formula_ref: Some(formula_ref), formula_shared_index: None, + formula_value_present: value != "", style_id: 0, }) sheet.record_cell_index_if_valid(row, col, index) @@ -621,6 +632,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_type: Some(Shared), formula_ref, formula_shared_index: Some(shared_index), + formula_value_present: value != "", style_id, } } @@ -637,6 +649,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_type: Some(Shared), formula_ref, formula_shared_index: Some(shared_index), + formula_value_present: value != "", style_id: 0, }) sheet.record_cell_index_if_valid(row, col, index) @@ -889,6 +902,7 @@ pub fn Worksheet::set_cell_style( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id, } } @@ -905,6 +919,7 @@ pub fn Worksheet::set_cell_style( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, }) self.record_cell_index_if_valid(row, col, index) @@ -950,6 +965,7 @@ pub fn Worksheet::set_cell_style_rc( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id, } } @@ -966,6 +982,7 @@ pub fn Worksheet::set_cell_style_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, }) self.record_cell_index_if_valid(row, col, index) @@ -2388,6 +2405,7 @@ pub fn Worksheet::set_cell_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, } } @@ -2404,6 +2422,7 @@ pub fn Worksheet::set_cell_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -2436,6 +2455,7 @@ pub fn Worksheet::set_cell_value_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id, } } @@ -2452,6 +2472,7 @@ pub fn Worksheet::set_cell_value_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -2483,6 +2504,7 @@ pub fn Worksheet::set_cell_formula_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: value != "", style_id, } } @@ -2499,6 +2521,7 @@ pub fn Worksheet::set_cell_formula_rc( formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: value != "", style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -3091,6 +3114,7 @@ fn Worksheet::insert_rows( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, }) } else { @@ -3244,6 +3268,7 @@ fn Worksheet::remove_rows( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, }) } @@ -3548,6 +3573,7 @@ fn Worksheet::duplicate_row_to( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, }) } @@ -3599,6 +3625,7 @@ fn Worksheet::insert_cols( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, }) } else { @@ -3748,6 +3775,7 @@ fn Worksheet::remove_cols( formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, + formula_value_present: cell.formula_value_present, style_id: cell.style_id, }) } diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index cb31911d..d96cafd0 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -59,6 +59,7 @@ fn raw_cell(reference : String, row : Int, col : Int, value : String) -> Cell { formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, } } diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 78b8be76..14b79068 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -10,6 +10,9 @@ pub struct Cell { formula_type : FormulaType? formula_ref : String? formula_shared_index : Int? + /// Whether this formula cell contained an OOXML `` cached-result node. + /// Empty cached strings are semantically distinct from a missing cache. + formula_value_present : Bool style_id : Int } @@ -24,6 +27,8 @@ pub struct CellFormulaInfo { formula_type : FormulaType? range_ref : String? shared_index : Int? + /// True when the formula has a cached `` result, including ``. + cached_value_present : Bool } ///| diff --git a/xlsx/write.mbt b/xlsx/write.mbt index 093b4dcc..4ca7ad3a 100644 --- a/xlsx/write.mbt +++ b/xlsx/write.mbt @@ -1799,7 +1799,10 @@ fn write_worksheet_xml( match cell.formula { Some(formula) => { match cell.value_type { - String => if cell.value != "" { sb.write_view(" t=\"str\"") } + String => + if cell.formula_value_present { + sb.write_view(" t=\"str\"") + } Bool => sb.write_view(" t=\"b\"") Error => sb.write_view(" t=\"e\"") Number => () @@ -1829,7 +1832,7 @@ fn write_worksheet_xml( sb.write_view(">") sb.write_view(escape_xml_text(formula)) sb.write_view("") - if cell.value != "" { + if cell.formula_value_present { sb.write_view("") match cell.value_type { String => sb.write_view(escape_xml_text(cell.value)) @@ -3780,6 +3783,7 @@ test "write hardening wb: write_worksheet_xml covers formula bool and error bran formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: true, style_id: 0, }) sheet.cells.push({ @@ -3793,6 +3797,7 @@ test "write hardening wb: write_worksheet_xml covers formula bool and error bran formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: true, style_id: 0, }) let xml = write_worksheet_xml( @@ -3832,6 +3837,7 @@ test "write hardening wb: write_worksheet_xml covers shared-string and hyperlink formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) let missing_rich : Result[String, Error] = Ok( @@ -3870,6 +3876,7 @@ test "write hardening wb: write_worksheet_xml covers shared-string and hyperlink formula_type: None, formula_ref: None, formula_shared_index: None, + formula_value_present: false, style_id: 0, }) let missing_plain : Result[String, Error] = Ok( From a5c81532c74bfbb6344c2ea4ef831d6bea5ae0ba Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 10:05:47 +0800 Subject: [PATCH 013/150] fix(office): harden xlsx structured reads --- .github/workflows/ci.yml | 4 +- office/cmd/office/xlsx_read_model.mbt | 7 +- office/cmd/office/xlsx_read_wbtest.mbt | 78 +++++++++++++- xlsx/compat_test.mbt | 54 ++++++++++ xlsx/ooxml_rels.mbt | 67 +++++++++--- xlsx/ooxml_rels_error_test.mbt | 26 +++-- xlsx/read.mbt | 139 +++++++++++++++++++++---- xlsx/read_sheet_rel_parts.mbt | 7 +- xlsx/read_worksheet_xml.mbt | 20 +++- xlsx/workbook_compat.mbt | 4 +- xlsx/write_ooxml_constants.mbt | 9 ++ 11 files changed, 359 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaf82d62..8496e6f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -312,8 +312,8 @@ jobs: xq=$(moon run --target wasm office/cmd/office -- query "$xlsx_fixture" 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json 2>office-wasm.err) || { echo "office XLSX query failed; stderr:"; cat office-wasm.err; exit 1; } echo "$xq" | jq -e '.success == true and .data.schema == "office.xlsx.query/1" - and .data.matched_total == 2 - and [.data.matches[].reference] == ["F11", "G11"]' + and .data.matched_total == 4 + and [.data.matches[].reference] == ["F11", "G11", "H11", "I11"]' # DOCX structured reads share one canonical bounded projection. # Drive every command through the wasm async filesystem path. o=$(moon run --target wasm office/cmd/office -- outline docx2html/tests/cram/fixtures/single-paragraph.docx --json 2>office-wasm.err) || { echo "office DOCX outline failed; stderr:"; cat office-wasm.err; exit 1; } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 4232741a..ed57793e 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -207,7 +207,8 @@ fn make_xlsx_projection( let sheets : Array[XlsxSheetTarget] = [] let sheet_index : Map[String, Int] = Map([]) for index, name in workbook.get_sheet_list() { - if sheet_index.contains(name) { + let normalized_name = name.to_lower() + if sheet_index.contains(normalized_name) { raise xlsx_cli_failure( "office.xlsx.read_failed", "workbook tab order contains a duplicate sheet name", @@ -237,7 +238,7 @@ fn make_xlsx_projection( shared_formula_masters: Map([]), shared_formulas_indexed: content is ChartSheet(_), } - sheet_index[name] = index + sheet_index[normalized_name] = index sheets.push(target) } { @@ -335,7 +336,7 @@ fn XlsxProjection::find_sheet_by_name( self : XlsxProjection, name : String, ) -> XlsxSheetTarget? { - match self.sheet_index.get(name) { + match self.sheet_index.get(name.to_lower()) { Some(index) => self.sheets.get(index) None => None } diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 88179d79..0a8b4e2c 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -60,6 +60,35 @@ fn xlsx_strict_test_bytes() -> Bytes raise { @zip.write(archive) } +///| +fn xlsx_relocated_test_bytes() -> Bytes raise { + let archive = @zip.Archive::new() + let content_types = + #| + let root_relationships = + #| + let workbook = + #| + let workbook_relationships = + #| + let worksheet = + #|0 + let shared_strings = + #|relocated-value + archive.add("[Content_Types].xml", @utf8.encode(content_types)) + archive.add("_rels/.rels", @utf8.encode(root_relationships)) + archive.add("custom/book.xml", @utf8.encode(workbook)) + archive.add( + "custom/_rels/book.xml.rels", + @utf8.encode(workbook_relationships), + ) + archive.add("data/first.xml", @utf8.encode(worksheet)) + // OPC part lookup is case-insensitive. Keep the relationship spelling lower + // case and the actual ZIP entry mixed case to cover canonical resolution. + archive.add("data/Strings.xml", @utf8.encode(shared_strings)) + @zip.write(archive) +} + ///| test "XLSX selectors resolve to stable named canonical paths" { let projection = xlsx_read_test_projection() @@ -89,6 +118,12 @@ test "XLSX selectors resolve to stable named canonical paths" { assert_eq(singleton.range_reference(projection.file), "B2:B2") assert_eq(path, "/xlsx/sheet[name=\"Data\"]/range[B2:B2]") assert_eq(@lib.parse_selector(path).render(), path) + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"data\"]/cell[A1]") + is Cell(case_insensitive_sheet, _, path~) else { + fail("expected case-insensitive sheet selector") + } + assert_eq(case_insensitive_sheet.name, "Data") + assert_eq(path, "/xlsx/sheet[name=\"Data\"]/cell[A1]") } ///| @@ -107,6 +142,29 @@ test "XLSX structured reads accept detector-valid Strict OOXML relationships" { assert_eq(cell.get("value"), Some(Json::string("strict-value"))) } +///| +test "XLSX structured reads resolve relationships from relocated workbooks" { + let source = xlsx_bounded_test_source( + "relocated.xlsx", + xlsx_relocated_test_bytes(), + ) + assert_true(@lib.detect_archive_format(source.file, source.archive) is Xlsx) + let projection = open_xlsx_projection(source, 10) + guard xlsx_get_data( + projection, + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"moved\"]/cell[A1]"), + ) + is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected relocated XLSX cell") + } + assert_eq( + get.get("path"), + Some(Json::string("/xlsx/sheet[name=\"Moved\"]/cell[A1]")), + ) + assert_eq(cell.get("value"), Some(Json::string("relocated-value"))) +} + ///| test "XLSX selector failures distinguish format, absence, and scan limits" { let projection = xlsx_read_test_projection() @@ -606,7 +664,7 @@ test "XLSX cached empty formula results remain distinct from a missing cache" { assert_true(sheet_xml.contains("1+0")) let cached_xml = sheet_xml .replace(old="1+0")) + let compact_archive = @zip.read(@xlsx.write(workbook)) + assert_true( + compact_archive.replace( + "xl/worksheets/sheet1.xml", + @utf8.encode( + sheet_xml + .replace(old="SUM(1,2) String raise XlsxError { segments.join("/") } +///| +fn resolve_part_rel_target( + source_part : StringView, + target : StringView, +) -> String raise XlsxError { + let source = source_part.to_owned() + let target_str = target.to_owned() + if source == "" || source.has_prefix("/") || target_str == "" { + raise InvalidXml(msg="relationship target invalid") + } + let normalized_source = normalize_rel_part_path(source) + if normalized_source != source { + raise InvalidXml(msg="relationship target invalid") + } + let raw = if target_str.has_prefix("/") { + drop_first_path_segment(target_str) + } else { + match source.rev_find("/") { + Some(pos) => source[:pos + 1].to_owned() + target_str + None => target_str + } + } + normalize_rel_part_path(raw) +} + ///| fn resolve_rel_target( target : StringView, @@ -135,21 +160,7 @@ fn resolve_rel_target( ///| fn resolve_workbook_rel_target(target : StringView) -> String raise XlsxError { - let target_str = target.to_owned() - let raw = if target_str.has_prefix("/") { - drop_first_path_segment(target_str) - } else if target_str.has_prefix("../") { - "xl/" + drop_first_path_segment(target_str) - } else if target_str.has_prefix("xl/") { - target_str - } else { - "xl/" + target_str - } - let normalized = normalize_rel_part_path(raw) - if !normalized.has_prefix("xl/") { - raise InvalidXml(msg="relationship target invalid") - } - normalized + resolve_part_rel_target(workbook_part_path, target) } ///| @@ -261,6 +272,22 @@ test "ooxml_rels: resolve_workbook_rel_target treats relative as under xl/" { ) } +///| +test "ooxml_rels: part relationships resolve from the actual source part" { + inspect( + resolve_part_rel_target("custom/book.xml", "../data/first.xml"), + content="data/first.xml", + ) + inspect( + resolve_part_rel_target("custom/book.xml", "/data/first.xml"), + content="data/first.xml", + ) + inspect( + resolve_part_rel_target("book.xml", "data/first.xml"), + content="data/first.xml", + ) +} + ///| test "ooxml_rels: resolve target normalizes dot segments" { inspect( @@ -300,6 +327,16 @@ test "ooxml_rels: resolve target rejects escaping root via dot segments" { "expected InvalidXml for relative escape in resolve_workbook_rel_target", ) } + + let relocated_result = Ok( + resolve_part_rel_target("custom/book.xml", "../../sheet1.xml"), + ) catch { + e => Err(e) + } + match relocated_result { + Err(InvalidXml(msg~)) => inspect(msg, content="relationship target invalid") + _ => fail("expected relocated relationship target to reject a root escape") + } } ///| diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index c10c1b3a..448eeb82 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -156,7 +156,7 @@ test "ooxml rels: expanded Relationship tags still read" { } ///| -test "ooxml rels: workbook targets with ../ prefix" { +test "ooxml rels: workbook targets resolve parent segments from their source" { let workbook = @xlsx.Workbook::new() ignore(workbook.add_sheet("Sheet1")) let bytes = @xlsx.write(workbook) @@ -176,12 +176,12 @@ test "ooxml rels: workbook targets with ../ prefix" { "xl/_rels/workbook.xml.rels", @encoding/utf8.encode(patched), ) - let parsed = @xlsx.read(updated) - debug_inspect(parsed.get_sheet_list(), content="[\"Sheet1\"]") + let result = Ok(@xlsx.read(updated)) catch { error => Err(error) } + assert_true(result is Err(MissingPart(_))) } ///| -test "ooxml rels: workbook targets with xl/ prefix" { +test "ooxml rels: workbook targets treat xl prefix as source relative" { let workbook = @xlsx.Workbook::new() ignore(workbook.add_sheet("Sheet1")) let bytes = @xlsx.write(workbook) @@ -201,8 +201,22 @@ test "ooxml rels: workbook targets with xl/ prefix" { "xl/_rels/workbook.xml.rels", @encoding/utf8.encode(patched), ) - let parsed = @xlsx.read(updated) - debug_inspect(parsed.get_sheet_list(), content="[\"Sheet1\"]") + let result = Ok(@xlsx.read(updated)) catch { error => Err(error) } + assert_true(result is Err(MissingPart(_))) +} + +///| +test "ooxml rels: reader rejects case-equivalent part collisions" { + let archive = @zip.read(@xlsx.write(@xlsx.Workbook::new())) + guard archive.get("xl/workbook.xml") is Some(workbook_xml) else { + fail("missing workbook part") + } + archive.add("xl/Workbook.xml", workbook_xml) + let bytes = @zip.write(archive) + let result : Result[@xlsx.Workbook, @xlsx.XlsxError] = Ok(@xlsx.read(bytes)) catch { + error => Err(error) + } + assert_true(result is Err(InvalidPackage(_))) } ///| diff --git a/xlsx/read.mbt b/xlsx/read.mbt index c603be8f..f96760a2 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3051,17 +3051,71 @@ fn first_existing_rel_target_path( ///| fn first_existing_workbook_rel_target_path( targets : Map[String, String], - archive : @zip.Archive, + workbook_part : StringView, + part_names : Map[String, String], ) -> String? raise XlsxError { for _, target in targets { - let path = resolve_workbook_rel_target(target) - if archive.get(path) is Some(_) { - return Some(path) + let path = resolve_part_rel_target(workbook_part, target) + match part_names.get(path.to_lower()) { + Some(actual) => return Some(actual) + None => () } } None } +///| +fn archive_part_name_index( + archive : @zip.Archive, +) -> Map[String, String] raise XlsxError { + let names : Map[String, String] = Map([]) + for entry in archive.entries() { + let name = entry.name() + let key = name.to_lower() + match names.get(key) { + Some(existing) if existing != name => + raise InvalidPackage(msg="case-insensitive ZIP part name collision") + Some(_) => () + None => names[key] = name + } + } + names +} + +///| +fn actual_archive_part_path( + part_names : Map[String, String], + requested : String, +) -> String { + part_names.get(requested.to_lower()).unwrap_or(requested) +} + +///| +fn workbook_related_part_path( + workbook_rels : StringView, + relationship_type : StringView, + workbook_part : StringView, + fallback : StringView, + part_names : Map[String, String], +) -> String? raise XlsxError { + let targets = parse_relationship_targets(workbook_rels, relationship_type) + match + first_existing_workbook_rel_target_path(targets, workbook_part, part_names) { + Some(path) => Some(path) + None => + match first_relationship_target(targets) { + Some(target) => + Some( + actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_part, target), + ), + ) + None => part_names.get(fallback.to_owned().to_lower()) + } + } +} + ///| fn read_zip_archive_core_unchecked( archive : @zip.Archive, @@ -3069,6 +3123,7 @@ fn read_zip_archive_core_unchecked( limits : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { + let part_names = archive_part_name_index(archive) let decode = (value : BytesView) => { if value.length() > limits.max_xml_part_bytes { raise ResourceLimitExceeded( @@ -3079,12 +3134,23 @@ fn read_zip_archive_core_unchecked( } decode_utf8(value, transcoder) } - let workbook_xml_part_path = resolve_workbook_xml_part_path(archive, decode) + let workbook_xml_part_path = actual_archive_part_path( + part_names, + resolve_workbook_xml_part_path(archive, decode), + ) let workbook_bytes = match archive.get(workbook_xml_part_path) { Some(value) => value None => raise MissingPart(path=workbook_xml_part_path) } let workbook_xml = decode(workbook_bytes) + let workbook_rels_path = actual_archive_part_path( + part_names, + rels_path_for_part(workbook_xml_part_path), + ) + let workbook_rels = match archive.get(workbook_rels_path) { + Some(value) => decode(value) + None => raise MissingPart(path=workbook_rels_path) + } let sheets = parse_workbook_sheets(workbook_xml) let sheet_names : Array[String] = [] for entry in sheets { @@ -3100,10 +3166,21 @@ fn read_zip_archive_core_unchecked( } else { active_index } - let shared_strings = match archive.get("xl/sharedStrings.xml") { - Some(value) => parse_shared_strings(decode(value)) + let shared_strings_path = workbook_related_part_path( + workbook_rels, rel_shared_strings, workbook_xml_part_path, shared_strings_part_path, + part_names, + ) + let shared_strings = match shared_strings_path { + Some(path) => + match archive.get(path) { + Some(value) => parse_shared_strings(decode(value)) + None => raise MissingPart(path~) + } None => [] } + let styles_path = workbook_related_part_path( + workbook_rels, rel_styles, workbook_xml_part_path, styles_part_path, part_names, + ) let ( styles, conditional_styles, @@ -3113,8 +3190,12 @@ fn read_zip_archive_core_unchecked( indexed_colors, mru_colors_xml, styles_ext_lst_xml, - ) = match archive.get("xl/styles.xml") { - Some(value) => { + ) = match styles_path { + Some(path) => { + let value = match archive.get(path) { + Some(value) => value + None => raise MissingPart(path~) + } let styles_xml = decode(value) let (styles, conditional_styles) = parse_styles(styles_xml) let (default_table_style, default_pivot_style) = parse_table_styles_defaults( @@ -3146,8 +3227,15 @@ fn read_zip_archive_core_unchecked( None, ) } - let theme_xml = match archive.get("xl/theme/theme1.xml") { - Some(value) => Some(decode(value)) + let theme_path = workbook_related_part_path( + workbook_rels, rel_theme, workbook_xml_part_path, theme_part_path, part_names, + ) + let theme_xml = match theme_path { + Some(path) => + match archive.get(path) { + Some(value) => Some(decode(value)) + None => raise MissingPart(path~) + } None => None } let theme_colors = match theme_xml { @@ -3233,21 +3321,24 @@ fn read_zip_archive_core_unchecked( rich_value_images, rich_value_media, } - let workbook_rels_path = rels_path_for_part(workbook_xml_part_path) - let workbook_rels = match archive.get(workbook_rels_path) { - Some(value) => decode(value) - None => raise MissingPart(path=workbook_rels_path) - } let slicer_cache_defs = parse_slicer_cache_definitions( - workbook_rels, archive, decode, + workbook_rels, workbook_xml_part_path, part_names, archive, decode, ) let vba_targets = parse_relationship_targets(workbook_rels, rel_vba_project) let vba_path = match - first_existing_workbook_rel_target_path(vba_targets, archive) { + first_existing_workbook_rel_target_path( + vba_targets, workbook_xml_part_path, part_names, + ) { Some(path) => Some(path) None => match first_relationship_target(vba_targets) { - Some(target) => Some(resolve_workbook_rel_target(target)) + Some(target) => + Some( + actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_xml_part_path, target), + ), + ) None => None } } @@ -3281,7 +3372,10 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="worksheet target missing") } - let sheet_path = resolve_workbook_rel_target(target) + let sheet_path = actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_xml_part_path, target), + ) let sheet_bytes = match archive.get(sheet_path) { Some(value) => value None => raise MissingPart(path=sheet_path) @@ -3818,7 +3912,10 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="chartsheet target missing") } - let chartsheet_path = resolve_workbook_rel_target(target) + let chartsheet_path = actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_xml_part_path, target), + ) let chartsheet_bytes = match archive.get(chartsheet_path) { Some(value) => value None => raise MissingPart(path=chartsheet_path) diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index 4480f02d..b21ad6b7 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -340,13 +340,18 @@ fn parse_slicer_cache_definition( ///| fn parse_slicer_cache_definitions( workbook_rels_xml : StringView, + workbook_part : StringView, + part_names : Map[String, String], archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, ) -> Map[String, SlicerCacheDef] raise XlsxError { let defs : Map[String, SlicerCacheDef] = Map([]) let targets = parse_relationship_targets(workbook_rels_xml, rel_slicer_cache) for _rel_id, target in targets { - let cache_path = resolve_workbook_rel_target(target) + let cache_path = actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_part, target), + ) let cache_bytes = match archive.get(cache_path) { Some(value) => value None => raise MissingPart(path=cache_path) diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index e3e538b4..703f5c47 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -117,14 +117,24 @@ fn parse_worksheet( } None => (None, None, None, None) } - let raw_value_opt = match rest.find("") { + let raw_value_opt = match find_xml_open_tag_start(rest, "v") { Some(pos) => { - let v_rest = rest[pos + 3:] - let v_end = match v_rest.find("") { + let v_rest = rest[pos:] + let open_end = match xml_open_tag_end_from(v_rest, 0) { Some(end) => end - None => raise InvalidXml(msg="cell value not closed") + None => raise InvalidXml(msg="cell value tag not closed") + } + let open_tag = v_rest[:open_end].to_owned() + if open_tag.trim().has_suffix("/") { + Some("") + } else { + let value_rest = v_rest[open_end + 1:] + let v_end = match value_rest.find("") { + Some(end) => end + None => raise InvalidXml(msg="cell value not closed") + } + Some(value_rest[:v_end].to_owned()) } - Some(v_rest[:v_end].to_owned()) } None => None } diff --git a/xlsx/workbook_compat.mbt b/xlsx/workbook_compat.mbt index 48240411..14e8124d 100644 --- a/xlsx/workbook_compat.mbt +++ b/xlsx/workbook_compat.mbt @@ -259,7 +259,7 @@ pub fn Workbook::update_linked_value(self : Workbook) -> Unit raise XlsxError { for i, cell in sheet.cells { match cell.formula { Some(_) => - if cell.value != "" { + if cell.value != "" || cell.formula_value_present { sheet.cells[i] = { reference: cell.reference, row: cell.row, @@ -271,7 +271,7 @@ pub fn Workbook::update_linked_value(self : Workbook) -> Unit raise XlsxError { formula_type: cell.formula_type, formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, - formula_value_present: cell.formula_value_present, + formula_value_present: false, style_id: cell.style_id, } } diff --git a/xlsx/write_ooxml_constants.mbt b/xlsx/write_ooxml_constants.mbt index 581911e6..103e0f58 100644 --- a/xlsx/write_ooxml_constants.mbt +++ b/xlsx/write_ooxml_constants.mbt @@ -13,6 +13,15 @@ let rel_chart = "http://schemas.openxmlformats.org/officeDocument/2006/relations ///| let rel_chartsheet = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/chartsheet" +///| +let rel_shared_strings = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" + +///| +let rel_styles = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" + +///| +let rel_theme = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/theme" + ///| let rel_table = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/table" From 105cbd1caa29e2b4a4c3d8622e3a8282a88d68b0 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 12:43:10 +0800 Subject: [PATCH 014/150] fix(xlsx): translate shared formulas and relocated parts --- office/cmd/office/xlsx_read_model.mbt | 19 +- office/cmd/office/xlsx_read_wbtest.mbt | 80 +++- xlsx/formula_opts_test.mbt | 6 +- xlsx/header_footer_image_read.mbt | 16 +- xlsx/pkg.generated.mbti | 7 +- xlsx/read.mbt | 140 ++++-- xlsx/read_drawing_xml.mbt | 75 +++- xlsx/shared_formula_translate.mbt | 573 +++++++++++++++++++++++++ xlsx/worksheet.mbt | 52 ++- 9 files changed, 892 insertions(+), 76 deletions(-) create mode 100644 xlsx/shared_formula_translate.mbt diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index ed57793e..8113b853 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -28,7 +28,7 @@ struct XlsxSheetTarget { index : Int path : String content : XlsxSheetContent - shared_formula_masters : Map[Int, String] + shared_formula_masters : Map[Int, @xlsx.SharedFormulaMaster] mut shared_formulas_indexed : Bool } @@ -398,7 +398,7 @@ fn XlsxSheetTarget::shared_formula_master( projection : XlsxProjection, worksheet : @xlsx.Worksheet, shared_index : Int, -) -> String? raise CliFailure { +) -> @xlsx.SharedFormulaMaster? raise CliFailure { if !self.shared_formulas_indexed { let masters = xlsx_call(projection.file, () => { worksheet.shared_formula_masters() @@ -644,11 +644,16 @@ fn xlsx_cell_formula( // `Some("")` is intentional when a malformed package has a shared // follower but no resolvable master: preserve formula presence even // when no formula text can be supplied. - Some( - sheet - .shared_formula_master(projection, worksheet, shared_index) - .unwrap_or(""), - ) + match + sheet.shared_formula_master(projection, worksheet, shared_index) { + Some(master) => + Some( + xlsx_call(projection.file, () => { + master.translate_to(row, column) + }), + ) + None => Some("") + } None => Some("") }, info.cached_value_present, diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 0a8b4e2c..da5c9c64 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -64,17 +64,37 @@ fn xlsx_strict_test_bytes() -> Bytes raise { fn xlsx_relocated_test_bytes() -> Bytes raise { let archive = @zip.Archive::new() let content_types = - #| + #| let root_relationships = #| let workbook = - #| + #| let workbook_relationships = - #| + #| let worksheet = - #|0 + #|0 + let worksheet_relationships = + #| let shared_strings = #|relocated-value + let table = + #|
+ let sheet_drawing = + #|00001000 + let sheet_drawing_relationships = + #| + let sheet_chart = + #| + let chart_sheet = + #| + let chart_sheet_relationships = + #| + let drawing = + #| + let drawing_relationships = + #| + let chart = + #| archive.add("[Content_Types].xml", @utf8.encode(content_types)) archive.add("_rels/.rels", @utf8.encode(root_relationships)) archive.add("custom/book.xml", @utf8.encode(workbook)) @@ -83,9 +103,31 @@ fn xlsx_relocated_test_bytes() -> Bytes raise { @utf8.encode(workbook_relationships), ) archive.add("data/first.xml", @utf8.encode(worksheet)) + // Relationship parts and their targets are also allowed outside xl/*. + // Mixed-case entry names verify lookup through the canonical part index. + archive.add( + "data/_rels/First.xml.rels", + @utf8.encode(worksheet_relationships), + ) // OPC part lookup is case-insensitive. Keep the relationship spelling lower // case and the actual ZIP entry mixed case to cover canonical resolution. archive.add("data/Strings.xml", @utf8.encode(shared_strings)) + archive.add("objects/Table.xml", @utf8.encode(table)) + archive.add("art/SheetDrawing.xml", @utf8.encode(sheet_drawing)) + archive.add( + "art/_rels/SheetDrawing.xml.rels", + @utf8.encode(sheet_drawing_relationships), + ) + archive.add("charts/SheetChart.xml", @utf8.encode(sheet_chart)) + archive.add("media/Photo.png", b"png") + archive.add("pages/Chart.xml", @utf8.encode(chart_sheet)) + archive.add( + "pages/_rels/Chart.xml.rels", + @utf8.encode(chart_sheet_relationships), + ) + archive.add("art/Drawing.xml", @utf8.encode(drawing)) + archive.add("art/_rels/Drawing.xml.rels", @utf8.encode(drawing_relationships)) + archive.add("charts/Chart.xml", @utf8.encode(chart)) @zip.write(archive) } @@ -163,6 +205,18 @@ test "XLSX structured reads resolve relationships from relocated workbooks" { Some(Json::string("/xlsx/sheet[name=\"Moved\"]/cell[A1]")), ) assert_eq(cell.get("value"), Some(Json::string("relocated-value"))) + guard xlsx_outline_data(projection) is Object(outline) && + outline.get("sheets") is Some(Array([Object(moved), Object(chart)])) && + moved.get("counts") is Some(Object(moved_counts)) && + chart.get("counts") is Some(Object(chart_counts)) else { + fail("expected relocated worksheet and chart-sheet outline") + } + assert_eq(moved_counts.get("tables"), Some(Json::number(1))) + assert_eq(moved_counts.get("hyperlinks"), Some(Json::number(1))) + assert_eq(moved_counts.get("charts"), Some(Json::number(1))) + assert_eq(moved_counts.get("images"), Some(Json::number(1))) + assert_eq(chart.get("kind"), Some(Json::string("chart-sheet"))) + assert_eq(chart_counts.get("charts"), Some(Json::number(1))) } ///| @@ -351,8 +405,8 @@ test "XLSX snapshots preserve and resolve shared-formula followers" { resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), ) let spec : XlsxQuerySpec = { - selector: "cell[type=formula][formula~=D1]", - predicates: parse_xlsx_query_selector("cell[type=formula][formula~=D1]"), + selector: "cell[type=formula]", + predicates: parse_xlsx_query_selector("cell[type=formula]"), offset: 0, limit: 10, } @@ -363,9 +417,19 @@ test "XLSX snapshots preserve and resolve shared-formula followers" { assert_eq(fields.get("matched_total"), Some(Json::number(3))) assert_eq(matches.length(), 3) assert_true(projection.sheets[0].shared_formulas_indexed) - for match_record in matches { - assert_true(match_record.stringify().contains("\"formula\":\"D1+1\"")) + assert_true(matches[0].stringify().contains("\"formula\":\"D1+1\"")) + assert_true(matches[1].stringify().contains("\"formula\":\"E1+1\"")) + assert_true(matches[2].stringify().contains("\"formula\":\"F1+1\"")) + let master_only : XlsxQuerySpec = { + selector: "cell[formula~=D1]", + predicates: parse_xlsx_query_selector("cell[formula~=D1]"), + offset: 0, + limit: 10, + } + guard xlsx_query_data(projection, under, master_only) is Object(master_fields) else { + fail("expected translated shared-formula query") } + assert_eq(master_fields.get("matched_total"), Some(Json::number(1))) } ///| diff --git a/xlsx/formula_opts_test.mbt b/xlsx/formula_opts_test.mbt index 2aa7b9dc..ea18a7b9 100644 --- a/xlsx/formula_opts_test.mbt +++ b/xlsx/formula_opts_test.mbt @@ -115,7 +115,11 @@ test "formula opts shared writes typed formula" { assert_true(follower.formula_type is Some(Shared)) assert_eq(follower.range_ref, None) assert_eq(follower.shared_index, Some(0)) - assert_eq(parsed_sheet.shared_formula_masters().get(0), Some("A1+B1")) + guard parsed_sheet.shared_formula_masters().get(0) is Some(shared_master) else { + fail("missing indexed shared-formula master") + } + assert_eq(shared_master.translate_to(1, 3), "A1+B1") + assert_eq(shared_master.translate_to(2, 3), "A2+B2") } ///| diff --git a/xlsx/header_footer_image_read.mbt b/xlsx/header_footer_image_read.mbt index 5214bbdc..05dd958a 100644 --- a/xlsx/header_footer_image_read.mbt +++ b/xlsx/header_footer_image_read.mbt @@ -80,6 +80,8 @@ fn extension_from_path(path : StringView) -> String? { fn parse_header_footer_images_from_vml( vml_xml : StringView, vml_rels_xml : String?, + vml_part : StringView, + part_names : Map[String, String], archive : @zip.Archive, ) -> Array[HeaderFooterImage] raise XlsxError { let shapes = extract_vml_shapes(vml_xml) @@ -131,7 +133,9 @@ fn parse_header_footer_images_from_vml( Some(value) => value None => raise InvalidXml(msg="header/footer image relationship missing") } - let image_path = resolve_rel_target(target, "drawings") + let image_path = actual_relationship_target_path( + vml_part, target, part_names, + ) let image_bytes = match archive.get(image_path) { Some(value) => value None => raise MissingPart(path=image_path) @@ -238,6 +242,8 @@ test "header/footer image read wb: parse images with r:id fallback and duplicate let images = parse_header_footer_images_from_vml( vml_xml, Some(rels_xml), + "xl/drawings/vmlDrawing1.vml", + Map([]), archive, ) inspect(images.length(), content="1") @@ -263,7 +269,13 @@ test "header/footer image read wb: missing target part raises MissingPart" { #| #| let result : Result[Array[HeaderFooterImage], Error] = Ok( - parse_header_footer_images_from_vml(vml_xml, Some(rels_xml), archive), + parse_header_footer_images_from_vml( + vml_xml, + Some(rels_xml), + "xl/drawings/vmlDrawing1.vml", + Map([]), + archive, + ), ) catch { e => Err(e) } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index e0b0d535..f48720b0 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1349,6 +1349,11 @@ pub struct ShapeLine { pub fn ShapeLine::to_repr(Self) -> @debug.Repr pub fn ShapeLine::with_values(color? : String, width? : Double) -> Self +pub struct SharedFormulaMaster { + // private fields +} +pub fn SharedFormulaMaster::translate_to(Self, Int, Int) -> String raise XlsxError + pub struct SheetBackground { data : Bytes extension : String @@ -2170,7 +2175,7 @@ pub fn Worksheet::set_row_style(Self, Int, Int) -> Unit raise XlsxError pub fn Worksheet::set_row_visible(Self, Int, Bool) -> Unit raise XlsxError pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxError pub fn Worksheet::shapes(Self) -> ArrayView[Shape] -pub fn Worksheet::shared_formula_masters(Self) -> Map[Int, String] raise XlsxError +pub fn Worksheet::shared_formula_masters(Self) -> Map[Int, SharedFormulaMaster] raise XlsxError pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? pub fn Worksheet::slicers(Self) -> ArrayView[Slicer] diff --git a/xlsx/read.mbt b/xlsx/read.mbt index f96760a2..8795bf68 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3036,12 +3036,12 @@ fn check_source_package_size( ///| fn first_existing_rel_target_path( targets : Map[String, String], - base_folder : StringView, - archive : @zip.Archive, + source_part : StringView, + part_names : Map[String, String], ) -> String? raise XlsxError { for _, target in targets { - let path = resolve_rel_target(target, base_folder) - if archive.get(path) is Some(_) { + let path = actual_relationship_target_path(source_part, target, part_names) + if part_names.contains(path.to_lower()) { return Some(path) } } @@ -3090,6 +3090,38 @@ fn actual_archive_part_path( part_names.get(requested.to_lower()).unwrap_or(requested) } +///| +fn actual_relationship_part_path( + source_part : StringView, + part_names : Map[String, String], +) -> String raise XlsxError { + actual_archive_part_path(part_names, rels_path_for_part(source_part)) +} + +///| +fn actual_relationship_target_path( + source_part : StringView, + target : StringView, + part_names : Map[String, String], +) -> String raise XlsxError { + let source_relative = resolve_part_rel_target(source_part, target) + match part_names.get(source_relative.to_lower()) { + Some(actual) => actual + None => { + // Some producers emit package-root paths without the leading slash. + // Standards-compliant source-relative lookup wins when both exist; this + // fallback retains tolerant reads only when the root spelling names an + // actual archive part. + let package_root = try normalize_rel_part_path(target) catch { + _ => None + } noraise { + path => part_names.get(path.to_lower()) + } + package_root.unwrap_or(source_relative) + } + } +} + ///| fn workbook_related_part_path( workbook_rels : StringView, @@ -3436,7 +3468,7 @@ fn read_zip_archive_core_unchecked( legacy_drawing_rel_id is Some(_) || legacy_drawing_hf_rel_id is Some(_) || picture_rel_id is Some(_) - let rels_path = rels_path_for(sheet_path, "worksheets") + let rels_path = actual_relationship_part_path(sheet_path, part_names) let rels_xml = match archive.get(rels_path) { Some(rels_bytes) => decode(rels_bytes) None => if needs_rels { raise MissingPart(path=rels_path) } else { "" } @@ -3455,7 +3487,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="slicer relationship missing") } - let slicer_path = resolve_rel_target(target, "worksheets") + let slicer_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) if seen_paths.contains(slicer_path) { continue } @@ -3479,7 +3513,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="picture target missing") } - let image_path = resolve_rel_target(target, "worksheets") + let image_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let image_bytes = match archive.get(image_path) { Some(value) => value None => raise MissingPart(path=image_path) @@ -3556,7 +3592,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="table relationship missing") } - let table_path = resolve_rel_target(target, "worksheets") + let table_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let table_bytes = match archive.get(table_path) { Some(value) => value None => raise MissingPart(path=table_path) @@ -3591,13 +3629,17 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="pivot table relationship missing") } - let pivot_path = resolve_rel_target(target, "worksheets") + let pivot_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let pivot_bytes = match archive.get(pivot_path) { Some(value) => value None => raise MissingPart(path=pivot_path) } let pivot_xml = decode(pivot_bytes) - let pivot_rels_path = rels_path_for(pivot_path, "pivotTables") + let pivot_rels_path = actual_relationship_part_path( + pivot_path, part_names, + ) let pivot_rels_bytes = match archive.get(pivot_rels_path) { Some(value) => value None => raise MissingPart(path=pivot_rels_path) @@ -3608,7 +3650,7 @@ fn read_zip_archive_core_unchecked( ) let cache_path = match first_existing_rel_target_path( - cache_targets, "pivotTables", archive, + cache_targets, pivot_path, part_names, ) { Some(path) => path None => { @@ -3618,7 +3660,9 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="pivot cache relationship missing") } - resolve_rel_target(cache_target, "pivotTables") + actual_relationship_target_path( + pivot_path, cache_target, part_names, + ) } } let cache_bytes = match archive.get(cache_path) { @@ -3633,7 +3677,7 @@ fn read_zip_archive_core_unchecked( pivot_path, "xl/pivotTables/pivotTable", ) let cache_records_xml = match - archive.get(rels_path_for(cache_path, "pivotCache")) { + archive.get(actual_relationship_part_path(cache_path, part_names)) { Some(value) => { let cache_rels_xml = decode(value) let record_targets = parse_relationship_targets( @@ -3641,13 +3685,17 @@ fn read_zip_archive_core_unchecked( ) let record_path = match first_existing_rel_target_path( - record_targets, "pivotCache", archive, + record_targets, cache_path, part_names, ) { Some(path) => Some(path) None => match first_relationship_target(record_targets) { Some(record_target) => - Some(resolve_rel_target(record_target, "pivotCache")) + Some( + actual_relationship_target_path( + cache_path, record_target, part_names, + ), + ) None => None } } @@ -3688,7 +3736,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="vml drawing relationship missing") } - let vml_path = resolve_rel_target(target, "worksheets") + let vml_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let vml_bytes = match archive.get(vml_path) { Some(value) => value None => raise MissingPart(path=vml_path) @@ -3703,7 +3753,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="vml drawing hf relationship missing") } - let vml_path = resolve_rel_target(target, "worksheets") + let vml_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let vml_bytes = match archive.get(vml_path) { Some(value) => value None => raise MissingPart(path=vml_path) @@ -3716,19 +3768,19 @@ fn read_zip_archive_core_unchecked( Some((xml, path)) => (Some(xml), Some(path)) None => (None, None) } - let header_footer_images = match vml_drawing_hf_xml { - Some(xml) => { - let vml_rels_xml = match vml_drawing_hf_path { - Some(path) => - match archive.get(rels_path_for(path, "drawings")) { - Some(value) => Some(decode(value)) - None => None - } + let header_footer_images = match + (vml_drawing_hf_xml, vml_drawing_hf_path) { + (Some(xml), Some(path)) => { + let vml_rels_xml = match + archive.get(actual_relationship_part_path(path, part_names)) { + Some(value) => Some(decode(value)) None => None } - parse_header_footer_images_from_vml(xml, vml_rels_xml, archive) + parse_header_footer_images_from_vml( + xml, vml_rels_xml, path, part_names, archive, + ) } - None => [] + _ => [] } let comments = if rels_xml == "" { [] @@ -3736,7 +3788,9 @@ fn read_zip_archive_core_unchecked( let comment_targets = parse_relationship_targets(rels_xml, rel_comments) let parsed_comments : Array[Comment] = [] for _rel_id, target in comment_targets { - let comment_path = resolve_rel_target(target, "worksheets") + let comment_path = actual_relationship_target_path( + sheet_path, target, part_names, + ) let comment_bytes = match archive.get(comment_path) { Some(value) => value None => raise MissingPart(path=comment_path) @@ -3810,7 +3864,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="drawing target missing") } - let drawing_path = resolve_rel_target(drawing_target, "worksheets") + let drawing_path = actual_relationship_target_path( + sheet_path, drawing_target, part_names, + ) let drawing_bytes = match archive.get(drawing_path) { Some(value) => value None => raise MissingPart(path=drawing_path) @@ -3823,16 +3879,18 @@ fn read_zip_archive_core_unchecked( ) } let drawing_rels_xml = match - archive.get(rels_path_for(drawing_path, "drawings")) { + archive.get(actual_relationship_part_path(drawing_path, part_names)) { Some(value) => decode(value) None => "" } let drawing_images = parse_drawing_images( - drawing_xml, drawing_rels_xml, drawing_metrics, archive, + drawing_xml, drawing_rels_xml, drawing_path, part_names, drawing_metrics, + archive, ) sheet.images.append(drawing_images) let drawing_charts = parse_drawing_charts( - drawing_xml, drawing_rels_xml, drawing_metrics, archive, decode, + drawing_xml, drawing_rels_xml, drawing_path, part_names, drawing_metrics, + archive, decode, ) sheet.charts.append(drawing_charts) let drawing_shapes = parse_drawing_shapes(drawing_xml) @@ -3928,7 +3986,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="chartsheet drawing missing") } - let chartsheet_rels_path = rels_path_for(chartsheet_path, "chartsheets") + let chartsheet_rels_path = actual_relationship_part_path( + chartsheet_path, part_names, + ) let chartsheet_rels_bytes = match archive.get(chartsheet_rels_path) { Some(value) => value None => raise MissingPart(path=chartsheet_rels_path) @@ -3941,8 +4001,12 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="drawing target missing") } - let drawing_path = resolve_rel_target(drawing_target, "chartsheets") - let drawing_rels_path = rels_path_for(drawing_path, "drawings") + let drawing_path = actual_relationship_target_path( + chartsheet_path, drawing_target, part_names, + ) + let drawing_rels_path = actual_relationship_part_path( + drawing_path, part_names, + ) let drawing_rels_bytes = match archive.get(drawing_rels_path) { Some(value) => value None => raise MissingPart(path=drawing_rels_path) @@ -3952,14 +4016,16 @@ fn read_zip_archive_core_unchecked( drawing_rels_xml, rel_chart, ) let chart_path = match - first_existing_rel_target_path(chart_targets, "drawings", archive) { + first_existing_rel_target_path(chart_targets, drawing_path, part_names) { Some(path) => path None => { let chart_target = match first_relationship_target(chart_targets) { Some(value) => value None => raise InvalidXml(msg="chart target missing") } - resolve_rel_target(chart_target, "drawings") + actual_relationship_target_path( + drawing_path, chart_target, part_names, + ) } } let chart_bytes = match archive.get(chart_path) { diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index b2c89e44..0c1e6ba8 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -162,6 +162,8 @@ fn parse_drawing_anchors( fn parse_drawing_images( drawing_xml : StringView, drawing_rels_xml : StringView, + drawing_part : StringView, + part_names : Map[String, String], metrics : DrawingMetrics, archive : @zip.Archive, ) -> Array[Image] raise XlsxError { @@ -299,7 +301,9 @@ fn parse_drawing_images( Some(value) => value None => raise InvalidXml(msg="drawing image target missing") } - let media_path = resolve_rel_target(target, "drawings") + let media_path = actual_relationship_target_path( + drawing_part, target, part_names, + ) let data = match archive.get(media_path) { Some(value) => value None => raise MissingPart(path=media_path) @@ -367,6 +371,8 @@ fn parse_drawing_images( fn parse_drawing_charts( drawing_xml : StringView, drawing_rels_xml : StringView, + drawing_part : StringView, + part_names : Map[String, String], metrics : DrawingMetrics, archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, @@ -392,7 +398,9 @@ fn parse_drawing_charts( Some(value) => value None => raise InvalidXml(msg="drawing chart target missing") } - let chart_path = resolve_rel_target(target, "drawings") + let chart_path = actual_relationship_target_path( + drawing_part, target, part_names, + ) let chart_bytes = match archive.get(chart_path) { Some(value) => value None => raise MissingPart(path=chart_path) @@ -1073,7 +1081,14 @@ test "read_drawing_xml wb: parse_drawing_images empty rel map branch" { let archive = @zip.Archive::new() let metrics = drawing_metrics_for_sheet(Worksheet::new("Sheet1")) let result : Result[Array[Image], Error] = Ok( - parse_drawing_images(drawing_xml, "", metrics, archive), + parse_drawing_images( + drawing_xml, + "", + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { e => Err(e) } @@ -1094,7 +1109,14 @@ test "read_drawing_xml wb: signed marker offsets are retained" { "", ) let metrics = drawing_metrics_for_sheet(Worksheet::new("Sheet1")) - let images = parse_drawing_images(drawing_xml, rels_xml, metrics, archive) + let images = parse_drawing_images( + drawing_xml, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ) inspect(images.length(), content="1") inspect(images[0].reference, content="A1") inspect(images[0].offset_x, content="-2") @@ -1113,7 +1135,14 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { "", "", ) let missing_from_result : Result[Array[Image], Error] = Ok( - parse_drawing_images(missing_from, rels_xml, metrics, archive), + parse_drawing_images( + missing_from, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { e => Err(e) } @@ -1125,7 +1154,14 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { let missing_ext = wb_one_cell_anchor(wb_from_xml(), "") let missing_ext_result : Result[Array[Image], Error] = Ok( - parse_drawing_images(missing_ext, rels_xml, metrics, archive), + parse_drawing_images( + missing_ext, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { e => Err(e) } @@ -1140,7 +1176,14 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { "", ) let cx_invalid_result : Result[Array[Image], Error] = Ok( - parse_drawing_images(cx_invalid, rels_xml, metrics, archive), + parse_drawing_images( + cx_invalid, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { e => Err(e) } @@ -1152,7 +1195,14 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { let cy_missing = wb_one_cell_anchor(wb_from_xml(), "") let cy_missing_result : Result[Array[Image], Error] = Ok( - parse_drawing_images(cy_missing, rels_xml, metrics, archive), + parse_drawing_images( + cy_missing, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { e => Err(e) } @@ -1167,7 +1217,14 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { "2147483647000", ) let invalid_to_result : Result[Array[Image], Error] = Ok( - parse_drawing_images(invalid_to, rels_xml, metrics, archive), + parse_drawing_images( + invalid_to, + rels_xml, + "xl/drawings/drawing1.xml", + Map([]), + metrics, + archive, + ), ) catch { error => Err(error) } diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt new file mode 100644 index 00000000..09817a89 --- /dev/null +++ b/xlsx/shared_formula_translate.mbt @@ -0,0 +1,573 @@ +///| +/// One non-empty shared-formula master. The coordinate is retained because +/// OOXML stores follower formulas as relative translations of this cell, not +/// as copies of the master's literal text. +pub struct SharedFormulaMaster { + priv formula : String + priv row : Int + priv column : Int + priv row_lo : Int + priv column_lo : Int + priv row_hi : Int + priv column_hi : Int +} + +///| +priv struct FormulaCellToken { + start : Int + end : Int + column_start : Int + column_end : Int + row_start : Int + row_end : Int + absolute_column : Bool + absolute_row : Bool + column : Int + row : Int +} + +///| +priv struct FormulaColumnToken { + start : Int + end : Int + absolute : Bool + column : Int +} + +///| +priv struct FormulaRowToken { + start : Int + end : Int + absolute : Bool + row : Int +} + +///| +fn formula_chars_text( + chars : ArrayView[Char], + start : Int, + end : Int, +) -> String { + let out = StringBuilder::new() + for index in start.. Bool { + character.is_ascii_alphabetic() || + character.is_ascii_digit() || + character == '_' || + character == '.' || + character == '$' || + character == '\\' || + character == '?' || + character.to_int() > 0x7f +} + +///| +fn formula_reference_start_boundary( + chars : ArrayView[Char], + start : Int, +) -> Bool { + start == 0 || !formula_reference_word_char(chars[start - 1]) +} + +///| +fn formula_reference_end_boundary(chars : ArrayView[Char], end : Int) -> Bool { + if end >= chars.length() { + return true + } + let character = chars[end] + !formula_reference_word_char(character) && + character != '(' && + character != '[' && + character != '!' +} + +///| +fn formula_column_number( + chars : ArrayView[Char], + start : Int, + end : Int, +) -> Int? { + let mut column = 0 + for index in start.. excel_max_cols { + return None + } + } + if column > 0 { + Some(column) + } else { + None + } +} + +///| +fn formula_bounded_decimal( + chars : ArrayView[Char], + start : Int, + end : Int, + maximum : Int, +) -> Int? { + if start >= end { + return None + } + let mut value = 0 + for index in start.. (maximum - digit) / 10 { + return None + } + value = value * 10 + digit + } + if value > 0 && value <= maximum { + Some(value) + } else { + None + } +} + +///| +fn parse_formula_cell_token( + chars : ArrayView[Char], + start : Int, +) -> FormulaCellToken? { + if start < 0 || start >= chars.length() { + return None + } + let mut index = start + let absolute_column = chars[index] == '$' + if absolute_column { + index += 1 + } + let column_start = index + while index < chars.length() && chars[index].is_ascii_alphabetic() { + index += 1 + } + let column_end = index + if column_end == column_start || column_end - column_start > 3 { + return None + } + let absolute_row = index < chars.length() && chars[index] == '$' + if absolute_row { + index += 1 + } + let row_start = index + while index < chars.length() && chars[index].is_ascii_digit() { + index += 1 + } + let row_end = index + guard formula_column_number(chars, column_start, column_end) is Some(column) && + formula_bounded_decimal(chars, row_start, row_end, excel_max_rows) + is Some(row) else { + return None + } + Some({ + start, + end: index, + column_start, + column_end, + row_start, + row_end, + absolute_column, + absolute_row, + column, + row, + }) +} + +///| +fn parse_formula_column_token( + chars : ArrayView[Char], + start : Int, +) -> FormulaColumnToken? { + if start < 0 || start >= chars.length() { + return None + } + let mut index = start + let absolute = chars[index] == '$' + if absolute { + index += 1 + } + let letters_start = index + while index < chars.length() && chars[index].is_ascii_alphabetic() { + index += 1 + } + let letters_end = index + if letters_end == letters_start || letters_end - letters_start > 3 { + return None + } + guard formula_column_number(chars, letters_start, letters_end) is Some(column) else { + return None + } + Some({ start, end: index, absolute, column }) +} + +///| +fn parse_formula_row_token( + chars : ArrayView[Char], + start : Int, +) -> FormulaRowToken? { + if start < 0 || start >= chars.length() { + return None + } + let mut index = start + let absolute = chars[index] == '$' + if absolute { + index += 1 + } + let digits_start = index + while index < chars.length() && chars[index].is_ascii_digit() { + index += 1 + } + let digits_end = index + guard formula_bounded_decimal(chars, digits_start, digits_end, excel_max_rows) + is Some(row) else { + return None + } + Some({ start, end: index, absolute, row }) +} + +///| +fn formula_shifted_column(column : Int, delta : Int) -> String? { + let shifted = column + delta + if shifted < 1 || shifted > excel_max_cols { + None + } else { + Some(format_column_name_unchecked(shifted)) + } +} + +///| +fn formula_shifted_row(row : Int, delta : Int) -> String? { + let shifted = row + delta + if shifted < 1 || shifted > excel_max_rows { + None + } else { + Some(shifted.to_string()) + } +} + +///| +fn shifted_formula_cell_text( + chars : ArrayView[Char], + token : FormulaCellToken, + column_delta : Int, + row_delta : Int, +) -> String? { + let out = StringBuilder::new() + if token.absolute_column { + out.write_char('$') + for index in token.column_start.. String? { + if token.absolute { + return Some(formula_chars_text(chars, token.start, token.end)) + } + formula_shifted_column(token.column, delta) +} + +///| +fn shifted_formula_row_text( + chars : ArrayView[Char], + token : FormulaRowToken, + delta : Int, +) -> String? { + if token.absolute { + return Some(formula_chars_text(chars, token.start, token.end)) + } + formula_shifted_row(token.row, delta) +} + +///| +fn formula_range_separator_end(chars : ArrayView[Char], start : Int) -> Int? { + let mut index = start + while index < chars.length() && chars[index].is_ascii_whitespace() { + index += 1 + } + if index >= chars.length() || chars[index] != ':' { + return None + } + index += 1 + while index < chars.length() && chars[index].is_ascii_whitespace() { + index += 1 + } + Some(index) +} + +///| +fn translate_formula_cell_reference_at( + chars : ArrayView[Char], + start : Int, + column_delta : Int, + row_delta : Int, +) -> (String, Int)? { + guard formula_reference_start_boundary(chars, start) && + parse_formula_cell_token(chars, start) is Some(first) else { + return None + } + match formula_range_separator_end(chars, first.end) { + Some(second_start) => + match parse_formula_cell_token(chars, second_start) { + Some(second) if formula_reference_end_boundary(chars, second.end) => { + let first_text = shifted_formula_cell_text( + chars, first, column_delta, row_delta, + ) + let second_text = shifted_formula_cell_text( + chars, second, column_delta, row_delta, + ) + return Some( + ( + first_text.unwrap_or("#REF!") + + formula_chars_text(chars, first.end, second.start) + + second_text.unwrap_or("#REF!"), + second.end, + ), + ) + } + _ => () + } + None => () + } + if !formula_reference_end_boundary(chars, first.end) { + return None + } + Some( + ( + shifted_formula_cell_text(chars, first, column_delta, row_delta).unwrap_or( + "#REF!", + ), + first.end, + ), + ) +} + +///| +fn translate_formula_column_range_at( + chars : ArrayView[Char], + start : Int, + column_delta : Int, +) -> (String, Int)? { + guard formula_reference_start_boundary(chars, start) && + parse_formula_column_token(chars, start) is Some(first) && + formula_range_separator_end(chars, first.end) is Some(second_start) && + parse_formula_column_token(chars, second_start) is Some(second) && + formula_reference_end_boundary(chars, second.end) else { + return None + } + let first_text = shifted_formula_column_text(chars, first, column_delta) + let second_text = shifted_formula_column_text(chars, second, column_delta) + Some( + ( + first_text.unwrap_or("#REF!") + + formula_chars_text(chars, first.end, second.start) + + second_text.unwrap_or("#REF!"), + second.end, + ), + ) +} + +///| +fn translate_formula_row_range_at( + chars : ArrayView[Char], + start : Int, + row_delta : Int, +) -> (String, Int)? { + guard formula_reference_start_boundary(chars, start) && + parse_formula_row_token(chars, start) is Some(first) && + formula_range_separator_end(chars, first.end) is Some(second_start) && + parse_formula_row_token(chars, second_start) is Some(second) && + formula_reference_end_boundary(chars, second.end) else { + return None + } + let first_text = shifted_formula_row_text(chars, first, row_delta) + let second_text = shifted_formula_row_text(chars, second, row_delta) + Some( + ( + first_text.unwrap_or("#REF!") + + formula_chars_text(chars, first.end, second.start) + + second_text.unwrap_or("#REF!"), + second.end, + ), + ) +} + +///| +fn copy_formula_quoted_token( + chars : ArrayView[Char], + start : Int, + quote : Char, + out : StringBuilder, +) -> Int { + let mut index = start + out.write_char(chars[index]) + index += 1 + while index < chars.length() { + let character = chars[index] + out.write_char(character) + index += 1 + if character == quote { + if index < chars.length() && chars[index] == quote { + out.write_char(chars[index]) + index += 1 + } else { + break + } + } + } + index +} + +///| +fn copy_formula_bracket_token( + chars : ArrayView[Char], + start : Int, + out : StringBuilder, +) -> Int { + let mut depth = 0 + let mut index = start + while index < chars.length() { + let character = chars[index] + out.write_char(character) + index += 1 + if character == '[' { + depth += 1 + } else if character == ']' { + depth -= 1 + if depth == 0 { + break + } + } + } + index +} + +///| +/// Applies Excel copy semantics to A1 references while preserving formula +/// text outside reference tokens. Double-quoted literals, quoted sheet names, +/// structured-reference brackets, and external-workbook brackets are copied +/// verbatim. Cell/range, whole-column, and whole-row references honor `$` +/// anchors; translations outside the XLSX grid become `#REF!`. +fn translate_shared_formula( + formula : String, + column_delta : Int, + row_delta : Int, +) -> String { + if formula == "" || (column_delta == 0 && row_delta == 0) { + return formula + } + let chars = formula.to_array() + let out = StringBuilder::new() + let mut index = 0 + while index < chars.length() { + let character = chars[index] + if character == '"' || character == '\'' { + index = copy_formula_quoted_token(chars, index, character, out) + continue + } + if character == '[' { + index = copy_formula_bracket_token(chars, index, out) + continue + } + let translated = match + translate_formula_cell_reference_at(chars, index, column_delta, row_delta) { + Some(value) => Some(value) + None => + match translate_formula_column_range_at(chars, index, column_delta) { + Some(value) => Some(value) + None => translate_formula_row_range_at(chars, index, row_delta) + } + } + match translated { + Some((text, next)) => { + out.write_string(text) + index = next + } + None => { + out.write_char(character) + index += 1 + } + } + } + out.to_string() +} + +///| +/// Resolves the formula text represented by this shared master at a follower +/// coordinate. Coordinates outside the master's declared shared range are +/// rejected as malformed OOXML. +pub fn SharedFormulaMaster::translate_to( + self : SharedFormulaMaster, + row : Int, + column : Int, +) -> String raise XlsxError { + ignore(cell_ref_from(row, column)) + if row < self.row_lo || + row > self.row_hi || + column < self.column_lo || + column > self.column_hi { + raise InvalidXml(msg="shared formula follower is outside the master range") + } + translate_shared_formula(self.formula, column - self.column, row - self.row) +} + +///| +test "shared formulas translate relative A1 references without touching literals" { + let formula = + #|D1+$D1+D$1+$D$1+SUM(D1:E2)+'Q1'!D1+"D1"+Table1[D1]+A:A+$A:B+1:1+$1:2+LOG10(100)+A1!B2+[Book.xlsx]Sheet1!C3 + inspect( + translate_shared_formula(formula, 2, 2), + content=( + #|F3+$D3+F$1+$D$1+SUM(F3:G4)+'Q1'!F3+"D1"+Table1[D1]+C:C+$A:D+3:3+$1:4+LOG10(100)+A1!D4+[Book.xlsx]Sheet1!E5 + ), + ) +} + +///| +test "shared formulas emit REF for translated coordinates outside the grid" { + inspect( + translate_shared_formula("A1:XFD1048576", -1, 1), + content="#REF!:#REF!", + ) + inspect(translate_shared_formula("A1:B2", -1, 0), content="#REF!:A2") + inspect(translate_shared_formula("XFD1", 1, 0), content="#REF!") +} diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index e3011465..00522b63 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -386,21 +386,51 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { ///| /// Returns non-empty shared-formula masters keyed by OOXML shared index. -/// Followers are deliberately omitted, so master formula text is retained -/// only once even for large shared ranges. Conflicting masters are rejected -/// rather than resolved by iteration order. +/// Followers are deliberately omitted, so master formula text and its copy +/// anchor are retained only once even for large shared ranges. Conflicting +/// masters are rejected rather than resolved by iteration order. pub fn Worksheet::shared_formula_masters( self : Worksheet, -) -> Map[Int, String] raise XlsxError { - let masters : Map[Int, String] = Map([]) +) -> Map[Int, SharedFormulaMaster] raise XlsxError { + let masters : Map[Int, SharedFormulaMaster] = Map([]) for cell in self.cells { - match (cell.formula_type, cell.formula, cell.formula_shared_index) { - (Some(Shared), Some(formula), Some(shared_index)) if formula != "" => - match masters.get(shared_index) { - Some(existing) if existing != formula => - raise InvalidXml(msg="shared formula index has conflicting masters") - _ => masters[shared_index] = formula + match + ( + cell.formula_type, + cell.formula, + cell.formula_shared_index, + cell.formula_ref, + ) { + (Some(Shared), Some(formula), Some(shared_index), Some(range_ref)) => + if formula != "" { + let (row_lo, column_lo, row_hi, column_hi) = parse_range_ref( + range_ref, + ) + if cell.row < row_lo || + cell.row > row_hi || + cell.col < column_lo || + cell.col > column_hi { + raise InvalidXml(msg="shared formula master is outside its range") + } + let master : SharedFormulaMaster = { + formula, + row: cell.row, + column: cell.col, + row_lo, + column_lo, + row_hi, + column_hi, + } + match masters.get(shared_index) { + Some(_) => + raise InvalidXml( + msg="shared formula index has conflicting masters", + ) + None => masters[shared_index] = master + } } + (Some(Shared), Some(formula), Some(_), None) if formula != "" => + raise InvalidXml(msg="shared formula master range missing") _ => () } } From 8329a5ab27b7d2ccf04a7f2d764fb07a82653ea1 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 13:55:41 +0800 Subject: [PATCH 015/150] fix(xlsx): preserve 3-D formulas and default main parts --- office/cmd/office/xlsx_read_wbtest.mbt | 51 +++++++++++++++ xlsx/read.mbt | 2 +- xlsx/read_package_parts.mbt | 86 ++++++++++++++++++++++---- xlsx/shared_formula_translate.mbt | 71 ++++++++++++++++++++- 4 files changed, 194 insertions(+), 16 deletions(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index da5c9c64..14e5e5f5 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -131,6 +131,38 @@ fn xlsx_relocated_test_bytes() -> Bytes raise { @zip.write(archive) } +///| +fn xlsx_default_declared_relocated_test_bytes() -> Bytes raise { + let source = @zip.read(xlsx_relocated_test_bytes()) + let archive = @zip.Archive::new() + let workbook_override = + #| + let workbook_default = + #| + for entry in source.entries() { + let data = if entry.name() == "[Content_Types].xml" { + let content_types = @utf8.decode(entry.data(), ignore_bom=true) + let updated = content_types.replace( + old=workbook_override, + new=workbook_default, + ) + if updated == content_types { + fail("relocated fixture did not contain its workbook override") + } + @utf8.encode(updated) + } else { + entry.data().to_owned() + } + archive.add( + entry.name(), + data, + compression=entry.compression(), + data_descriptor=entry.data_descriptor(), + ) + } + @zip.write(archive) +} + ///| test "XLSX selectors resolve to stable named canonical paths" { let projection = xlsx_read_test_projection() @@ -219,6 +251,25 @@ test "XLSX structured reads resolve relationships from relocated workbooks" { assert_eq(chart_counts.get("charts"), Some(Json::number(1))) } +///| +test "XLSX structured reads retain a default-declared relocated workbook path" { + let source = xlsx_bounded_test_source( + "relocated-default.xlsx", + xlsx_default_declared_relocated_test_bytes(), + ) + assert_true(@lib.detect_archive_format(source.file, source.archive) is Xlsx) + let projection = open_xlsx_projection(source, 10) + guard xlsx_get_data( + projection, + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Moved\"]/cell[A1]"), + ) + is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected default-declared relocated XLSX cell") + } + assert_eq(cell.get("value"), Some(Json::string("relocated-value"))) +} + ///| test "XLSX selector failures distinguish format, absence, and scan limits" { let projection = xlsx_read_test_projection() diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 8795bf68..f7587375 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3168,7 +3168,7 @@ fn read_zip_archive_core_unchecked( } let workbook_xml_part_path = actual_archive_part_path( part_names, - resolve_workbook_xml_part_path(archive, decode), + resolve_workbook_xml_part_path(archive, part_names, decode), ) let workbook_bytes = match archive.get(workbook_xml_part_path) { Some(value) => value diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index c33a0656..c623ebd9 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -6,12 +6,42 @@ fn workbook_content_type_candidates() -> Array[String] { ] } +///| +fn workbook_part_from_root_relationships( + archive : @zip.Archive, + part_names : Map[String, String], + decode : (BytesView) -> String raise XlsxError, +) -> String? raise XlsxError { + let root_rels_path = actual_archive_part_path(part_names, root_rels_part_path) + let root_rels_xml = match archive.get(root_rels_path) { + Some(bytes) => decode(bytes) + None => return None + } + let office_document_type = transitional_office_relationship_prefix + + "officeDocument" + let targets = parse_relationship_targets(root_rels_xml, office_document_type) + if targets.length() > 1 { + raise InvalidXml(msg="root relationships contain multiple workbook targets") + } + match first_relationship_target(targets) { + Some(target) => { + let part_name = archive_path_from_part_name(unescape_xml_text(target)) + Some(normalize_rel_part_path(part_name)) + } + None => None + } +} + ///| fn resolve_workbook_xml_part_path( archive : @zip.Archive, + part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, ) -> String raise XlsxError { - match archive.get(content_types_part_path) { + let content_types_path = actual_archive_part_path( + part_names, content_types_part_path, + ) + match archive.get(content_types_path) { Some(content_types_bytes) => { let content_types_xml = decode(content_types_bytes) let workbook_part_name = @ooxml.find_override_part_by_content_types( @@ -22,10 +52,16 @@ fn resolve_workbook_xml_part_path( } match workbook_part_name { Some(part_name) => archive_path_from_part_name(part_name) - None => workbook_part_path + None => + workbook_part_from_root_relationships(archive, part_names, decode).unwrap_or( + workbook_part_path, + ) } } - None => workbook_part_path + None => + workbook_part_from_root_relationships(archive, part_names, decode).unwrap_or( + workbook_part_path, + ) } } @@ -36,11 +72,22 @@ fn decode_utf8_or_invalid_xml(bytes : BytesView) -> String raise XlsxError { } } +///| +fn resolve_workbook_xml_part_path_for_test( + archive : @zip.Archive, +) -> String raise XlsxError { + resolve_workbook_xml_part_path( + archive, + archive_part_name_index(archive), + decode_utf8_or_invalid_xml, + ) +} + ///| test "read package parts: default workbook path when [Content_Types] missing" { let archive = @zip.Archive::new() inspect( - resolve_workbook_xml_part_path(archive, decode_utf8_or_invalid_xml), + resolve_workbook_xml_part_path_for_test(archive), content="xl/workbook.xml", ) } @@ -56,7 +103,7 @@ test "read package parts: workbook path resolved from [Content_Types] override" #| archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) inspect( - resolve_workbook_xml_part_path(archive, decode_utf8_or_invalid_xml), + resolve_workbook_xml_part_path_for_test(archive), content="xl/workbook2.xml", ) } @@ -72,7 +119,7 @@ test "read package parts: fallback to default when workbook content type absent" #| archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) inspect( - resolve_workbook_xml_part_path(archive, decode_utf8_or_invalid_xml), + resolve_workbook_xml_part_path_for_test(archive), content="xl/workbook.xml", ) } @@ -87,9 +134,7 @@ test "read package parts: malformed [Content_Types] raises InvalidXml" { #| #| archive.add(content_types_part_path, @encoding/utf8.encode(malformed)) - let result = Ok( - resolve_workbook_xml_part_path(archive, decode_utf8_or_invalid_xml), - ) catch { + let result = Ok(resolve_workbook_xml_part_path_for_test(archive)) catch { e => Err(e) } match result { @@ -109,9 +154,7 @@ test "read package parts: invalid workbook part name raises InvalidXml" { #| #| archive.add(content_types_part_path, @encoding/utf8.encode(invalid_part_name)) - let result = Ok( - resolve_workbook_xml_part_path(archive, decode_utf8_or_invalid_xml), - ) catch { + let result = Ok(resolve_workbook_xml_part_path_for_test(archive)) catch { e => Err(e) } match result { @@ -120,3 +163,22 @@ test "read package parts: invalid workbook part name raises InvalidXml" { _ => fail("expected content type part name empty") } } + +///| +test "read package parts: default-declared workbook follows the root relationship" { + let archive = @zip.Archive::new() + let content_types = + #| + #| + #| + let root_relationships = + #| + #| + #| + archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + inspect( + resolve_workbook_xml_part_path_for_test(archive), + content="custom/book.xml", + ) +} diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index 09817a89..e69979e2 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -481,12 +481,55 @@ fn copy_formula_bracket_token( index } +///| +fn formula_unquoted_sheet_name_char(character : Char) -> Bool { + character.is_ascii_alphabetic() || + character.is_ascii_digit() || + character == '_' || + character == '.' || + character == '$' || + character.to_int() > 0x7f +} + +///| +/// Returns the end of an unquoted 3-D sheet qualifier such as +/// `Sheet1:Sheet3!`. Sheet names that happen to look like A1 coordinates must +/// remain opaque; only the cell/range after `!` participates in copy +/// translation. +fn formula_unquoted_3d_qualifier_end( + chars : ArrayView[Char], + start : Int, +) -> Int? { + if start < 0 || + start >= chars.length() || + !formula_reference_start_boundary(chars, start) { + return None + } + let mut index = start + while index < chars.length() && formula_unquoted_sheet_name_char(chars[index]) { + index += 1 + } + if index == start || index >= chars.length() || chars[index] != ':' { + return None + } + index += 1 + let second_start = index + while index < chars.length() && formula_unquoted_sheet_name_char(chars[index]) { + index += 1 + } + if index == second_start || index >= chars.length() || chars[index] != '!' { + return None + } + Some(index + 1) +} + ///| /// Applies Excel copy semantics to A1 references while preserving formula /// text outside reference tokens. Double-quoted literals, quoted sheet names, -/// structured-reference brackets, and external-workbook brackets are copied -/// verbatim. Cell/range, whole-column, and whole-row references honor `$` -/// anchors; translations outside the XLSX grid become `#REF!`. +/// unquoted 3-D sheet ranges, structured-reference brackets, and +/// external-workbook brackets are copied verbatim. Cell/range, whole-column, +/// and whole-row references honor `$` anchors; translations outside the XLSX +/// grid become `#REF!`. fn translate_shared_formula( formula : String, column_delta : Int, @@ -508,6 +551,16 @@ fn translate_shared_formula( index = copy_formula_bracket_token(chars, index, out) continue } + match formula_unquoted_3d_qualifier_end(chars, index) { + Some(end) => { + for position in index.. () + } let translated = match translate_formula_cell_reference_at(chars, index, column_delta, row_delta) { Some(value) => Some(value) @@ -571,3 +624,15 @@ test "shared formulas emit REF for translated coordinates outside the grid" { inspect(translate_shared_formula("A1:B2", -1, 0), content="#REF!:A2") inspect(translate_shared_formula("XFD1", 1, 0), content="#REF!") } + +///| +test "shared formulas preserve unquoted 3-D sheet qualifiers" { + let formula = + #|SUM(Q1:Q4!A1)+SUM(Sheet_1:Sheet_4!B2:C3)+SUM([Book.xlsx]Q1:Q4!$D4) + inspect( + translate_shared_formula(formula, 2, 1), + content=( + #|SUM(Q1:Q4!C2)+SUM(Sheet_1:Sheet_4!D3:E4)+SUM([Book.xlsx]Q1:Q4!$D5) + ), + ) +} From 7c7f0a1768198a544790debd157a340b5841025d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 22:07:15 +0800 Subject: [PATCH 016/150] chore: migrate executable package manifests --- cmd/demos/moon.pkg | 4 +--- cmd/main/moon.pkg | 4 +--- cmd/parity/moon.pkg | 4 +--- cmd/xlsx/moon.pkg | 6 ++---- docx2html/cmd/docx/moon.pkg | 4 +--- docx2html/cmd/docx2html/moon.pkg | 4 +--- pdflite/cmd/main/moon.pkg | 4 +--- pdflite/markdown/cmd/moon.pkg | 4 +--- 8 files changed, 9 insertions(+), 25 deletions(-) diff --git a/cmd/demos/moon.pkg b/cmd/demos/moon.pkg index 2d03c6d1..038f3fec 100644 --- a/cmd/demos/moon.pkg +++ b/cmd/demos/moon.pkg @@ -8,6 +8,4 @@ import { supported_targets = "native" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/cmd/main/moon.pkg b/cmd/main/moon.pkg index 2fdbf8ee..1bb66b3b 100644 --- a/cmd/main/moon.pkg +++ b/cmd/main/moon.pkg @@ -4,6 +4,4 @@ import { supported_targets = "native+js" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/cmd/parity/moon.pkg b/cmd/parity/moon.pkg index e3b38c63..633fc775 100644 --- a/cmd/parity/moon.pkg +++ b/cmd/parity/moon.pkg @@ -9,6 +9,4 @@ import { supported_targets = "native" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/cmd/xlsx/moon.pkg b/cmd/xlsx/moon.pkg index 8bb62058..52dc32d8 100644 --- a/cmd/xlsx/moon.pkg +++ b/cmd/xlsx/moon.pkg @@ -7,7 +7,7 @@ import { "moonbitlang/core/argparse", "moonbitlang/async", "moonbitlang/async/fs" @async/fs, - "moonbitlang/async/os_error" @os_error, + "moonbitlang/async/os_error", "moonbitlang/core/env", "moonbitlang/core/encoding/utf8" @encoding/utf8, "moonbitlang/core/string", @@ -16,6 +16,4 @@ import { supported_targets = "native+wasm" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/docx2html/cmd/docx/moon.pkg b/docx2html/cmd/docx/moon.pkg index e30b6f8c..3a62532a 100644 --- a/docx2html/cmd/docx/moon.pkg +++ b/docx2html/cmd/docx/moon.pkg @@ -18,6 +18,4 @@ import { supported_targets = "native+wasm" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/docx2html/cmd/docx2html/moon.pkg b/docx2html/cmd/docx2html/moon.pkg index 9b533d3f..67ffbcf4 100644 --- a/docx2html/cmd/docx2html/moon.pkg +++ b/docx2html/cmd/docx2html/moon.pkg @@ -9,6 +9,4 @@ import { supported_targets = "native+wasm" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/pdflite/cmd/main/moon.pkg b/pdflite/cmd/main/moon.pkg index 30d0fb23..b0be1ecb 100644 --- a/pdflite/cmd/main/moon.pkg +++ b/pdflite/cmd/main/moon.pkg @@ -23,6 +23,4 @@ import { supported_targets = "+native" -options( - "is-main": true, -) +pkgtype(kind: "executable") diff --git a/pdflite/markdown/cmd/moon.pkg b/pdflite/markdown/cmd/moon.pkg index d5b1be6e..e426f91d 100644 --- a/pdflite/markdown/cmd/moon.pkg +++ b/pdflite/markdown/cmd/moon.pkg @@ -19,6 +19,4 @@ import { supported_targets = "+native" -options( - "is-main": true, -) +pkgtype(kind: "executable") From acc8234ee05c2e96f8979067106e68d860fa73ed Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 22:09:05 +0800 Subject: [PATCH 017/150] fix(xlsx): bound semantic read work --- office/cmd/office/docx_commands.mbt | 16 +- office/cmd/office/read_package.mbt | 15 +- office/cmd/office/xlsx_read_model.mbt | 3 +- xlsx/errors.mbt | 1 + xlsx/io.mbt | 28 ++- xlsx/options.mbt | 55 +++++- xlsx/options_test.mbt | 18 ++ xlsx/pkg.generated.mbti | 12 +- xlsx/read.mbt | 105 ++++++++--- xlsx/read_budget.mbt | 240 ++++++++++++++++++++++++++ xlsx/read_limits_test.mbt | 123 +++++++++++++ xlsx/read_workbook_xml.mbt | 8 + 12 files changed, 580 insertions(+), 44 deletions(-) create mode 100644 xlsx/read_budget.mbt diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index f0b34400..497acd0a 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -156,7 +156,9 @@ async fn run_docx_outline(matches : @argparse.Matches) -> Unit { } } Xlsx => { - let projection = open_xlsx_projection(source, max_elements) + let projection = open_xlsx_projection(source, max_elements, cancelled=() => { + @async.is_being_cancelled() + }) if matches.flags.get_or_default("json", false) { emit_docx_success( checked_office_json_output( @@ -217,7 +219,9 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { } } Xlsx => { - let projection = open_xlsx_projection(source, max_elements) + let projection = open_xlsx_projection(source, max_elements, cancelled=() => { + @async.is_being_cancelled() + }) let resolved = resolve_xlsx_selector(projection, selector) if matches.flags.get_or_default("json", false) { emit_docx_success( @@ -283,7 +287,9 @@ async fn run_docx_text(matches : @argparse.Matches) -> Unit { } } Xlsx => { - let projection = open_xlsx_projection(source, max_elements) + let projection = open_xlsx_projection(source, max_elements, cancelled=() => { + @async.is_being_cancelled() + }) let under = resolve_optional_xlsx_under(projection, matches) if matches.flags.get_or_default("json", false) { emit_docx_success( @@ -567,7 +573,9 @@ async fn run_docx_query(matches : @argparse.Matches) -> Unit { Xlsx => { reject_xlsx_query_options(matches) let selector = optional_value(matches, "selector").unwrap_or("cell") - let projection = open_xlsx_projection(source, max_elements) + let projection = open_xlsx_projection(source, max_elements, cancelled=() => { + @async.is_being_cancelled() + }) let under = resolve_optional_xlsx_under(projection, matches) let spec : XlsxQuerySpec = { selector, diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 551e1cd6..433ecc2d 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -16,6 +16,15 @@ let office_read_max_preserved_source_bytes : Int = 64 * 1024 * 1024 + 65_535 ///| let office_read_max_xml_part_bytes : Int = 16 * 1024 * 1024 +///| +let office_read_max_workbook_sheets : Int = 256 + +///| +let office_read_max_parser_items : Int = 500_000 + +///| +let office_read_max_parser_work_units : Int = 256 * 1024 * 1024 + ///| struct OfficeReadPackage { file : String @@ -79,6 +88,9 @@ fn office_xlsx_read_limits() -> @xlsx.ReadLimits raise @xlsx.XlsxError { max_total_uncompressed_bytes=office_read_max_zip_total_bytes, max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, max_xml_part_bytes=office_read_max_xml_part_bytes, + max_workbook_sheets=office_read_max_workbook_sheets, + max_parser_items=office_read_max_parser_items, + max_parser_work_units=office_read_max_parser_work_units, ) } @@ -153,6 +165,7 @@ fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { ///| fn open_xlsx_read_package( source : OfficeReadPackage, + cancelled? : () -> Bool = () => false, ) -> @xlsx.Workbook raise CliFailure { if source.format is Docx { raise CliFailure( @@ -164,7 +177,7 @@ fn open_xlsx_read_package( let limits = office_xlsx_read_limits() catch { error => raise xlsx_read_failure(error, source.file) } - @xlsx.read_bounded_archive(source.archive, limits~) catch { + @xlsx.read_bounded_archive(source.archive, limits~, cancelled~) catch { error => raise xlsx_read_failure(error, source.file) } } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 8113b853..794aa968 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -254,8 +254,9 @@ fn make_xlsx_projection( fn open_xlsx_projection( source : OfficeReadPackage, max_elements : Int, + cancelled? : () -> Bool = () => false, ) -> XlsxProjection raise CliFailure { - let workbook = open_xlsx_read_package(source) + let workbook = open_xlsx_read_package(source, cancelled~) make_xlsx_projection(source.file, workbook, max_elements) } diff --git a/xlsx/errors.mbt b/xlsx/errors.mbt index 877211ba..5d9888db 100644 --- a/xlsx/errors.mbt +++ b/xlsx/errors.mbt @@ -49,5 +49,6 @@ pub suberror XlsxError { UnsupportedFeature(msg~ : String) InvalidPackage(msg~ : String) UnboundedArchive + ReadCancelled ResourceLimitExceeded(kind~ : String, limit~ : Int, actual~ : Int) } derive(Debug) diff --git a/xlsx/io.mbt b/xlsx/io.mbt index c39dc7e1..4b2df3d6 100644 --- a/xlsx/io.mbt +++ b/xlsx/io.mbt @@ -242,6 +242,7 @@ fn read_workbook_from_bytes( options : Options, limits : ReadLimits, io_context : WorkbookIOContext, + cancelled : () -> Bool, ) -> Workbook raise XlsxError { let resolved_password = if password == "" { options.password @@ -256,9 +257,11 @@ fn read_workbook_from_bytes( bytes, options=resolved_options, limits~, + cancelled~, transcoder=value, ) - None => read_zip_bytes(bytes, options=resolved_options, limits~) + None => + read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) } } else { match io_context.charset_transcoder { @@ -268,6 +271,7 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, + cancelled~, transcoder=value, ) None => @@ -276,6 +280,7 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, + cancelled~, ) } } @@ -298,7 +303,12 @@ pub async fn[R : @async/io.Reader] open_reader( ) let io_context = read_io_context(transcoder) let workbook = read_workbook_from_bytes( - bytes, password, options, limits, io_context, + bytes, + password, + options, + limits, + io_context, + () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook @@ -347,7 +357,12 @@ pub async fn[R : @async/io.Reader] Workbook::read_zip_reader( resource="package_bytes", ) let workbook = read_workbook_from_bytes( - bytes, password, resolved_options, limits, io_context, + bytes, + password, + resolved_options, + limits, + io_context, + () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook @@ -375,7 +390,12 @@ pub async fn open_file( Some(path), ) let workbook = read_workbook_from_bytes( - bytes, password, options, limits, io_context, + bytes, + password, + options, + limits, + io_context, + () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook diff --git a/xlsx/options.mbt b/xlsx/options.mbt index c737b66c..69ece69d 100644 --- a/xlsx/options.mbt +++ b/xlsx/options.mbt @@ -20,13 +20,24 @@ let default_max_xml_part_bytes : Int = 16 * 1024 * 1024 ///| let default_max_kdf_iterations : Int = 1_000_000 +///| +let default_max_workbook_sheets : Int = 1024 + +///| +let default_max_parser_items : Int = 2_000_000 + +///| +let default_max_parser_work_units : Int = 512 * 1024 * 1024 + ///| /// A complete fail-closed resource policy for reading an XLSX package. /// /// The type is opaque so every public read path receives a validated, /// internally consistent policy. ZIP limits are enforced while inflating; -/// `max_xml_part_bytes` is enforced before any individual XML part is decoded, -/// and `max_kdf_iterations` bounds encrypted-package password derivation. +/// `max_xml_part_bytes` is enforced before any individual XML part is decoded. +/// Semantic ceilings independently bound workbook fan-out, XML-backed parser +/// items, derived materialization, and cumulative parser work. +/// `max_kdf_iterations` bounds encrypted-package password derivation. pub struct ReadLimits { priv max_package_bytes : Int priv max_archive_entries : Int @@ -35,12 +46,16 @@ pub struct ReadLimits { priv max_total_preserved_source_bytes : Int priv max_xml_part_bytes : Int priv max_kdf_iterations : Int + priv max_workbook_sheets : Int + priv max_parser_items : Int + priv max_parser_work_units : Int } ///| /// Returns the production XLSX read policy: 128 MiB source packages, 8,192 /// entries, 64 MiB per entry, 256 MiB aggregate expansion, 16 MiB per XML part, -/// and one million Agile KDF iterations. Call `with_values` for stricter limits. +/// 1,024 sheets, two million parser items, 512 Mi parser work units, and one +/// million Agile KDF iterations. Call `with_values` for stricter limits. pub fn ReadLimits::new() -> ReadLimits { { max_package_bytes: default_max_package_bytes, @@ -50,6 +65,9 @@ pub fn ReadLimits::new() -> ReadLimits { max_total_preserved_source_bytes: default_max_total_preserved_source_bytes, max_xml_part_bytes: default_max_xml_part_bytes, max_kdf_iterations: default_max_kdf_iterations, + max_workbook_sheets: default_max_workbook_sheets, + max_parser_items: default_max_parser_items, + max_parser_work_units: default_max_parser_work_units, } } @@ -64,6 +82,9 @@ pub fn ReadLimits::with_values( max_total_preserved_source_bytes? : Int = default_max_total_preserved_source_bytes, max_xml_part_bytes? : Int = default_max_xml_part_bytes, max_kdf_iterations? : Int = default_max_kdf_iterations, + max_workbook_sheets? : Int = default_max_workbook_sheets, + max_parser_items? : Int = default_max_parser_items, + max_parser_work_units? : Int = default_max_parser_work_units, ) -> ReadLimits raise XlsxError { if max_package_bytes <= 0 || max_archive_entries <= 0 || @@ -71,7 +92,10 @@ pub fn ReadLimits::with_values( max_total_uncompressed_bytes <= 0 || max_total_preserved_source_bytes <= 0 || max_xml_part_bytes <= 0 || - max_kdf_iterations <= 0 { + max_kdf_iterations <= 0 || + max_workbook_sheets <= 0 || + max_parser_items <= 0 || + max_parser_work_units <= 0 { raise InvalidOptions(msg="XLSX read limits must be positive") } if max_entry_uncompressed_bytes > max_total_uncompressed_bytes { @@ -90,6 +114,9 @@ pub fn ReadLimits::with_values( max_total_preserved_source_bytes, max_xml_part_bytes, max_kdf_iterations, + max_workbook_sheets, + max_parser_items, + max_parser_work_units, } } @@ -136,6 +163,26 @@ pub fn ReadLimits::max_kdf_iterations(self : ReadLimits) -> Int { self.max_kdf_iterations } +///| +/// Returns the maximum number of logical sheets accepted in one workbook. +pub fn ReadLimits::max_workbook_sheets(self : ReadLimits) -> Int { + self.max_workbook_sheets +} + +///| +/// Returns the maximum cumulative XML-backed and derived items that a read may +/// inspect or materialize. +pub fn ReadLimits::max_parser_items(self : ReadLimits) -> Int { + self.max_parser_items +} + +///| +/// Returns the maximum cumulative XML source units plus derived expansion work +/// accepted by the parser. +pub fn ReadLimits::max_parser_work_units(self : ReadLimits) -> Int { + self.max_parser_work_units +} + ///| pub struct Options { max_calc_iterations : UInt diff --git a/xlsx/options_test.mbt b/xlsx/options_test.mbt index 505300c7..6d4ae129 100644 --- a/xlsx/options_test.mbt +++ b/xlsx/options_test.mbt @@ -100,6 +100,21 @@ test "read limits reject a non-positive Agile KDF ceiling" { inspect(result is Err(@xlsx.InvalidOptions(_)), content="true") } +///| +test "read limits reject non-positive semantic ceilings" { + for + limits in [ + () => @xlsx.ReadLimits::with_values(max_workbook_sheets=0), + () => @xlsx.ReadLimits::with_values(max_parser_items=0), + () => @xlsx.ReadLimits::with_values(max_parser_work_units=0), + ] { + let result : Result[@xlsx.ReadLimits, Error] = Ok(limits()) catch { + error => Err(error) + } + inspect(result is Err(@xlsx.InvalidOptions(_)), content="true") + } +} + ///| test "default read limits expose the complete ZIP policy" { let limits = @xlsx.ReadLimits::new() @@ -113,4 +128,7 @@ test "default read limits expose the complete ZIP policy" { ) assert_eq(limits.max_xml_part_bytes(), 16 * 1024 * 1024) assert_eq(limits.max_kdf_iterations(), 1_000_000) + assert_eq(limits.max_workbook_sheets(), 1024) + assert_eq(limits.max_parser_items(), 2_000_000) + assert_eq(limits.max_parser_work_units(), 512 * 1024 * 1024) } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index f48720b0..38489e4b 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -31,11 +31,11 @@ pub async fn open_file(String, password? : String, options? : Options, limits? : pub async fn[R : @io.Reader] open_reader(R, password? : String, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook -pub fn read(BytesView, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError +pub fn read(BytesView, options? : Options, limits? : ReadLimits, cancelled? : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError -pub fn read_bounded_archive(@zip.Archive, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError +pub fn read_bounded_archive(@zip.Archive, options? : Options, limits? : ReadLimits, cancelled? : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError -pub fn read_with_password(BytesView, String, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError +pub fn read_with_password(BytesView, String, options? : Options, limits? : ReadLimits, cancelled? : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook raise XlsxError pub async fn[R : @io.Reader] read_zip_reader(R, password? : String, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Workbook @@ -108,6 +108,7 @@ pub suberror XlsxError { UnsupportedFeature(msg~ : String) InvalidPackage(msg~ : String) UnboundedArchive + ReadCancelled ResourceLimitExceeded(kind~ : String, limit~ : Int, actual~ : Int) } derive(@debug.Debug) #deprecated @@ -1218,11 +1219,14 @@ pub fn ReadLimits::max_archive_entries(Self) -> Int pub fn ReadLimits::max_entry_uncompressed_bytes(Self) -> Int pub fn ReadLimits::max_kdf_iterations(Self) -> Int pub fn ReadLimits::max_package_bytes(Self) -> Int +pub fn ReadLimits::max_parser_items(Self) -> Int +pub fn ReadLimits::max_parser_work_units(Self) -> Int pub fn ReadLimits::max_total_preserved_source_bytes(Self) -> Int pub fn ReadLimits::max_total_uncompressed_bytes(Self) -> Int +pub fn ReadLimits::max_workbook_sheets(Self) -> Int pub fn ReadLimits::max_xml_part_bytes(Self) -> Int pub fn ReadLimits::new() -> Self -pub fn ReadLimits::with_values(max_package_bytes? : Int, max_archive_entries? : Int, max_entry_uncompressed_bytes? : Int, max_total_uncompressed_bytes? : Int, max_total_preserved_source_bytes? : Int, max_xml_part_bytes? : Int, max_kdf_iterations? : Int) -> Self raise XlsxError +pub fn ReadLimits::with_values(max_package_bytes? : Int, max_archive_entries? : Int, max_entry_uncompressed_bytes? : Int, max_total_uncompressed_bytes? : Int, max_total_preserved_source_bytes? : Int, max_xml_part_bytes? : Int, max_kdf_iterations? : Int, max_workbook_sheets? : Int, max_parser_items? : Int, max_parser_work_units? : Int) -> Self raise XlsxError pub(all) struct RichTextFont { bold : Bool diff --git a/xlsx/read.mbt b/xlsx/read.mbt index f7587375..b10ad946 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3153,9 +3153,11 @@ fn read_zip_archive_core_unchecked( archive : @zip.Archive, options : Options, limits : ReadLimits, + cancelled : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { let part_names = archive_part_name_index(archive) + let read_budget = ReadBudget::new(limits, cancelled~) let decode = (value : BytesView) => { if value.length() > limits.max_xml_part_bytes { raise ResourceLimitExceeded( @@ -3164,7 +3166,9 @@ fn read_zip_archive_core_unchecked( actual=value.length(), ) } - decode_utf8(value, transcoder) + let decoded = decode_utf8(value, transcoder) + read_budget.scan_xml(decoded) + decoded } let workbook_xml_part_path = actual_archive_part_path( part_names, @@ -3183,7 +3187,7 @@ fn read_zip_archive_core_unchecked( Some(value) => decode(value) None => raise MissingPart(path=workbook_rels_path) } - let sheets = parse_workbook_sheets(workbook_xml) + let sheets = parse_workbook_sheets(workbook_xml, limits.max_workbook_sheets) let sheet_names : Array[String] = [] for entry in sheets { sheet_names.push(entry.name) @@ -3390,24 +3394,47 @@ fn read_zip_archive_core_unchecked( let chartsheet_targets = parse_relationship_targets( workbook_rels, rel_chartsheet, ) + let seen_sheet_parts : Set[String] = Set([]) + let resolved_sheet_parts : Map[String, String] = Map([]) for entry in sheets { - let name = entry.name - let state = entry.state let rel_id = entry.rel_id let is_worksheet = worksheet_targets.contains(rel_id) let is_chartsheet = chartsheet_targets.contains(rel_id) - if !is_worksheet && !is_chartsheet { - raise InvalidXml(msg="sheet relationship missing") + if is_worksheet == is_chartsheet { + raise InvalidXml(msg="sheet relationship missing or ambiguous") + } + let target = if is_worksheet { + match worksheet_targets.get(rel_id) { + Some(value) => value + None => raise InvalidXml(msg="worksheet target missing") + } + } else { + match chartsheet_targets.get(rel_id) { + Some(value) => value + None => raise InvalidXml(msg="chartsheet target missing") + } } + let path = actual_archive_part_path( + part_names, + resolve_part_rel_target(workbook_xml_part_path, target), + ) + let part_key = path.to_lower() + if seen_sheet_parts.contains(part_key) { + raise InvalidXml(msg="sheet part referenced more than once") + } + seen_sheet_parts.add(part_key) + resolved_sheet_parts[rel_id] = path + } + for entry in sheets { + let name = entry.name + let state = entry.state + let rel_id = entry.rel_id + let is_worksheet = worksheet_targets.contains(rel_id) if is_worksheet { - let target = match worksheet_targets.get(rel_id) { + let sheet_path = match resolved_sheet_parts.get(rel_id) { Some(value) => value None => raise InvalidXml(msg="worksheet target missing") } - let sheet_path = actual_archive_part_path( - part_names, - resolve_part_rel_target(workbook_xml_part_path, target), - ) let sheet_bytes = match archive.get(sheet_path) { Some(value) => value None => raise MissingPart(path=sheet_path) @@ -3966,14 +3993,10 @@ fn read_zip_archive_core_unchecked( workbook.sheets.push(sheet) workbook.sheet_order.push(Worksheet(workbook.sheets.length() - 1)) } else { - let target = match chartsheet_targets.get(rel_id) { + let chartsheet_path = match resolved_sheet_parts.get(rel_id) { Some(value) => value None => raise InvalidXml(msg="chartsheet target missing") } - let chartsheet_path = actual_archive_part_path( - part_names, - resolve_part_rel_target(workbook_xml_part_path, target), - ) let chartsheet_bytes = match archive.get(chartsheet_path) { Some(value) => value None => raise MissingPart(path=chartsheet_path) @@ -4048,6 +4071,7 @@ fn read_zip_archive_core( archive : @zip.Archive, options : Options, limits : ReadLimits, + cancelled : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { match transcoder { @@ -4056,6 +4080,7 @@ fn read_zip_archive_core( archive, options, limits, + cancelled, transcoder=decode, ) catch { InvalidCellRef(value~) => @@ -4063,7 +4088,7 @@ fn read_zip_archive_core( other => raise other } None => - read_zip_archive_core_unchecked(archive, options, limits) catch { + read_zip_archive_core_unchecked(archive, options, limits, cancelled) catch { InvalidCellRef(value~) => raise InvalidXml(msg="coordinate reference invalid: \{value}") other => raise other @@ -4076,14 +4101,21 @@ fn read_zip_bytes( bytes : BytesView, options? : Options = Options::new(), limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { let archive = read_limited_archive(bytes, limits) require_bounded_archive(archive, limits) match transcoder { Some(value) => - read_zip_archive_core(archive, options, limits, transcoder=value) - None => read_zip_archive_core(archive, options, limits) + read_zip_archive_core( + archive, + options, + limits, + cancelled, + transcoder=value, + ) + None => read_zip_archive_core(archive, options, limits, cancelled) } } @@ -4096,13 +4128,20 @@ pub fn read_bounded_archive( archive : @zip.Archive, options? : Options = Options::new(), limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { require_bounded_archive(archive, limits) match transcoder { Some(value) => - read_zip_archive_core(archive, options, limits, transcoder=value) - None => read_zip_archive_core(archive, options, limits) + read_zip_archive_core( + archive, + options, + limits, + cancelled, + transcoder=value, + ) + None => read_zip_archive_core(archive, options, limits, cancelled) } } @@ -4116,6 +4155,7 @@ pub fn read( bytes : BytesView, options? : Options = Options::new(), limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { check_source_package_size(bytes, limits) @@ -4127,18 +4167,26 @@ pub fn read( options.password, options~, limits~, + cancelled~, transcoder=value, ) None => - return read_with_password(bytes, options.password, options~, limits~) + return read_with_password( + bytes, + options.password, + options~, + limits~, + cancelled~, + ) } } if is_encrypted_package(bytes) { raise EncryptedPackage } match transcoder { - Some(value) => read_zip_bytes(bytes, options~, limits~, transcoder=value) - None => read_zip_bytes(bytes, options~, limits~) + Some(value) => + read_zip_bytes(bytes, options~, limits~, cancelled~, transcoder=value) + None => read_zip_bytes(bytes, options~, limits~, cancelled~) } } @@ -4152,6 +4200,7 @@ pub fn read_with_password( password : String, options? : Options = Options::new(), limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { check_source_package_size(bytes, limits) @@ -4165,9 +4214,11 @@ pub fn read_with_password( archive, resolved_options, limits, + cancelled, transcoder=value, ) - None => read_zip_archive_core(archive, resolved_options, limits) + None => + read_zip_archive_core(archive, resolved_options, limits, cancelled) } } else { match transcoder { @@ -4176,9 +4227,11 @@ pub fn read_with_password( bytes, options=resolved_options, limits~, + cancelled~, transcoder=value, ) - None => read_zip_bytes(bytes, options=resolved_options, limits~) + None => + read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) } } } diff --git a/xlsx/read_budget.mbt b/xlsx/read_budget.mbt new file mode 100644 index 00000000..45e98fa7 --- /dev/null +++ b/xlsx/read_budget.mbt @@ -0,0 +1,240 @@ +///| +/// Cumulative semantic budget for one workbook read. Every decoded XML source +/// is charged before downstream parsers inspect it. Opening elements account +/// for XML-backed items; range-like constructs that expand into additional +/// in-memory entries charge those entries separately. +priv struct ReadBudget { + limits : ReadLimits + cancelled : () -> Bool + mut parser_items : Int + mut parser_work_units : Int +} + +///| +fn ReadBudget::new( + limits : ReadLimits, + cancelled? : () -> Bool = () => false, +) -> ReadBudget { + { limits, cancelled, parser_items: 0, parser_work_units: 0 } +} + +///| +fn read_budget_actual(current : Int, amount : Int, limit : Int) -> Int { + if amount > limit - current { + limit + 1 + } else { + current + amount + } +} + +///| +fn ReadBudget::charge_items( + self : ReadBudget, + amount : Int, +) -> Unit raise XlsxError { + if amount <= 0 { + return + } + let actual = read_budget_actual( + self.parser_items, + amount, + self.limits.max_parser_items, + ) + if actual > self.limits.max_parser_items { + raise ResourceLimitExceeded( + kind="parser_items", + limit=self.limits.max_parser_items, + actual~, + ) + } + self.parser_items = actual +} + +///| +fn ReadBudget::charge_work( + self : ReadBudget, + amount : Int, +) -> Unit raise XlsxError { + if amount <= 0 { + return + } + let actual = read_budget_actual( + self.parser_work_units, + amount, + self.limits.max_parser_work_units, + ) + if actual > self.limits.max_parser_work_units { + raise ResourceLimitExceeded( + kind="parser_work_units", + limit=self.limits.max_parser_work_units, + actual~, + ) + } + self.parser_work_units = actual +} + +///| +fn ReadBudget::checkpoint(self : ReadBudget) -> Unit raise XlsxError { + if (self.cancelled)() { + raise ReadCancelled + } +} + +///| +fn ReadBudget::tag_end( + self : ReadBudget, + xml : StringView, + start : Int, +) -> Int? raise XlsxError { + let mut index = start + let mut next_checkpoint = start + let mut quote : UInt16? = None + while index < xml.length() { + if index >= next_checkpoint { + self.checkpoint() + next_checkpoint = index + 4096 + } + let unit = xml[index] + match quote { + Some(delimiter) => if unit == delimiter { quote = None } + None => + if unit == ('"' : UInt16) || unit == ('\'' : UInt16) { + quote = Some(unit) + } else if unit == ('>' : UInt16) { + return Some(index) + } + } + index = index + 1 + } + None +} + +///| +fn xml_name_local_start(xml : StringView, start : Int, end : Int) -> Int { + let mut local_start = start + for index in start.. Bool { + let local_start = xml_name_local_start(xml, start, end) + if end - local_start != expected.length() { + return false + } + for index in 0.. Unit raise XlsxError { + guard attr_value(tag, "min") is Some(min_text) && + attr_value(tag, "max") is Some(max_text) else { + return + } + let min_col = @string.parse_int(min_text, base=10) catch { _ => return } + let max_col = @string.parse_int(max_text, base=10) catch { _ => return } + if min_col <= 0 || + max_col < min_col || + min_col > cell_ref_max_cols || + max_col > cell_ref_max_cols { + return + } + let span = max_col - min_col + 1 + self.charge_items(span) + self.charge_work(span) +} + +///| +/// Preflights one decoded XML source in a single quote-aware pass. This runs +/// before any feature parser materializes arrays or maps from the source. +fn ReadBudget::scan_xml( + self : ReadBudget, + xml : StringView, +) -> Unit raise XlsxError { + self.checkpoint() + self.charge_work(xml.length()) + let mut index = 0 + let mut next_checkpoint = 0 + while index < xml.length() { + if index >= next_checkpoint { + self.checkpoint() + next_checkpoint = index + 4096 + } + if xml[index] != ('<' : UInt16) || index + 1 >= xml.length() { + index = index + 1 + continue + } + let first = xml[index + 1] + if first == ('/' : UInt16) || + first == ('?' : UInt16) || + first == ('!' : UInt16) { + index = index + 2 + continue + } + let name_start = index + 1 + let mut name_end = name_start + while name_end < xml.length() && + !is_xml_attr_space_unit(xml[name_end]) && + xml[name_end] != ('/' : UInt16) && + xml[name_end] != ('>' : UInt16) { + name_end = name_end + 1 + } + if name_end == name_start { + index = index + 1 + continue + } + self.charge_items(1) + let tag_end = match self.tag_end(xml, name_end) { + Some(value) => value + None => return + } + if xml_name_local_is(xml, name_start, name_end, "col") { + self.charge_col_expansion(xml[name_end:tag_end]) + } + index = tag_end + 1 + } +} + +///| +test "read budget counts source, elements, and derived column expansion" { + let limits = ReadLimits::with_values( + max_parser_items=5, + max_parser_work_units=1024, + ) + let budget = ReadBudget::new(limits) + budget.scan_xml("") + assert_eq(budget.parser_items, 5) +} + +///| +test "read budget checks cancellation inside a long start tag" { + let checks = [0] + let budget = ReadBudget::new(ReadLimits::new(), cancelled=() => { + checks[0] += 1 + checks[0] >= 3 + }) + let xml = "" + try budget.scan_xml(xml) catch { + ReadCancelled => assert_true(checks[0] >= 3) + _ => fail("expected read cancellation") + } noraise { + _ => fail("expected read cancellation") + } +} diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index 9cae6de6..6fee0be1 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -25,6 +25,19 @@ fn bounded_fixture_archive( ) } +///| +fn replace_xlsx_part( + source : BytesView, + path : String, + content : String, +) -> Bytes raise { + let archive = @zip.read(source) + if !archive.replace(path, @encoding/utf8.encode(content)) { + fail("missing XLSX fixture part: \{path}") + } + @zip.write(archive) +} + ///| fn lying_deflate_package() -> Bytes raise { let archive = @zip.Archive::new() @@ -275,6 +288,116 @@ test "archive read enforces XML limits after provenance verification" { } } +///| +test "semantic read limits reject XML items and derived column expansion" { + let bytes = bounded_workbook_fixture() + let worksheet = + #| + #| + #| + #| + #| + #| + let expanded = replace_xlsx_part(bytes, "xl/worksheets/sheet1.xml", worksheet) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read( + expanded, + limits=@xlsx.ReadLimits::with_values(max_parser_items=1000), + ), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="parser_items") + assert_eq(limit, 1000) + assert_true(actual > limit) + } + _ => fail("expected parser item limit") + } +} + +///| +test "semantic read limits bound cumulative parser work" { + let bytes = bounded_workbook_fixture() + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read( + bytes, + limits=@xlsx.ReadLimits::with_values(max_parser_work_units=1), + ), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="parser_work_units") + assert_eq(limit, 1) + assert_true(actual > limit) + } + _ => fail("expected parser work limit") + } +} + +///| +test "semantic read limits bound sheet fan-out before worksheet parsing" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("One")) + ignore(workbook.add_sheet("Two")) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read( + @xlsx.write(workbook), + limits=@xlsx.ReadLimits::with_values(max_workbook_sheets=1), + ), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="workbook_sheets") + assert_eq(limit, 1) + assert_eq(actual, 2) + } + _ => fail("expected workbook sheet limit") + } +} + +///| +test "duplicate logical sheets cannot reparse one worksheet part" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("One")) + ignore(workbook.add_sheet("Two")) + let bytes = @xlsx.write(workbook) + let archive = @zip.read(bytes) + let rels_path = "xl/_rels/workbook.xml.rels" + let rels = match archive.get(rels_path) { + Some(value) => @encoding/utf8.decode(value) + None => fail("missing workbook relationships") + } + let duplicated = rels.replace_all( + old="worksheets/sheet2.xml", + new="worksheets/sheet1.xml", + ) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(replace_xlsx_part(bytes, rels_path, duplicated)), + ) catch { + error => Err(error) + } + inspect( + result is Err(@xlsx.InvalidXml(msg="sheet part referenced more than once")), + content="true", + ) +} + +///| +test "read cancellation callback stops semantic preflight" { + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(bounded_workbook_fixture(), cancelled=() => true), + ) catch { + error => Err(error) + } + inspect(result is Err(@xlsx.ReadCancelled), content="true") +} + ///| test "logical XML limit follows a worksheet relationship with a non-XML suffix" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index 0d387cf2..84c006d3 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -8,6 +8,7 @@ priv struct WorkbookSheetInfo { ///| fn parse_workbook_sheets( xml : StringView, + max_sheets : Int, ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] let mut first = true @@ -39,6 +40,13 @@ fn parse_workbook_sheets( } None => Visible } + if sheets.length() >= max_sheets { + raise ResourceLimitExceeded( + kind="workbook_sheets", + limit=max_sheets, + actual=max_sheets + 1, + ) + } sheets.push({ name, state, rel_id }) } sheets From 03852d57a449877fccb1e32d03de02bf82c8b46c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 22:29:27 +0800 Subject: [PATCH 018/150] fix(ooxml): resolve namespace-qualified package XML --- office/cmd/office/xlsx_read_wbtest.mbt | 8 +- ooxml/moon.pkg | 1 + ooxml/pkg.generated.mbti | 14 + ooxml/read_parse.mbt | 121 ++-- ooxml/read_parse_test.mbt | 34 +- ooxml/start_tag_scanner.mbt | 821 +++++++++++++++++++++++++ xlsx/ooxml_rels.mbt | 27 +- xlsx/ooxml_rels_error_test.mbt | 2 +- xlsx/read_namespace_test.mbt | 101 +++ xlsx/read_package_parts.mbt | 5 +- xlsx/read_workbook_xml.mbt | 77 ++- 11 files changed, 1118 insertions(+), 93 deletions(-) create mode 100644 ooxml/start_tag_scanner.mbt create mode 100644 xlsx/read_namespace_test.mbt diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 14e5e5f5..0ab7c559 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -40,13 +40,13 @@ fn xlsx_bounded_test_source( fn xlsx_strict_test_bytes() -> Bytes raise { let archive = @zip.Archive::new() let content_types = - #| + #| let root_relationships = - #| + #| let workbook = - #| + #| let workbook_relationships = - #| + #| let worksheet = #|strict-value archive.add("[Content_Types].xml", @utf8.encode(content_types)) diff --git a/ooxml/moon.pkg b/ooxml/moon.pkg index 221b993e..b619fd35 100644 --- a/ooxml/moon.pkg +++ b/ooxml/moon.pkg @@ -1,3 +1,4 @@ import { "moonbitlang/core/debug", + "moonbitlang/core/string", } diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 36ed6e6d..5c05c540 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -8,6 +8,8 @@ import { // Values pub fn attr_value(StringView, StringView) -> String? raise ParseXmlError +pub fn decode_xml_attribute(StringView) -> String raise ParseXmlError + pub fn escape_xml_attr(StringView) -> String pub fn escape_xml_text(StringView) -> String @@ -16,6 +18,8 @@ pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> Str pub fn parse_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError +pub fn parse_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError + pub fn xml_needs_escape(StringView, attr~ : Bool) -> Bool // Errors @@ -38,6 +42,16 @@ pub fn WorkbookManifest::root_rels_xml(Self) -> String pub fn WorkbookManifest::set_workbook_content_type(Self, String) -> Unit pub fn WorkbookManifest::workbook_rels_xml(Self) -> String +pub struct XmlStartTagScanner { + // private fields +} +pub fn XmlStartTagScanner::attribute(Self, StringView, StringView) -> String? raise ParseXmlError +pub fn XmlStartTagScanner::attribute_view(Self, StringView, StringView) -> StringView? raise ParseXmlError +pub fn XmlStartTagScanner::local_name(Self) -> StringView +pub fn XmlStartTagScanner::namespace_uri(Self) -> StringView +pub fn XmlStartTagScanner::new(StringView) -> Self +pub fn XmlStartTagScanner::next(Self) -> Bool raise ParseXmlError + // Type aliases // Traits diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 72177a49..91aa50b2 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -79,93 +79,114 @@ pub fn attr_value( } ///| -fn first_tag_end(text : StringView) -> Int? { - let mut i = 0 - let mut quote : UInt16? = None - while i < text.length() { - let ch = text[i] - match quote { - Some(value) => if ch == value { quote = None } - None => - if ch == ('"' : UInt16) || ch == ('\'' : UInt16) { - quote = Some(ch) - } else { - if ch == ('<' : UInt16) { - return None - } - if ch == ('>' : UInt16) { - return Some(i) - } - } - } - i = i + 1 - } - None -} +let package_relationships_namespace = "http://schemas.openxmlformats.org/package/2006/relationships" ///| -pub fn parse_relationship_targets( +let package_content_types_namespace = "http://schemas.openxmlformats.org/package/2006/content-types" + +///| +/// Reads relationship targets whose decoded `Type` is in `rel_types`. Element +/// names are matched by expanded namespace name, so the package namespace may +/// use either a default namespace or any declared prefix. +pub fn parse_relationship_targets_by_types( xml : StringView, - rel_type : StringView, + rel_types : ArrayView[String], ) -> Map[String, String] raise ParseXmlError { + let max_records = 16_384 + let max_id_chars = 256 + let max_type_chars = 2_048 + let max_target_chars = 4_096 + let max_retained_chars = 8 * 1024 * 1024 let targets : Map[String, String] = Map([]) - let mut first = true - for chunk in xml.split(" pos - None => raise InvalidXml(msg="relationship tag not closed") + records = records + 1 + if records > max_records { + raise InvalidXml(msg="relationship record limit exceeded") } - let tag = text.unsafe_substring(start=0, end~) - let rel = match attr_value(tag, "Type") { + let rel_view = match scanner.attribute_view("", "Type") { Some(value) => value None => continue } - if rel != rel_type.to_owned() { + if rel_view.length() > max_type_chars { + raise InvalidXml(msg="relationship type is too long") + } + let rel = decode_xml_attribute(rel_view) + let mut wanted = false + for rel_type in rel_types { + if rel == rel_type { + wanted = true + break + } + } + if !wanted { continue } - let id = match attr_value(tag, "Id") { + let id_view = match scanner.attribute_view("", "Id") { Some(value) => value None => raise InvalidXml(msg="relationship id missing") } - let target = match attr_value(tag, "Target") { + if id_view.length() > max_id_chars { + raise InvalidXml(msg="relationship id is too long") + } + let target_view = match scanner.attribute_view("", "Target") { Some(value) => value None => raise InvalidXml(msg="relationship target missing") } + if target_view.length() > max_target_chars { + raise InvalidXml(msg="relationship target is too long") + } + let id = decode_xml_attribute(id_view) + let target = decode_xml_attribute(target_view) + let retained = id.length() + target.length() + if retained > max_retained_chars - retained_chars { + raise InvalidXml(msg="relationship retained text limit exceeded") + } + retained_chars = retained_chars + retained + if targets.contains(id) { + raise InvalidXml(msg="duplicate relationship id") + } targets[id] = target } targets } +///| +pub fn parse_relationship_targets( + xml : StringView, + rel_type : StringView, +) -> Map[String, String] raise ParseXmlError { + parse_relationship_targets_by_types(xml, [rel_type.to_owned()]) +} + ///| fn parse_content_type_overrides( xml : StringView, ) -> Map[String, String] raise ParseXmlError { let overrides : Map[String, String] = Map([]) - let mut first = true - for chunk in xml.split(" pos - None => raise InvalidXml(msg="content type override tag not closed") - } - let tag = text.unsafe_substring(start=0, end~) - let part_name = match attr_value(tag, "PartName") { + let part_name = match scanner.attribute("", "PartName") { Some(value) => value None => raise InvalidXml(msg="content type override part name missing") } - let content_type = match attr_value(tag, "ContentType") { + let content_type = match scanner.attribute("", "ContentType") { Some(value) => value None => raise InvalidXml(msg="content type override content type missing") } + if overrides.contains(part_name) { + raise InvalidXml(msg="duplicate content type override") + } overrides[part_name] = content_type } overrides diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index f568b7b4..fcdca493 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -50,12 +50,27 @@ test "ooxml read_parse: parse_relationship_targets reports missing id/target and #| #| try @ooxml.parse_relationship_targets(malformed, "t1") catch { - InvalidXml(msg~) => inspect(msg, content="relationship tag not closed") + InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") } noraise { _ => fail("expected relationship tag not closed") } } +///| +test "ooxml read_parse: prefixed relationships decode entities exactly once" { + let xml = + #| + #| + #| + let targets = @ooxml.parse_relationship_targets(xml, "urn:type1") + debug_inspect( + targets, + content=( + #|{ "r&1": "custom/a b.xml" } + ), + ) +} + ///| test "ooxml read_parse: parse_content_type_overrides and find workbook part" { let xml = @@ -89,9 +104,22 @@ test "ooxml read_parse: parse_content_type_overrides reports malformed override "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", ]) catch { - InvalidXml(msg~) => - inspect(msg, content="content type override tag not closed") + InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") } noraise { _ => fail("expected content type override tag not closed") } } + +///| +test "ooxml read_parse: prefixed content types decode names and media types" { + let xml = + #| + #| + #| + debug_inspect( + @ooxml.find_override_part_by_content_types(xml, [ + "application/vnd.example+xml", + ]), + content="Some(\"/xl/a&b.xml\")", + ) +} diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt new file mode 100644 index 00000000..d70e3d8d --- /dev/null +++ b/ooxml/start_tag_scanner.mbt @@ -0,0 +1,821 @@ +///| +let xml_namespace_uri = "http://www.w3.org/XML/1998/namespace" + +///| +let xmlns_namespace_uri = "http://www.w3.org/2000/xmlns/" + +///| +let max_xml_start_tag_chars : Int = 1024 * 1024 + +///| +let max_xml_qname_chars : Int = 1024 + +///| +let max_xml_start_tag_attributes : Int = 4096 + +///| +let max_xml_namespace_bindings : Int = 4096 + +///| +let max_xml_element_depth : Int = 256 + +///| +priv struct XmlNamespaceBinding { + prefix : String + uri : String + depth : Int +} + +///| +priv struct XmlAttributeSpan { + name_start : Int + name_end : Int + value_start : Int + value_end : Int + next : Int +} + +///| +/// A forward-only, allocation-bounded XML start-tag cursor. Element and +/// attribute names are exposed as expanded namespace names while source slices +/// remain borrowed from the caller's XML string. +pub struct XmlStartTagScanner { + priv xml : StringView + priv mut index : Int + priv mut depth : Int + priv namespace_bindings : Array[XmlNamespaceBinding] + priv open_name_starts : Array[Int] + priv open_name_ends : Array[Int] + priv mut pending_namespace_pop : Int? + priv mut current_name_start : Int + priv mut current_name_end : Int + priv mut current_tag_end : Int + priv mut current_namespace_uri : String +} + +///| +fn is_xml_name_start(unit : UInt16) -> Bool { + (unit >= ('A' : UInt16) && unit <= ('Z' : UInt16)) || + (unit >= ('a' : UInt16) && unit <= ('z' : UInt16)) || + unit == ('_' : UInt16) || + unit >= 0x80 +} + +///| +fn is_xml_name_unit(unit : UInt16) -> Bool { + is_xml_name_start(unit) || + (unit >= ('0' : UInt16) && unit <= ('9' : UInt16)) || + unit == ('-' : UInt16) || + unit == ('.' : UInt16) +} + +///| +fn validate_xml_qname( + xml : StringView, + start : Int, + end : Int, +) -> Int? raise ParseXmlError { + if start >= end { + raise InvalidXml(msg="XML qualified name is empty") + } + if end - start > max_xml_qname_chars { + raise InvalidXml(msg="XML qualified name is too long") + } + let mut colon : Int? = None + for index in start.. index == value + 1 + None => false + } + } + if local_start { + if !is_xml_name_start(unit) { + raise InvalidXml(msg="XML qualified name is invalid") + } + } else if !is_xml_name_unit(unit) { + raise InvalidXml(msg="XML qualified name is invalid") + } + } + } + colon +} + +///| +fn xml_string_equals_view(value : String, expected : StringView) -> Bool { + if value.length() != expected.length() { + return false + } + for index in 0.. Bool { + if end - start != expected.length() { + return false + } + for offset in 0.. Bool { + if first_end - first_start != second_end - second_start { + return false + } + for offset in 0..<(first_end - first_start) { + if xml[first_start + offset] != xml[second_start + offset] { + return false + } + } + true +} + +///| +fn find_xml_sequence( + xml : StringView, + start : Int, + sequence : StringView, +) -> Int? { + if sequence.length() == 0 { + return Some(start) + } + let last = xml.length() - sequence.length() + let mut index = start + while index <= last { + if xml_span_equals(xml, index, index + sequence.length(), sequence) { + return Some(index) + } + index = index + 1 + } + None +} + +///| +fn find_xml_tag_end( + xml : StringView, + search_start : Int, + markup_start : Int, +) -> Int raise ParseXmlError { + let mut index = search_start + let mut quote : UInt16? = None + while index < xml.length() { + if index - markup_start > max_xml_start_tag_chars { + raise InvalidXml(msg="XML start tag is too long") + } + let unit = xml[index] + match quote { + Some(delimiter) => { + if unit == ('<' : UInt16) { + raise InvalidXml(msg="XML attribute contains '<'") + } + if unit == delimiter { + quote = None + } + } + None => + if unit == ('"' : UInt16) || unit == ('\'' : UInt16) { + quote = Some(unit) + } else if unit == ('>' : UInt16) { + return index + } else if unit == ('<' : UInt16) { + raise InvalidXml(msg="XML start tag is not closed") + } + } + index = index + 1 + } + raise InvalidXml(msg="XML start tag is not closed") +} + +///| +fn next_xml_attribute( + xml : StringView, + start : Int, + tag_end : Int, +) -> XmlAttributeSpan? raise ParseXmlError { + let mut index = start + while index < tag_end && is_attr_space(xml[index]) { + index = index + 1 + } + if index >= tag_end || xml[index] == ('/' : UInt16) { + return None + } + let name_start = index + while index < tag_end && + !is_attr_space(xml[index]) && + xml[index] != ('=' : UInt16) && + xml[index] != ('/' : UInt16) { + index = index + 1 + } + let name_end = index + ignore(validate_xml_qname(xml, name_start, name_end)) + while index < tag_end && is_attr_space(xml[index]) { + index = index + 1 + } + if index >= tag_end || xml[index] != ('=' : UInt16) { + raise InvalidXml(msg="XML attribute has no value") + } + index = index + 1 + while index < tag_end && is_attr_space(xml[index]) { + index = index + 1 + } + if index >= tag_end || + (xml[index] != ('"' : UInt16) && xml[index] != ('\'' : UInt16)) { + raise InvalidXml(msg="XML attribute value is not quoted") + } + let quote = xml[index] + index = index + 1 + let value_start = index + while index < tag_end && xml[index] != quote { + if xml[index] == ('<' : UInt16) { + raise InvalidXml(msg="XML attribute contains '<'") + } + index = index + 1 + } + if index >= tag_end { + raise InvalidXml(msg="XML attribute is not closed") + } + let value_end = index + index = index + 1 + Some({ name_start, name_end, value_start, value_end, next: index }) +} + +///| +fn is_valid_xml_char_reference(value : Int) -> Bool { + value == 0x09 || + value == 0x0a || + value == 0x0d || + (value >= 0x20 && value <= 0xd7ff) || + (value >= 0xe000 && value <= 0xfffd) || + (value >= 0x10000 && value <= 0x10ffff) +} + +///| +fn xml_entity_value(entity : StringView) -> Char? { + match entity { + "lt" => Some('<') + "gt" => Some('>') + "quot" => Some('"') + "apos" => Some('\'') + "amp" => Some('&') + _ => { + let (digits, base) = if entity.length() >= 2 && + entity[0] == ('#' : UInt16) && + (entity[1] == ('x' : UInt16) || entity[1] == ('X' : UInt16)) { + (entity[2:], 16) + } else if entity.length() >= 1 && entity[0] == ('#' : UInt16) { + (entity[1:], 10) + } else { + return None + } + if digits.length() == 0 { + return None + } + for unit in digits { + let valid = if base == 10 { + unit >= '0' && unit <= '9' + } else { + (unit >= '0' && unit <= '9') || + (unit >= 'a' && unit <= 'f') || + (unit >= 'A' && unit <= 'F') + } + if !valid { + return None + } + } + let value = @string.parse_int(digits, base~) catch { _ => return None } + if !is_valid_xml_char_reference(value) { + return None + } + Some(value.unsafe_to_char()) + } + } +} + +///| +/// Decodes one XML attribute value in a single pass. Unknown, malformed, or +/// XML-forbidden entity references are rejected instead of being decoded a +/// second time by a downstream path resolver. +pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { + let mut needs_decode = false + for unit in value { + if unit == '&' || unit == '\t' || unit == '\n' || unit == '\r' { + needs_decode = true + break + } + if unit == '<' { + raise InvalidXml(msg="XML attribute contains '<'") + } + } + if !needs_decode { + return value.to_owned() + } + let output = StringBuilder::new() + let mut index = 0 + let mut literal_start = 0 + while index < value.length() { + let unit = value[index] + if unit == ('&' : UInt16) { + output.write_view(value[literal_start:index]) + let entity_end = match find_xml_sequence(value, index + 1, ";") { + Some(found) => found + None => raise InvalidXml(msg="XML entity is not closed") + } + let entity = value[index + 1:entity_end] + let decoded = match xml_entity_value(entity) { + Some(character) => character + None => raise InvalidXml(msg="XML entity is invalid") + } + output.write_char(decoded) + index = entity_end + 1 + literal_start = index + continue + } + if unit == ('\t' : UInt16) || + unit == ('\n' : UInt16) || + unit == ('\r' : UInt16) { + output.write_view(value[literal_start:index]) + output.write_char(' ') + index = index + 1 + literal_start = index + continue + } + if unit == ('<' : UInt16) { + raise InvalidXml(msg="XML attribute contains '<'") + } + index = index + 1 + } + output.write_view(value[literal_start:]) + output.to_string() +} + +///| +fn XmlStartTagScanner::pop_namespace_depth( + self : XmlStartTagScanner, + depth : Int, +) -> Unit { + while self.namespace_bindings.length() > 0 { + let last = self.namespace_bindings.length() - 1 + if self.namespace_bindings[last].depth != depth { + break + } + ignore(self.namespace_bindings.pop()) + } +} + +///| +fn XmlStartTagScanner::resolve_prefix( + self : XmlStartTagScanner, + prefix : StringView, + attribute : Bool, +) -> String raise ParseXmlError { + if prefix == "xml" { + return xml_namespace_uri + } + if prefix.length() == 0 && attribute { + return "" + } + for offset in 0.. String? raise ParseXmlError { + if xml_span_equals(xml, attribute.name_start, attribute.name_end, "xmlns") { + return Some("") + } + let colon = match + validate_xml_qname(xml, attribute.name_start, attribute.name_end) { + Some(value) => value + None => return None + } + if !xml_span_equals(xml, attribute.name_start, colon, "xmlns") { + return None + } + Some(xml[colon + 1:attribute.name_end].to_owned()) +} + +///| +fn XmlStartTagScanner::add_namespace_declarations( + self : XmlStartTagScanner, + attributes_start : Int, + tag_end : Int, + scope_depth : Int, +) -> Unit raise ParseXmlError { + let declared : Set[String] = Set([]) + let mut next = attributes_start + let mut count = 0 + while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { + count = count + 1 + if count > max_xml_start_tag_attributes { + raise InvalidXml(msg="XML start tag has too many attributes") + } + next = attribute.next + guard xml_namespace_declaration_prefix(self.xml, attribute) is Some(prefix) else { + continue + } + if declared.contains(prefix) { + raise InvalidXml(msg="XML namespace prefix is declared twice") + } + declared.add(prefix) + let uri = decode_xml_attribute( + self.xml[attribute.value_start:attribute.value_end], + ) + if prefix == "xmlns" || uri == xmlns_namespace_uri { + raise InvalidXml(msg="XML namespace declaration is reserved") + } + if prefix == "xml" && uri != xml_namespace_uri { + raise InvalidXml(msg="XML namespace declaration for xml is invalid") + } + if prefix != "xml" && uri == xml_namespace_uri { + raise InvalidXml(msg="XML namespace declaration for xml is invalid") + } + if prefix != "" && uri == "" { + raise InvalidXml(msg="XML prefixed namespace cannot be empty") + } + if self.namespace_bindings.length() >= max_xml_namespace_bindings { + raise InvalidXml(msg="XML has too many active namespace bindings") + } + self.namespace_bindings.push({ prefix, uri, depth: scope_depth }) + } +} + +///| +fn XmlStartTagScanner::validate_attributes( + self : XmlStartTagScanner, + attributes_start : Int, + tag_end : Int, +) -> Unit raise ParseXmlError { + let expanded_names : Set[String] = Set([]) + let mut next = attributes_start + let mut count = 0 + while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { + count = count + 1 + if count > max_xml_start_tag_attributes { + raise InvalidXml(msg="XML start tag has too many attributes") + } + next = attribute.next + if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { + continue + } + let colon = validate_xml_qname( + self.xml, + attribute.name_start, + attribute.name_end, + ) + let (prefix, local_name_view) = match colon { + Some(value) => + ( + self.xml[attribute.name_start:value], + self.xml[value + 1:attribute.name_end], + ) + None => + ( + self.xml[attribute.name_start:attribute.name_start], + self.xml[attribute.name_start:attribute.name_end], + ) + } + let uri = self.resolve_prefix(prefix, true) + let expanded = "\{uri.length()}:\{uri}\{local_name_view.to_owned()}" + if expanded_names.contains(expanded) { + raise InvalidXml(msg="XML attribute is duplicated") + } + expanded_names.add(expanded) + } +} + +///| +/// Creates a namespace-aware cursor over `xml`. The source is borrowed; no +/// copy of the full XML part is made. +pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { + { + xml, + index: 0, + depth: 0, + namespace_bindings: [], + open_name_starts: [], + open_name_ends: [], + pending_namespace_pop: None, + current_name_start: 0, + current_name_end: 0, + current_tag_end: 0, + current_namespace_uri: "", + } +} + +///| +fn XmlStartTagScanner::consume_end_tag( + self : XmlStartTagScanner, + start : Int, +) -> Unit raise ParseXmlError { + if self.depth <= 0 || self.open_name_starts.length() == 0 { + raise InvalidXml(msg="XML end tag has no matching start tag") + } + let name_start = start + 2 + let mut name_end = name_start + while name_end < self.xml.length() && + !is_attr_space(self.xml[name_end]) && + self.xml[name_end] != ('>' : UInt16) { + name_end = name_end + 1 + } + ignore(validate_xml_qname(self.xml, name_start, name_end)) + let tag_end = find_xml_tag_end(self.xml, name_end, start) + for index in name_end.. Bool raise ParseXmlError { + match self.pending_namespace_pop { + Some(depth) => { + self.pop_namespace_depth(depth) + self.pending_namespace_pop = None + } + None => () + } + while self.index < self.xml.length() { + while self.index < self.xml.length() && + self.xml[self.index] != ('<' : UInt16) { + self.index = self.index + 1 + } + if self.index >= self.xml.length() { + break + } + let start = self.index + if start + 1 >= self.xml.length() { + raise InvalidXml(msg="XML markup is not closed") + } + if self.xml[start + 1] == ('?' : UInt16) { + let end = match find_xml_sequence(self.xml, start + 2, "?>") { + Some(value) => value + None => raise InvalidXml(msg="XML processing instruction is not closed") + } + self.index = end + 2 + continue + } + if self.xml[start + 1] == ('!' : UInt16) { + if start + 4 <= self.xml.length() && + xml_span_equals(self.xml, start, start + 4, "") { + Some(value) => value + None => raise InvalidXml(msg="XML comment is not closed") + } + self.index = end + 3 + continue + } + if start + 9 <= self.xml.length() && + xml_span_equals(self.xml, start, start + 9, "") { + Some(value) => value + None => raise InvalidXml(msg="XML CDATA section is not closed") + } + self.index = end + 3 + continue + } + raise InvalidXml(msg="XML declarations are not allowed") + } + if self.xml[start + 1] == ('/' : UInt16) { + self.consume_end_tag(start) + continue + } + let name_start = start + 1 + let mut name_end = name_start + while name_end < self.xml.length() && + !is_attr_space(self.xml[name_end]) && + self.xml[name_end] != ('/' : UInt16) && + self.xml[name_end] != ('>' : UInt16) { + name_end = name_end + 1 + } + let colon = validate_xml_qname(self.xml, name_start, name_end) + let tag_end = find_xml_tag_end(self.xml, name_end, start) + let mut before_end = tag_end + while before_end > name_end && is_attr_space(self.xml[before_end - 1]) { + before_end = before_end - 1 + } + let self_closing = before_end > name_end && + self.xml[before_end - 1] == ('/' : UInt16) + let scope_depth = self.depth + 1 + if scope_depth > max_xml_element_depth { + raise InvalidXml(msg="XML element nesting is too deep") + } + self.add_namespace_declarations(name_end, tag_end, scope_depth) + self.validate_attributes(name_end, tag_end) + let prefix = match colon { + Some(value) => self.xml[name_start:value] + None => self.xml[name_start:name_start] + } + self.current_namespace_uri = self.resolve_prefix(prefix, false) + self.current_name_start = match colon { + Some(value) => value + 1 + None => name_start + } + self.current_name_end = name_end + self.current_tag_end = tag_end + self.index = tag_end + 1 + if self_closing { + self.pending_namespace_pop = Some(scope_depth) + } else { + self.depth = scope_depth + self.open_name_starts.push(name_start) + self.open_name_ends.push(name_end) + } + return true + } + if self.depth != 0 { + raise InvalidXml(msg="XML start tag has no matching end tag") + } + false +} + +///| +/// Returns the local name of the current start tag. +pub fn XmlStartTagScanner::local_name(self : XmlStartTagScanner) -> StringView { + self.xml[self.current_name_start:self.current_name_end] +} + +///| +/// Returns the namespace URI of the current start tag, or `""` when the tag +/// is not namespace-qualified. +pub fn XmlStartTagScanner::namespace_uri( + self : XmlStartTagScanner, +) -> StringView { + self.current_namespace_uri +} + +///| +/// Returns the raw, borrowed value of an attribute selected by expanded name. +/// The default namespace never applies to unprefixed attributes. +pub fn XmlStartTagScanner::attribute_view( + self : XmlStartTagScanner, + namespace_uri : StringView, + local_name : StringView, +) -> StringView? raise ParseXmlError { + let mut next = self.current_name_end + while next_xml_attribute(self.xml, next, self.current_tag_end) + is Some(attribute) { + next = attribute.next + if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { + continue + } + let colon = validate_xml_qname( + self.xml, + attribute.name_start, + attribute.name_end, + ) + let (prefix, local_start) = match colon { + Some(value) => (self.xml[attribute.name_start:value], value + 1) + None => + ( + self.xml[attribute.name_start:attribute.name_start], + attribute.name_start, + ) + } + if !xml_span_equals(self.xml, local_start, attribute.name_end, local_name) { + continue + } + if xml_string_equals_view(self.resolve_prefix(prefix, true), namespace_uri) { + return Some(self.xml[attribute.value_start:attribute.value_end]) + } + } + None +} + +///| +/// Returns a decoded attribute selected by expanded name. Entity references +/// are decoded exactly once. +pub fn XmlStartTagScanner::attribute( + self : XmlStartTagScanner, + namespace_uri : StringView, + local_name : StringView, +) -> String? raise ParseXmlError { + match self.attribute_view(namespace_uri, local_name) { + Some(value) => Some(decode_xml_attribute(value)) + None => None + } +} + +///| +test "start tag scanner resolves namespaces and decodes attributes once" { + let xml = + #| + let scanner = XmlStartTagScanner::new(xml) + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:root") + inspect(scanner.local_name(), content="root") + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:root") + inspect(scanner.local_name(), content="item") + debug_inspect(scanner.attribute("urn:rel", "id"), content="Some(\"a&b\")") + debug_inspect(scanner.attribute("", "plain"), content="Some(\"x y\")") + assert_false(scanner.next()) +} + +///| +test "start tag scanner rejects undeclared and duplicate expanded attributes" { + for + xml in [ + "", "", + ] { + let scanner = XmlStartTagScanner::new(xml) + try scanner.next() catch { + InvalidXml(_) => () + } noraise { + _ => fail("expected invalid namespace-qualified XML") + } + } +} + +///| +test "start tag scanner restores namespace scopes after self-closing tags" { + let scanner = XmlStartTagScanner::new( + "", + ) + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:outer") + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:inner") + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:outer") + assert_false(scanner.next()) +} + +///| +test "XML attributes decode once and reject invalid entities" { + inspect(decode_xml_attribute("a&#x20;b"), content="a b") + for value in ["&unknown;", "�", "�", "&"] { + try decode_xml_attribute(value) catch { + InvalidXml(_) => () + } noraise { + _ => fail("expected invalid XML entity") + } + } +} + +///| +test "start tag scanner bounds qualified names" { + let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") + try scanner.next() catch { + InvalidXml(msg~) => inspect(msg, content="XML qualified name is too long") + } noraise { + _ => fail("expected qualified-name limit") + } +} diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 1a7dc710..a6d9c625 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -14,6 +14,16 @@ fn parse_relationship_targets_exact( } } +///| +fn parse_relationship_targets_exact_types( + xml : StringView, + rel_types : ArrayView[String], +) -> Map[String, String] raise XlsxError { + @ooxml.parse_relationship_targets_by_types(xml, rel_types) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + ///| /// Reads both ISO Strict and ECMA Transitional aliases for relationship types /// from the Office relationship family. Microsoft extension relationships use @@ -23,20 +33,13 @@ fn parse_relationship_targets( rel_type : StringView, ) -> Map[String, String] raise XlsxError { let requested = rel_type.to_owned() - let targets = parse_relationship_targets_exact(xml, requested) - guard requested.strip_prefix(transitional_office_relationship_prefix) - is Some(suffix) else { - return targets - } - let strict_type = strict_office_relationship_prefix + suffix.to_owned() - let strict_targets = parse_relationship_targets_exact(xml, strict_type) - for id, target in strict_targets { - if targets.contains(id) { - raise InvalidXml(msg="relationship id has conflicting dialect aliases") + match requested.strip_prefix(transitional_office_relationship_prefix) { + Some(suffix) => { + let strict_type = strict_office_relationship_prefix + suffix.to_owned() + parse_relationship_targets_exact_types(xml, [requested, strict_type]) } - targets[id] = target + None => parse_relationship_targets_exact(xml, requested) } - targets } ///| diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index 448eeb82..0ab0eeab 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -77,7 +77,7 @@ test "ooxml rels: relationship tag not closed" { ) let result = Ok(@xlsx.read(updated)) catch { e => Err(e) } match result { - Err(InvalidXml(msg~)) => inspect(msg, content="relationship tag not closed") + Err(InvalidXml(msg~)) => inspect(msg, content="XML start tag is not closed") _ => fail("expected InvalidXml for relationship tag not closed") } } diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt new file mode 100644 index 00000000..7a236579 --- /dev/null +++ b/xlsx/read_namespace_test.mbt @@ -0,0 +1,101 @@ +///| +fn namespace_qualified_xlsx_fixture( + strict : Bool, + entity_paths : Bool, +) -> Bytes raise { + let spreadsheet_namespace = if strict { + "http://purl.oclc.org/ooxml/spreadsheetml/main" + } else { + "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + } + let office_relationship_prefix = if strict { + "http://purl.oclc.org/ooxml/officeDocument/relationships/" + } else { + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/" + } + let office_relationship_namespace = if strict { + "http://purl.oclc.org/ooxml/officeDocument/relationships" + } else { + "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + } + let workbook_path = if entity_paths { + "custom/a&b.xml" + } else { + "xl/workbook.xml" + } + let workbook_part_name = if entity_paths { + "/custom/a&b.xml" + } else { + "/xl/workbook.xml" + } + let workbook_rels_path = if entity_paths { + "custom/_rels/a&b.xml.rels" + } else { + "xl/_rels/workbook.xml.rels" + } + let worksheet_path = if entity_paths { + "data/a b.xml" + } else { + "xl/worksheets/sheet.xml" + } + let worksheet_part_name = if entity_paths { + "/data/a b.xml" + } else { + "/xl/worksheets/sheet.xml" + } + let worksheet_target = if entity_paths { + "../data/a b.xml" + } else { + "worksheets/sheet.xml" + } + let relationship_id = if entity_paths { "sheet&1" } else { "sheet" } + let root_target = if entity_paths { + "custom/a&b.xml" + } else { + "xl/workbook.xml" + } + let content_types = "\n" + + " \n" + + " \n" + + " \n" + + "" + let root_relationships = "\n" + + " \n" + + "" + let workbook = "\n" + + " \n" + + "" + let workbook_relationships = "\n" + + " \n" + + "" + let worksheet = "qualified-value" + let archive = @zip.Archive::new() + archive.add("[Content_Types].xml", @encoding/utf8.encode(content_types)) + archive.add("_rels/.rels", @encoding/utf8.encode(root_relationships)) + archive.add(workbook_path, @encoding/utf8.encode(workbook)) + archive.add(workbook_rels_path, @encoding/utf8.encode(workbook_relationships)) + archive.add(worksheet_path, @encoding/utf8.encode(worksheet)) + @zip.write(archive) +} + +///| +test "namespace-qualified Transitional and Strict workbook parts read end to end" { + for strict in [false, true] { + let workbook = @xlsx.read(namespace_qualified_xlsx_fixture(strict, false)) + debug_inspect(workbook.get_sheet_list(), content="[\"Qualified\"]") + debug_inspect( + workbook.get_cell_value("Qualified", "A1"), + content="Some(\"qualified-value\")", + ) + } +} + +///| +test "main-part names, relationship targets, and ids decode XML entities" { + let workbook = @xlsx.read(namespace_qualified_xlsx_fixture(false, true)) + debug_inspect(workbook.get_sheet_list(), content="[\"Qualified\"]") + debug_inspect( + workbook.get_cell_value("Qualified", "A1"), + content="Some(\"qualified-value\")", + ) +} diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index c623ebd9..5387d159 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -25,7 +25,7 @@ fn workbook_part_from_root_relationships( } match first_relationship_target(targets) { Some(target) => { - let part_name = archive_path_from_part_name(unescape_xml_text(target)) + let part_name = archive_path_from_part_name(target) Some(normalize_rel_part_path(part_name)) } None => None @@ -138,8 +138,7 @@ test "read package parts: malformed [Content_Types] raises InvalidXml" { e => Err(e) } match result { - Err(InvalidXml(msg~)) => - inspect(msg, content="content type override tag not closed") + Err(InvalidXml(msg~)) => inspect(msg, content="XML start tag is not closed") _ => fail("expected content type override tag not closed") } } diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index 84c006d3..134cc425 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -5,33 +5,77 @@ priv struct WorkbookSheetInfo { rel_id : String } +///| +let transitional_spreadsheet_namespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + +///| +let strict_spreadsheet_namespace = "http://purl.oclc.org/ooxml/spreadsheetml/main" + +///| +let transitional_relationship_attribute_namespace = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + +///| +let strict_relationship_attribute_namespace = "http://purl.oclc.org/ooxml/officeDocument/relationships" + +///| +fn workbook_scanner_next( + scanner : @ooxml.XmlStartTagScanner, +) -> Bool raise XlsxError { + scanner.next() catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + +///| +fn workbook_scanner_attribute( + scanner : @ooxml.XmlStartTagScanner, + namespace_uri : StringView, + local_name : StringView, +) -> String? raise XlsxError { + scanner.attribute(namespace_uri, local_name) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + ///| fn parse_workbook_sheets( xml : StringView, max_sheets : Int, ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] - let mut first = true - for chunk in xml.split("") { - Some(pos) => pos - None => raise InvalidXml(msg="sheet tag not closed") + if sheets.length() >= max_sheets { + raise ResourceLimitExceeded( + kind="workbook_sheets", + limit=max_sheets, + actual=max_sheets + 1, + ) } - let tag = text[:end] - let name = match attr_value(tag, "name") { - Some(value) => unescape_xml_text(value) + let name = match workbook_scanner_attribute(scanner, "", "name") { + Some(value) => value None => raise InvalidXml(msg="sheet name missing") } - let rel_id = match attr_value(tag, "r:id") { + let relationship_namespace = if namespace_uri == + strict_spreadsheet_namespace { + strict_relationship_attribute_namespace + } else { + transitional_relationship_attribute_namespace + } + let rel_id = match + workbook_scanner_attribute(scanner, relationship_namespace, "id") { Some(value) => value None => raise InvalidXml(msg="sheet relationship missing") } - let state = match attr_value(tag, "state") { + let state = match workbook_scanner_attribute(scanner, "", "state") { Some(value) => match value.to_lower() { "hidden" => Hidden @@ -40,13 +84,6 @@ fn parse_workbook_sheets( } None => Visible } - if sheets.length() >= max_sheets { - raise ResourceLimitExceeded( - kind="workbook_sheets", - limit=max_sheets, - actual=max_sheets + 1, - ) - } sheets.push({ name, state, rel_id }) } sheets From 4379012e8672f9d9a1dc219f6bc73b03c71e3797 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 17 Jul 2026 22:40:22 +0800 Subject: [PATCH 019/150] fix(xlsx): reject malformed shared formulas --- office/cmd/office/xlsx_read_model.mbt | 50 ++++++++++------- office/cmd/office/xlsx_read_wbtest.mbt | 29 ++++++++++ xlsx/read_worksheet_xml.mbt | 3 ++ xlsx/shared_formula_translate.mbt | 17 ++++-- xlsx/shared_formula_validation_test.mbt | 71 +++++++++++++++++++++++++ xlsx/worksheet.mbt | 47 +++++++++++++--- 6 files changed, 188 insertions(+), 29 deletions(-) create mode 100644 xlsx/shared_formula_validation_test.mbt diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 794aa968..3abbbeed 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -399,7 +399,7 @@ fn XlsxSheetTarget::shared_formula_master( projection : XlsxProjection, worksheet : @xlsx.Worksheet, shared_index : Int, -) -> @xlsx.SharedFormulaMaster? raise CliFailure { +) -> @xlsx.SharedFormulaMaster raise CliFailure { if !self.shared_formulas_indexed { let masters = xlsx_call(projection.file, () => { worksheet.shared_formula_masters() @@ -409,7 +409,18 @@ fn XlsxSheetTarget::shared_formula_master( } self.shared_formulas_indexed = true } - self.shared_formula_masters.get(shared_index) + match self.shared_formula_masters.get(shared_index) { + Some(master) => master + None => + raise xlsx_cli_failure( + "office.invalid_package", + "invalid XLSX package: shared formula follower has no master", + details=Json::object({ + "file": Json::string(bounded_text(projection.file, 160)), + "sheet": Json::string(bounded_text(self.name, 160)), + }), + ) + } } ///| @@ -640,22 +651,25 @@ fn xlsx_cell_formula( (Some(info.formula), info.cached_value_present) Some(info) => ( - match info.shared_index { - Some(shared_index) => - // `Some("")` is intentional when a malformed package has a shared - // follower but no resolvable master: preserve formula presence even - // when no formula text can be supplied. - match - sheet.shared_formula_master(projection, worksheet, shared_index) { - Some(master) => - Some( - xlsx_call(projection.file, () => { - master.translate_to(row, column) - }), - ) - None => Some("") - } - None => Some("") + match (info.formula_type, info.shared_index) { + (Some(Shared), Some(shared_index)) => { + let master = sheet.shared_formula_master( + projection, worksheet, shared_index, + ) + Some( + xlsx_call(projection.file, () => master.translate_to(row, column)), + ) + } + (Some(Shared), None) => + raise xlsx_cli_failure( + "office.invalid_package", + "invalid XLSX package: shared formula index missing", + details=Json::object({ + "file": Json::string(bounded_text(projection.file, 160)), + "sheet": Json::string(bounded_text(sheet.name, 160)), + }), + ) + _ => Some("") }, info.cached_value_present, ) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 0ab7c559..b98351c6 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -483,6 +483,35 @@ test "XLSX snapshots preserve and resolve shared-formula followers" { assert_eq(master_fields.get("matched_total"), Some(Json::number(1))) } +///| +test "XLSX structured reads reject shared-formula followers without a master" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("A1:C1"), + ) + let archive = @zip.read(@xlsx.write(workbook)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => @utf8.decode(value, ignore_bom=true) + None => fail("missing shared-formula worksheet") + } + let master = "D1+1" + let follower = "" + let malformed = xml.replace(old=master, new=follower) + assert_true(malformed != xml) + assert_true(archive.replace(path, @utf8.encode(malformed))) + let source = xlsx_bounded_test_source( + "missing-shared-master.xlsx", + @zip.write(archive), + ) + let error = xlsx_cli_error(() => ignore(open_xlsx_projection(source, 10))) + assert_eq(error.code, "office.invalid_package") + assert_true(error.message.contains("shared formula follower has no master")) +} + ///| test "XLSX formatting is deferred and custom-format work is charged" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index 703f5c47..53b61d89 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -236,6 +236,9 @@ fn parse_worksheet( style_id, }) } + // Shared formulas are a worksheet-wide construct: followers carry only an + // index, so validate the complete group before exposing a partial model. + ignore(validated_shared_formula_masters(cells)) let merged_cells = parse_merge_cells(xml) let auto_filter = parse_auto_filter(xml) let row_dimensions = parse_row_dimensions(xml, style_count) diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index e69979e2..bf684fa3 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -584,6 +584,18 @@ fn translate_shared_formula( out.to_string() } +///| +fn SharedFormulaMaster::contains_coordinate( + self : SharedFormulaMaster, + row : Int, + column : Int, +) -> Bool { + row >= self.row_lo && + row <= self.row_hi && + column >= self.column_lo && + column <= self.column_hi +} + ///| /// Resolves the formula text represented by this shared master at a follower /// coordinate. Coordinates outside the master's declared shared range are @@ -594,10 +606,7 @@ pub fn SharedFormulaMaster::translate_to( column : Int, ) -> String raise XlsxError { ignore(cell_ref_from(row, column)) - if row < self.row_lo || - row > self.row_hi || - column < self.column_lo || - column > self.column_hi { + if !self.contains_coordinate(row, column) { raise InvalidXml(msg="shared formula follower is outside the master range") } translate_shared_formula(self.formula, column - self.column, row - self.row) diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt new file mode 100644 index 00000000..08ee8523 --- /dev/null +++ b/xlsx/shared_formula_validation_test.mbt @@ -0,0 +1,71 @@ +///| +fn malformed_shared_formula_package( + old : String, + replacement : String, +) -> Bytes raise { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("A1:C1"), + ) + let archive = @zip.read(@xlsx.write(workbook)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("shared-formula worksheet is not UTF-8") + } + None => fail("missing shared-formula worksheet") + } + let updated = xml.replace(old~, new=replacement) + if updated == xml { + fail("shared-formula corruption marker was not found") + } + assert_true(archive.replace(path, @encoding/utf8.encode(updated))) + @zip.write(archive) +} + +///| +fn expect_shared_formula_read_error( + label : String, + bytes : BytesView, + expected : String, +) -> Unit raise { + let result : Result[@xlsx.Workbook, Error] = Ok(@xlsx.read(bytes)) catch { + error => Err(error) + } + match result { + Err(@xlsx.InvalidXml(msg~)) => assert_eq(msg, expected) + Err(error) => fail("\{label}: unexpected error \{error}") + Ok(_) => fail("\{label}: malformed shared-formula group was accepted") + } +} + +///| +test "XLSX read rejects missing, out-of-range, and conflicting shared masters" { + let master = + #|D1+1 + let follower = + #| + expect_shared_formula_read_error( + "missing master", + malformed_shared_formula_package(master, follower), + "shared formula follower has no master", + ) + expect_shared_formula_read_error( + "out-of-range follower", + malformed_shared_formula_package( + master, "D1+1", + ), + "shared formula follower is outside the master range", + ) + expect_shared_formula_read_error( + "conflicting masters", + malformed_shared_formula_package( + follower, "E1+1", + ), + "shared formula index has conflicting masters", + ) +} diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 00522b63..09ee85c8 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -385,15 +385,19 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { } ///| -/// Returns non-empty shared-formula masters keyed by OOXML shared index. -/// Followers are deliberately omitted, so master formula text and its copy -/// anchor are retained only once even for large shared ranges. Conflicting -/// masters are rejected rather than resolved by iteration order. -pub fn Worksheet::shared_formula_masters( - self : Worksheet, +/// Indexes and validates one worksheet's shared-formula groups. Followers are +/// omitted from the returned map, so master formula text and its copy anchor +/// are retained only once even for large shared ranges. +fn validated_shared_formula_masters( + cells : ArrayView[Cell], ) -> Map[Int, SharedFormulaMaster] raise XlsxError { let masters : Map[Int, SharedFormulaMaster] = Map([]) - for cell in self.cells { + for cell in cells { + match (cell.formula_type, cell.formula_shared_index) { + (Some(Shared), Some(shared_index)) if shared_index < 0 => + raise InvalidXml(msg="shared formula index is negative") + _ => () + } match ( cell.formula_type, @@ -431,12 +435,41 @@ pub fn Worksheet::shared_formula_masters( } (Some(Shared), Some(formula), Some(_), None) if formula != "" => raise InvalidXml(msg="shared formula master range missing") + (Some(Shared), Some(_), None, _) => + raise InvalidXml(msg="shared formula index missing") + (Some(Shared), None, _, _) => + raise InvalidXml(msg="shared formula text missing") + _ => () + } + } + for cell in cells { + match (cell.formula_type, cell.formula, cell.formula_shared_index) { + (Some(Shared), Some(""), Some(shared_index)) => + match masters.get(shared_index) { + Some(master) => + if !master.contains_coordinate(cell.row, cell.col) { + raise InvalidXml( + msg="shared formula follower is outside the master range", + ) + } + None => raise InvalidXml(msg="shared formula follower has no master") + } _ => () } } masters } +///| +/// Returns non-empty shared-formula masters keyed by OOXML shared index after +/// validating that every follower has exactly one master and lies within the +/// master's declared range. Conflicting or incomplete groups are rejected. +pub fn Worksheet::shared_formula_masters( + self : Worksheet, +) -> Map[Int, SharedFormulaMaster] raise XlsxError { + validated_shared_formula_masters(self.cells) +} + ///| pub fn Worksheet::set_cell_formula( self : Worksheet, From ea0d7d76f05990683fb035e09cb195a73e8f96bc Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 00:11:28 +0800 Subject: [PATCH 020/150] fix(ooxml): harden package metadata parsing --- ooxml/pkg.generated.mbti | 3 + ooxml/read_parse.mbt | 239 ++++++++++++++++++++++++++--------- ooxml/read_parse_test.mbt | 66 ++++++++++ ooxml/start_tag_scanner.mbt | 244 +++++++++++++++++++++++++++++++++--- xlsx/hyperlink.mbt | 5 + xlsx/hyperlink_test.mbt | 45 +++++++ xlsx/read_package_parts.mbt | 124 ++++++++++++------ xlsx/worksheet.mbt | 3 + 8 files changed, 619 insertions(+), 110 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 5c05c540..ebcc9bad 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -16,6 +16,8 @@ pub fn escape_xml_text(StringView) -> String pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> String? raise ParseXmlError +pub fn find_part_content_type(StringView, StringView) -> String? raise ParseXmlError + pub fn parse_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError pub fn parse_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError @@ -47,6 +49,7 @@ pub struct XmlStartTagScanner { } pub fn XmlStartTagScanner::attribute(Self, StringView, StringView) -> String? raise ParseXmlError pub fn XmlStartTagScanner::attribute_view(Self, StringView, StringView) -> StringView? raise ParseXmlError +pub fn XmlStartTagScanner::depth(Self) -> Int pub fn XmlStartTagScanner::local_name(Self) -> StringView pub fn XmlStartTagScanner::namespace_uri(Self) -> StringView pub fn XmlStartTagScanner::new(StringView) -> Self diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 91aa50b2..4a14542c 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -84,6 +84,59 @@ let package_relationships_namespace = "http://schemas.openxmlformats.org/package ///| let package_content_types_namespace = "http://schemas.openxmlformats.org/package/2006/content-types" +///| +let max_relationship_records = 1_000_000 + +///| +let max_relationship_id_chars = 1024 + +///| +let max_relationship_type_chars : Int = 16 * 1024 + +///| +let max_relationship_target_chars : Int = 16 * 1024 * 1024 + +///| +let max_relationship_retained_chars : Int = 16 * 1024 * 1024 + +///| +fn collapse_schema_whitespace(value : StringView) -> String { + let output = StringBuilder::new() + let mut pending_space = false + let mut wrote = false + for unit in value { + if unit == ' ' || unit == '\t' || unit == '\n' || unit == '\r' { + if wrote { + pending_space = true + } + } else { + if pending_space { + output.write_char(' ') + pending_space = false + } + output.write_char(unit) + wrote = true + } + } + output.to_string() +} + +///| +fn relationship_attribute( + scanner : XmlStartTagScanner, + name : StringView, +) -> String raise ParseXmlError { + let value = match scanner.attribute("", name) { + Some(value) => collapse_schema_whitespace(value) + None => + raise InvalidXml(msg="relationship \{name.to_owned().to_lower()} missing") + } + if value == "" { + raise InvalidXml(msg="relationship \{name.to_owned().to_lower()} is empty") + } + value +} + ///| /// Reads relationship targets whose decoded `Type` is in `rel_types`. Element /// names are matched by expanded namespace name, so the package namespace may @@ -92,13 +145,15 @@ pub fn parse_relationship_targets_by_types( xml : StringView, rel_types : ArrayView[String], ) -> Map[String, String] raise ParseXmlError { - let max_records = 16_384 - let max_id_chars = 256 - let max_type_chars = 2_048 - let max_target_chars = 4_096 - let max_retained_chars = 8 * 1024 * 1024 let targets : Map[String, String] = Map([]) + let ids : Set[String] = Set([]) let scanner = XmlStartTagScanner::new(xml) + if !scanner.next() || + scanner.depth() != 1 || + scanner.namespace_uri() != package_relationships_namespace || + scanner.local_name() != "Relationships" { + raise InvalidXml(msg="relationships document element invalid") + } let mut records = 0 let mut retained_chars = 0 while scanner.next() { @@ -106,18 +161,34 @@ pub fn parse_relationship_targets_by_types( scanner.local_name() != "Relationship" { continue } + if scanner.depth() != 2 { + raise InvalidXml(msg="relationship element is not a direct child") + } records = records + 1 - if records > max_records { + if records > max_relationship_records { raise InvalidXml(msg="relationship record limit exceeded") } - let rel_view = match scanner.attribute_view("", "Type") { - Some(value) => value - None => continue - } - if rel_view.length() > max_type_chars { + let rel = relationship_attribute(scanner, "Type") + let id = relationship_attribute(scanner, "Id") + let target = relationship_attribute(scanner, "Target") + if rel.length() > max_relationship_type_chars { raise InvalidXml(msg="relationship type is too long") } - let rel = decode_xml_attribute(rel_view) + if id.length() > max_relationship_id_chars { + raise InvalidXml(msg="relationship id is too long") + } + if target.length() > max_relationship_target_chars { + raise InvalidXml(msg="relationship target is too long") + } + if ids.contains(id) { + raise InvalidXml(msg="duplicate relationship id") + } + ids.add(id) + let retained = id.length() + target.length() + if retained > max_relationship_retained_chars - retained_chars { + raise InvalidXml(msg="relationship retained text limit exceeded") + } + retained_chars = retained_chars + retained let mut wanted = false for rel_type in rel_types { if rel == rel_type { @@ -128,30 +199,6 @@ pub fn parse_relationship_targets_by_types( if !wanted { continue } - let id_view = match scanner.attribute_view("", "Id") { - Some(value) => value - None => raise InvalidXml(msg="relationship id missing") - } - if id_view.length() > max_id_chars { - raise InvalidXml(msg="relationship id is too long") - } - let target_view = match scanner.attribute_view("", "Target") { - Some(value) => value - None => raise InvalidXml(msg="relationship target missing") - } - if target_view.length() > max_target_chars { - raise InvalidXml(msg="relationship target is too long") - } - let id = decode_xml_attribute(id_view) - let target = decode_xml_attribute(target_view) - let retained = id.length() + target.length() - if retained > max_retained_chars - retained_chars { - raise InvalidXml(msg="relationship retained text limit exceeded") - } - retained_chars = retained_chars + retained - if targets.contains(id) { - raise InvalidXml(msg="duplicate relationship id") - } targets[id] = target } targets @@ -166,30 +213,73 @@ pub fn parse_relationship_targets( } ///| -fn parse_content_type_overrides( +priv struct PackageContentTypes { + overrides : Map[String, String] + defaults : Map[String, String] +} + +///| +fn parse_content_types( xml : StringView, -) -> Map[String, String] raise ParseXmlError { +) -> PackageContentTypes raise ParseXmlError { let overrides : Map[String, String] = Map([]) + let defaults : Map[String, String] = Map([]) let scanner = XmlStartTagScanner::new(xml) + if !scanner.next() || + scanner.depth() != 1 || + scanner.namespace_uri() != package_content_types_namespace || + scanner.local_name() != "Types" { + raise InvalidXml(msg="content types document element invalid") + } while scanner.next() { - if scanner.namespace_uri() != package_content_types_namespace || - scanner.local_name() != "Override" { + if scanner.namespace_uri() != package_content_types_namespace { continue } - let part_name = match scanner.attribute("", "PartName") { - Some(value) => value - None => raise InvalidXml(msg="content type override part name missing") + if scanner.depth() != 2 { + raise InvalidXml(msg="content type record is not a direct child") } - let content_type = match scanner.attribute("", "ContentType") { - Some(value) => value - None => raise InvalidXml(msg="content type override content type missing") - } - if overrides.contains(part_name) { - raise InvalidXml(msg="duplicate content type override") + match scanner.local_name() { + "Override" => { + let part_name = match scanner.attribute("", "PartName") { + Some(value) => collapse_schema_whitespace(value) + None => + raise InvalidXml(msg="content type override part name missing") + } + let content_type = match scanner.attribute("", "ContentType") { + Some(value) => collapse_schema_whitespace(value) + None => + raise InvalidXml(msg="content type override content type missing") + } + if part_name == "" || content_type == "" { + raise InvalidXml(msg="content type override value is empty") + } + if overrides.contains(part_name) { + raise InvalidXml(msg="duplicate content type override") + } + overrides[part_name] = content_type + } + "Default" => { + let extension = match scanner.attribute("", "Extension") { + Some(value) => collapse_schema_whitespace(value).to_lower() + None => raise InvalidXml(msg="content type default extension missing") + } + let content_type = match scanner.attribute("", "ContentType") { + Some(value) => collapse_schema_whitespace(value) + None => + raise InvalidXml(msg="content type default content type missing") + } + if extension == "" || content_type == "" { + raise InvalidXml(msg="content type default value is empty") + } + if defaults.contains(extension) { + raise InvalidXml(msg="duplicate content type default") + } + defaults[extension] = content_type + } + _ => raise InvalidXml(msg="content types element is invalid") } - overrides[part_name] = content_type } - overrides + { overrides, defaults } } ///| @@ -197,9 +287,9 @@ pub fn find_override_part_by_content_types( xml : StringView, content_types : ArrayView[String], ) -> String? raise ParseXmlError { - let overrides = parse_content_type_overrides(xml) + let content_types_value = parse_content_types(xml) for wanted in content_types { - for part_name, content_type in overrides { + for part_name, content_type in content_types_value.overrides { if content_type == wanted { return Some(part_name) } @@ -208,6 +298,36 @@ pub fn find_override_part_by_content_types( None } +///| +/// Returns the declared content type for `part_name`. An exact override wins; +/// otherwise the case-insensitive extension default is used. +pub fn find_part_content_type( + xml : StringView, + part_name : StringView, +) -> String? raise ParseXmlError { + let content_types_value = parse_content_types(xml) + let normalized = if part_name.has_prefix("/") { + part_name.to_owned() + } else { + "/" + part_name.to_owned() + } + match content_types_value.overrides.get(normalized) { + Some(value) => Some(value) + None => { + let filename = match normalized.rev_find("/") { + Some(index) => normalized[index + 1:] + None => normalized[:] + } + let extension = match filename.rev_find(".") { + Some(index) if index + 1 < filename.length() => + filename[index + 1:].to_owned().to_lower() + _ => return None + } + content_types_value.defaults.get(extension) + } + } +} + ///| test "ooxml read_parse wb: attr_value parser branches" { debug_inspect(attr_value(" ", "Id"), content="None") @@ -230,15 +350,18 @@ test "ooxml read_parse wb: attr_value parser branches" { } ///| -test "ooxml read_parse wb: relationship type-missing branch continues" { +test "ooxml read_parse wb: every relationship record is validated" { let xml = #| #| #| #| #| - let targets = parse_relationship_targets(xml, "t1") - debug_inspect(targets, content="{}") + try parse_relationship_targets(xml, "t1") catch { + InvalidXml(msg~) => inspect(msg, content="relationship type missing") + } noraise { + _ => fail("expected relationship type missing") + } } ///| @@ -249,7 +372,7 @@ test "ooxml read_parse wb: content-type override required attrs" { #| #| #| - try parse_content_type_overrides(missing_part) catch { + try parse_content_types(missing_part) catch { InvalidXml(msg~) => inspect(msg, content="content type override part name missing") } noraise { @@ -262,7 +385,7 @@ test "ooxml read_parse wb: content-type override required attrs" { #| #| #| - try parse_content_type_overrides(missing_content_type) catch { + try parse_content_types(missing_content_type) catch { InvalidXml(msg~) => inspect(msg, content="content type override content type missing") } noraise { diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index fcdca493..3c399546 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -71,6 +71,55 @@ test "ooxml read_parse: prefixed relationships decode entities exactly once" { ) } +///| +test "ooxml read_parse: relationship schema whitespace is collapsed" { + let xml = + #| + #| + #| + let targets = @ooxml.parse_relationship_targets(xml, "urn:type one") + debug_inspect(targets, content="{ \"r 1\": \"custom/a b.xml\" }") +} + +///| +test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay unique" { + for + xml in [ + ( + #| + ), + ( + #| + ), + ] { + try @ooxml.parse_relationship_targets(xml, "wanted") catch { + InvalidXml(_) => () + } noraise { + _ => fail("expected invalid relationship document") + } + } +} + +///| +test "ooxml read_parse: relationship limits accept writer boundaries" { + let xml = StringBuilder::new() + xml.write_string( + "", + ) + for index in 0..<16_385 { + xml.write_string( + "", + ) + } + let long_target = "x".repeat(4097) + xml.write_string( + "", + ) + xml.write_string("") + let targets = @ooxml.parse_relationship_targets(xml.to_string(), "wanted") + assert_eq(targets.get("wanted"), Some(long_target)) +} + ///| test "ooxml read_parse: parse_content_type_overrides and find workbook part" { let xml = @@ -123,3 +172,20 @@ test "ooxml read_parse: prefixed content types decode names and media types" { content="Some(\"/xl/a&b.xml\")", ) } + +///| +test "ooxml read_parse: content type lookup honors exact override then default" { + let xml = + #| + #| + #| + #| + assert_eq( + @ooxml.find_part_content_type(xml, "custom/book.xml"), + Some("application/workbook+xml"), + ) + assert_eq( + @ooxml.find_part_content_type(xml, "custom/other.Xml"), + Some("application/default+xml"), + ) +} diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index d70e3d8d..3d5bfff1 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -5,7 +5,7 @@ let xml_namespace_uri = "http://www.w3.org/XML/1998/namespace" let xmlns_namespace_uri = "http://www.w3.org/2000/xmlns/" ///| -let max_xml_start_tag_chars : Int = 1024 * 1024 +let max_xml_start_tag_chars : Int = 16 * 1024 * 1024 ///| let max_xml_qname_chars : Int = 1024 @@ -23,6 +23,7 @@ let max_xml_element_depth : Int = 256 priv struct XmlNamespaceBinding { prefix : String uri : String + namespace_id : Int depth : Int } @@ -44,12 +45,18 @@ pub struct XmlStartTagScanner { priv mut index : Int priv mut depth : Int priv namespace_bindings : Array[XmlNamespaceBinding] + priv namespace_ids : Map[String, Int] + priv namespace_ref_counts : Map[Int, Int] + priv mut next_namespace_id : Int priv open_name_starts : Array[Int] priv open_name_ends : Array[Int] priv mut pending_namespace_pop : Int? + priv mut root_seen : Bool + priv mut root_closed : Bool priv mut current_name_start : Int priv mut current_name_end : Int priv mut current_tag_end : Int + priv mut current_depth : Int priv mut current_namespace_uri : String } @@ -226,7 +233,13 @@ fn next_xml_attribute( while index < tag_end && is_attr_space(xml[index]) { index = index + 1 } - if index >= tag_end || xml[index] == ('/' : UInt16) { + if index >= tag_end { + return None + } + if xml[index] == ('/' : UInt16) { + if index + 1 != tag_end { + raise InvalidXml(msg="XML self-closing slash is misplaced") + } return None } let name_start = index @@ -365,7 +378,13 @@ pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { unit == ('\r' : UInt16) { output.write_view(value[literal_start:index]) output.write_char(' ') - index = index + 1 + if unit == ('\r' : UInt16) && + index + 1 < value.length() && + value[index + 1] == ('\n' : UInt16) { + index = index + 2 + } else { + index = index + 1 + } literal_start = index continue } @@ -378,6 +397,88 @@ pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { output.to_string() } +///| +/// Checks an attribute value without retaining a decoded copy. This is used +/// for attributes a caller does not select: malformed entities must never be +/// hidden merely because an attribute is semantically irrelevant. +fn validate_xml_attribute_value(value : StringView) -> Unit raise ParseXmlError { + let mut index = 0 + while index < value.length() { + let unit = value[index] + if unit == ('&' : UInt16) { + let entity_end = match find_xml_sequence(value, index + 1, ";") { + Some(found) => found + None => raise InvalidXml(msg="XML entity is not closed") + } + if xml_entity_value(value[index + 1:entity_end]) is None { + raise InvalidXml(msg="XML entity is invalid") + } + index = entity_end + 1 + continue + } + if unit == ('<' : UInt16) { + raise InvalidXml(msg="XML attribute contains '<'") + } + if unit < 0x20 && + unit != ('\t' : UInt16) && + unit != ('\n' : UInt16) && + unit != ('\r' : UInt16) { + raise InvalidXml(msg="XML attribute contains an invalid character") + } + index = index + 1 + } +} + +///| +fn XmlStartTagScanner::namespace_identity( + self : XmlStartTagScanner, + uri : String, +) -> Int { + if uri == "" { + return 0 + } + if uri == xml_namespace_uri { + return 1 + } + match self.namespace_ids.get(uri) { + Some(namespace_id) => { + self.namespace_ref_counts[namespace_id] = self.namespace_ref_counts.get_or_default( + namespace_id, 0, + ) + + 1 + namespace_id + } + None => { + let namespace_id = self.next_namespace_id + self.next_namespace_id = self.next_namespace_id + 1 + self.namespace_ids[uri] = namespace_id + self.namespace_ref_counts[namespace_id] = 1 + namespace_id + } + } +} + +///| +fn XmlStartTagScanner::release_namespace_identity( + self : XmlStartTagScanner, + binding : XmlNamespaceBinding, +) -> Unit { + if binding.namespace_id < 2 { + return + } + let remaining = self.namespace_ref_counts.get_or_default( + binding.namespace_id, + 0, + ) - + 1 + if remaining <= 0 { + ignore(self.namespace_ref_counts.remove(binding.namespace_id)) + ignore(self.namespace_ids.remove(binding.uri)) + } else { + self.namespace_ref_counts[binding.namespace_id] = remaining + } +} + ///| fn XmlStartTagScanner::pop_namespace_depth( self : XmlStartTagScanner, @@ -388,36 +489,48 @@ fn XmlStartTagScanner::pop_namespace_depth( if self.namespace_bindings[last].depth != depth { break } + let binding = self.namespace_bindings[last] ignore(self.namespace_bindings.pop()) + self.release_namespace_identity(binding) } } ///| -fn XmlStartTagScanner::resolve_prefix( +fn XmlStartTagScanner::resolve_prefix_identity( self : XmlStartTagScanner, prefix : StringView, attribute : Bool, -) -> String raise ParseXmlError { +) -> (String, Int) raise ParseXmlError { if prefix == "xml" { - return xml_namespace_uri + return (xml_namespace_uri, 1) } if prefix.length() == 0 && attribute { - return "" + return ("", 0) } for offset in 0.. String raise ParseXmlError { + let (uri, _) = self.resolve_prefix_identity(prefix, attribute) + uri +} + ///| fn xml_namespace_declaration_prefix( xml : StringView, @@ -478,7 +591,13 @@ fn XmlStartTagScanner::add_namespace_declarations( if self.namespace_bindings.length() >= max_xml_namespace_bindings { raise InvalidXml(msg="XML has too many active namespace bindings") } - self.namespace_bindings.push({ prefix, uri, depth: scope_depth }) + let namespace_id = self.namespace_identity(uri) + self.namespace_bindings.push({ + prefix, + uri, + namespace_id, + depth: scope_depth, + }) } } @@ -488,7 +607,7 @@ fn XmlStartTagScanner::validate_attributes( attributes_start : Int, tag_end : Int, ) -> Unit raise ParseXmlError { - let expanded_names : Set[String] = Set([]) + let expanded_names : Map[Int, Set[String]] = Map([]) let mut next = attributes_start let mut count = 0 while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { @@ -497,6 +616,9 @@ fn XmlStartTagScanner::validate_attributes( raise InvalidXml(msg="XML start tag has too many attributes") } next = attribute.next + validate_xml_attribute_value( + self.xml[attribute.value_start:attribute.value_end], + ) if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { continue } @@ -517,12 +639,20 @@ fn XmlStartTagScanner::validate_attributes( self.xml[attribute.name_start:attribute.name_end], ) } - let uri = self.resolve_prefix(prefix, true) - let expanded = "\{uri.length()}:\{uri}\{local_name_view.to_owned()}" - if expanded_names.contains(expanded) { + let (_, namespace_id) = self.resolve_prefix_identity(prefix, true) + let local_names = match expanded_names.get(namespace_id) { + Some(values) => values + None => { + let values : Set[String] = Set([]) + expanded_names[namespace_id] = values + values + } + } + let local_name = local_name_view.to_owned() + if local_names.contains(local_name) { raise InvalidXml(msg="XML attribute is duplicated") } - expanded_names.add(expanded) + local_names.add(local_name) } } @@ -535,12 +665,18 @@ pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { index: 0, depth: 0, namespace_bindings: [], + namespace_ids: Map([]), + namespace_ref_counts: Map([]), + next_namespace_id: 2, open_name_starts: [], open_name_ends: [], pending_namespace_pop: None, + root_seen: false, + root_closed: false, current_name_start: 0, current_name_end: 0, current_tag_end: 0, + current_depth: 0, current_namespace_uri: "", } } @@ -576,6 +712,9 @@ fn XmlStartTagScanner::consume_end_tag( ignore(self.open_name_ends.pop()) self.pop_namespace_depth(self.depth) self.depth = self.depth - 1 + if self.depth == 0 { + self.root_closed = true + } self.index = tag_end + 1 } @@ -595,6 +734,9 @@ pub fn XmlStartTagScanner::next( while self.index < self.xml.length() { while self.index < self.xml.length() && self.xml[self.index] != ('<' : UInt16) { + if self.depth == 0 && !is_attr_space(self.xml[self.index]) { + raise InvalidXml(msg="XML text is outside the document element") + } self.index = self.index + 1 } if self.index >= self.xml.length() { @@ -624,6 +766,9 @@ pub fn XmlStartTagScanner::next( } if start + 9 <= self.xml.length() && xml_span_equals(self.xml, start, start + 9, "") { Some(value) => value None => raise InvalidXml(msg="XML CDATA section is not closed") @@ -637,6 +782,12 @@ pub fn XmlStartTagScanner::next( self.consume_end_tag(start) continue } + if self.depth == 0 { + if self.root_seen || self.root_closed { + raise InvalidXml(msg="XML has multiple document elements") + } + self.root_seen = true + } let name_start = start + 1 let mut name_end = name_start while name_end < self.xml.length() && @@ -670,9 +821,13 @@ pub fn XmlStartTagScanner::next( } self.current_name_end = name_end self.current_tag_end = tag_end + self.current_depth = scope_depth self.index = tag_end + 1 if self_closing { self.pending_namespace_pop = Some(scope_depth) + if self.depth == 0 { + self.root_closed = true + } } else { self.depth = scope_depth self.open_name_starts.push(name_start) @@ -683,6 +838,9 @@ pub fn XmlStartTagScanner::next( if self.depth != 0 { raise InvalidXml(msg="XML start tag has no matching end tag") } + if !self.root_seen { + raise InvalidXml(msg="XML document element is missing") + } false } @@ -701,6 +859,12 @@ pub fn XmlStartTagScanner::namespace_uri( self.current_namespace_uri } +///| +/// Returns the one-based nesting depth of the current start tag. +pub fn XmlStartTagScanner::depth(self : XmlStartTagScanner) -> Int { + self.current_depth +} + ///| /// Returns the raw, borrowed value of an attribute selected by expanded name. /// The default namespace never applies to unprefixed attributes. @@ -801,6 +965,7 @@ test "start tag scanner restores namespace scopes after self-closing tags" { ///| test "XML attributes decode once and reject invalid entities" { inspect(decode_xml_attribute("a&#x20;b"), content="a b") + inspect(decode_xml_attribute("a\r\nb"), content="a b") for value in ["&unknown;", "�", "�", "&"] { try decode_xml_attribute(value) catch { InvalidXml(_) => () @@ -810,6 +975,55 @@ test "XML attributes decode once and reject invalid entities" { } } +///| +test "start tag scanner validates unused attributes and document structure" { + for + xml in [ + "", "", "", "text", + "", + ] { + let scanner = XmlStartTagScanner::new(xml) + try { + while scanner.next() { + + } + } catch { + InvalidXml(_) => () + } noraise { + _ => fail("expected malformed XML document") + } + } +} + +///| +test "start tag scanner rejects a misplaced self-closing slash" { + let scanner = XmlStartTagScanner::new("x") + try scanner.next() catch { + InvalidXml(msg~) => + inspect(msg, content="XML self-closing slash is misplaced") + } noraise { + _ => fail("expected misplaced self-closing slash") + } +} + +///| +test "namespace duplicate checks retain namespace identities once" { + let attributes = StringBuilder::new() + for index in 0..<4095 { + attributes.write_string(" p:a\{index}='v'") + } + let scanner = XmlStartTagScanner::new( + "", + ) + assert_true(scanner.next()) + assert_eq(scanner.depth(), 1) + assert_false(scanner.next()) +} + ///| test "start tag scanner bounds qualified names" { let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") diff --git a/xlsx/hyperlink.mbt b/xlsx/hyperlink.mbt index 14b5f8dd..d2aa2c8e 100644 --- a/xlsx/hyperlink.mbt +++ b/xlsx/hyperlink.mbt @@ -50,6 +50,11 @@ pub fn HyperlinkOpts::with_values( ///| let max_sheet_hyperlinks = 65529 +///| +/// Keeps every writer-produced relationship start tag below the default +/// 16 MiB OOXML-part ceiling even when every character needs XML escaping. +let max_hyperlink_target_chars : Int = 1024 * 1024 + ///| test "hyperlink limit exceeded" { let sheet = Worksheet::new("Sheet1") diff --git a/xlsx/hyperlink_test.mbt b/xlsx/hyperlink_test.mbt index a18ca431..151221c8 100644 --- a/xlsx/hyperlink_test.mbt +++ b/xlsx/hyperlink_test.mbt @@ -284,6 +284,51 @@ test "hyperlink validates cell reference" { ) } +///| +test "hyperlink relationship reader matches writer record and target boundaries" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + for index in 0..<16_385 { + workbook.set_cell_hyperlink( + "Sheet1", + "A\{index + 1}", + "https://example.test/\{index}", + External, + ) + } + let long_target = "x".repeat(4097) + workbook.set_cell_hyperlink("Sheet1", "B1", long_target, External) + let parsed = @xlsx.read(@xlsx.write(workbook)) + assert_eq( + parsed.get_cell_hyperlink("Sheet1", "A16385").map(link => link.target), + Some("https://example.test/16384"), + ) + assert_eq( + parsed.get_cell_hyperlink("Sheet1", "B1").map(link => link.target), + Some(long_target), + ) +} + +///| +test "hyperlink writer rejects targets that cannot fit the read policy" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + try + workbook.set_cell_hyperlink( + "Sheet1", + "A1", + "x".repeat(1024 * 1024 + 1), + External, + ) + catch { + InvalidHyperlink(msg~) => + inspect(msg, content="hyperlink target is too long") + _ => fail("unexpected hyperlink target error") + } noraise { + _ => fail("expected hyperlink target limit") + } +} + ///| test "hyperlink remove clears range reference" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index 5387d159..6ce384b7 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -11,11 +11,11 @@ fn workbook_part_from_root_relationships( archive : @zip.Archive, part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, -) -> String? raise XlsxError { +) -> String raise XlsxError { let root_rels_path = actual_archive_part_path(part_names, root_rels_part_path) let root_rels_xml = match archive.get(root_rels_path) { Some(bytes) => decode(bytes) - None => return None + None => raise MissingPart(path=root_rels_path) } let office_document_type = transitional_office_relationship_prefix + "officeDocument" @@ -26,42 +26,47 @@ fn workbook_part_from_root_relationships( match first_relationship_target(targets) { Some(target) => { let part_name = archive_path_from_part_name(target) - Some(normalize_rel_part_path(part_name)) + normalize_rel_part_path(part_name) } - None => None + None => raise InvalidXml(msg="root workbook relationship missing") } } +///| +fn workbook_content_type_is_supported(value : StringView) -> Bool { + for candidate in workbook_content_type_candidates() { + if value == candidate { + return true + } + } + false +} + ///| fn resolve_workbook_xml_part_path( archive : @zip.Archive, part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, ) -> String raise XlsxError { + let workbook_part = workbook_part_from_root_relationships( + archive, part_names, decode, + ) let content_types_path = actual_archive_part_path( part_names, content_types_part_path, ) - match archive.get(content_types_path) { - Some(content_types_bytes) => { - let content_types_xml = decode(content_types_bytes) - let workbook_part_name = @ooxml.find_override_part_by_content_types( - content_types_xml, - workbook_content_type_candidates(), - ) catch { - InvalidXml(msg~) => raise InvalidXml(msg~) - } - match workbook_part_name { - Some(part_name) => archive_path_from_part_name(part_name) - None => - workbook_part_from_root_relationships(archive, part_names, decode).unwrap_or( - workbook_part_path, - ) - } - } - None => - workbook_part_from_root_relationships(archive, part_names, decode).unwrap_or( - workbook_part_path, - ) + let content_types_xml = match archive.get(content_types_path) { + Some(content_types_bytes) => decode(content_types_bytes) + None => raise MissingPart(path=content_types_path) + } + let content_type = @ooxml.find_part_content_type( + content_types_xml, workbook_part, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } + match content_type { + Some(value) if workbook_content_type_is_supported(value) => workbook_part + Some(_) => raise InvalidXml(msg="root workbook content type is invalid") + None => raise InvalidXml(msg="root workbook content type is missing") } } @@ -84,16 +89,18 @@ fn resolve_workbook_xml_part_path_for_test( } ///| -test "read package parts: default workbook path when [Content_Types] missing" { +test "read package parts: root relationship is required" { let archive = @zip.Archive::new() - inspect( - resolve_workbook_xml_part_path_for_test(archive), - content="xl/workbook.xml", - ) + try resolve_workbook_xml_part_path_for_test(archive) catch { + MissingPart(path~) => inspect(path, content="_rels/.rels") + _ => fail("unexpected missing-root error") + } noraise { + _ => fail("expected missing root relationships") + } } ///| -test "read package parts: workbook path resolved from [Content_Types] override" { +test "read package parts: workbook path follows the root relationship" { let archive = @zip.Archive::new() let content_types = #| @@ -102,6 +109,11 @@ test "read package parts: workbook path resolved from [Content_Types] override" #| #| archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) + let root_relationships = + #| + #| + #| + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) inspect( resolve_workbook_xml_part_path_for_test(archive), content="xl/workbook2.xml", @@ -109,7 +121,7 @@ test "read package parts: workbook path resolved from [Content_Types] override" } ///| -test "read package parts: fallback to default when workbook content type absent" { +test "read package parts: root target must have a workbook content type" { let archive = @zip.Archive::new() let content_types = #| @@ -118,10 +130,18 @@ test "read package parts: fallback to default when workbook content type absent" #| #| archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) - inspect( - resolve_workbook_xml_part_path_for_test(archive), - content="xl/workbook.xml", - ) + let root_relationships = + #| + #| + #| + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + try resolve_workbook_xml_part_path_for_test(archive) catch { + InvalidXml(msg~) => + inspect(msg, content="root workbook content type is missing") + _ => fail("unexpected workbook content-type error") + } noraise { + _ => fail("expected missing workbook content type") + } } ///| @@ -134,6 +154,11 @@ test "read package parts: malformed [Content_Types] raises InvalidXml" { #| #| archive.add(content_types_part_path, @encoding/utf8.encode(malformed)) + let root_relationships = + #| + #| + #| + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) let result = Ok(resolve_workbook_xml_part_path_for_test(archive)) catch { e => Err(e) } @@ -144,7 +169,7 @@ test "read package parts: malformed [Content_Types] raises InvalidXml" { } ///| -test "read package parts: invalid workbook part name raises InvalidXml" { +test "read package parts: invalid root workbook target raises InvalidXml" { let archive = @zip.Archive::new() let invalid_part_name = #| @@ -153,6 +178,11 @@ test "read package parts: invalid workbook part name raises InvalidXml" { #| #| archive.add(content_types_part_path, @encoding/utf8.encode(invalid_part_name)) + let root_relationships = + #| + #| + #| + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) let result = Ok(resolve_workbook_xml_part_path_for_test(archive)) catch { e => Err(e) } @@ -163,6 +193,26 @@ test "read package parts: invalid workbook part name raises InvalidXml" { } } +///| +test "read package parts: stale workbook override cannot redirect the root relationship" { + let archive = @zip.Archive::new() + let content_types = + #| + #| + #| + #| + let root_relationships = + #| + #| + #| + archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + inspect( + resolve_workbook_xml_part_path_for_test(archive), + content="actual/book.xml", + ) +} + ///| test "read package parts: default-declared workbook follows the root relationship" { let archive = @zip.Archive::new() diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 09ee85c8..6df57d90 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -760,6 +760,9 @@ pub fn Worksheet::set_cell_hyperlink( self.remove_cell_hyperlink(resolved) return } + if target.length() > max_hyperlink_target_chars { + raise InvalidHyperlink(msg="hyperlink target is too long") + } let display_value = if display == "" { None } else { Some(display) } let tooltip_value = if tooltip == "" { None } else { Some(tooltip) } let normalized_ref = normalize_cell_or_range_ref(resolved) From 5560a025da1e733041a742185a4d237b255532f2 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 00:24:30 +0800 Subject: [PATCH 021/150] fix(xlsx): canonicalize namespace-qualified parts --- ooxml/pkg.generated.mbti | 2 + ooxml/start_tag_scanner.mbt | 154 +++++++++++++++++++++++++++++++++-- xlsx/read.mbt | 16 +++- xlsx/read_namespace_test.mbt | 29 ++++++- xlsx/read_xml_namespaces.mbt | 28 +++++++ 5 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 xlsx/read_xml_namespaces.mbt diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index ebcc9bad..bd48ee07 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -8,6 +8,8 @@ import { // Values pub fn attr_value(StringView, StringView) -> String? raise ParseXmlError +pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)]) -> String raise ParseXmlError + pub fn decode_xml_attribute(StringView) -> String raise ParseXmlError pub fn escape_xml_attr(StringView) -> String diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 3d5bfff1..570e4264 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -50,6 +50,12 @@ pub struct XmlStartTagScanner { priv mut next_namespace_id : Int priv open_name_starts : Array[Int] priv open_name_ends : Array[Int] + priv open_name_rewrites : Array[Bool] + priv rewrite_element_namespaces : Array[String] + priv rewrite_attribute_prefixes : Map[String, String] + priv rewrite_output : StringBuilder + priv mut rewrite_cursor : Int + priv mut rewrite_count : Int priv mut pending_namespace_pop : Int? priv mut root_seen : Bool priv mut root_closed : Bool @@ -639,7 +645,7 @@ fn XmlStartTagScanner::validate_attributes( self.xml[attribute.name_start:attribute.name_end], ) } - let (_, namespace_id) = self.resolve_prefix_identity(prefix, true) + let (uri, namespace_id) = self.resolve_prefix_identity(prefix, true) let local_names = match expanded_names.get(namespace_id) { Some(values) => values None => { @@ -653,13 +659,66 @@ fn XmlStartTagScanner::validate_attributes( raise InvalidXml(msg="XML attribute is duplicated") } local_names.add(local_name) + match self.rewrite_attribute_prefixes.get(uri) { + Some(canonical_prefix) => { + let replacement = if canonical_prefix == "" { + local_name + } else { + canonical_prefix + ":" + local_name + } + if !xml_span_equals( + self.xml, + attribute.name_start, + attribute.name_end, + replacement, + ) { + self.record_name_rewrite( + attribute.name_start, + attribute.name_end, + replacement, + ) + } + } + None => () + } } } ///| -/// Creates a namespace-aware cursor over `xml`. The source is borrowed; no -/// copy of the full XML part is made. -pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { +fn XmlStartTagScanner::record_name_rewrite( + self : XmlStartTagScanner, + start : Int, + end : Int, + replacement : String, +) -> Unit raise ParseXmlError { + if start < self.rewrite_cursor || end < start { + raise InvalidXml(msg="XML name rewrites overlap") + } + self.rewrite_output.write_view(self.xml[self.rewrite_cursor:start]) + self.rewrite_output.write_string(replacement) + self.rewrite_cursor = end + self.rewrite_count = self.rewrite_count + 1 +} + +///| +fn XmlStartTagScanner::should_unprefix_element( + self : XmlStartTagScanner, + namespace_uri : StringView, +) -> Bool { + for wanted in self.rewrite_element_namespaces { + if xml_string_equals_view(wanted, namespace_uri) { + return true + } + } + false +} + +///| +fn xml_start_tag_scanner_with_rewrites( + xml : StringView, + element_namespaces : Array[String], + attribute_prefixes : Map[String, String], +) -> XmlStartTagScanner { { xml, index: 0, @@ -670,6 +729,12 @@ pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { next_namespace_id: 2, open_name_starts: [], open_name_ends: [], + open_name_rewrites: [], + rewrite_element_namespaces: element_namespaces, + rewrite_attribute_prefixes: attribute_prefixes, + rewrite_output: StringBuilder::new(), + rewrite_cursor: 0, + rewrite_count: 0, pending_namespace_pop: None, root_seen: false, root_closed: false, @@ -681,6 +746,13 @@ pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { } } +///| +/// Creates a namespace-aware cursor over `xml`. The source is borrowed; no +/// copy of the full XML part is made. +pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { + xml_start_tag_scanner_with_rewrites(xml, [], Map([])) +} + ///| fn XmlStartTagScanner::consume_end_tag( self : XmlStartTagScanner, @@ -696,7 +768,7 @@ fn XmlStartTagScanner::consume_end_tag( self.xml[name_end] != ('>' : UInt16) { name_end = name_end + 1 } - ignore(validate_xml_qname(self.xml, name_start, name_end)) + let colon = validate_xml_qname(self.xml, name_start, name_end) let tag_end = find_xml_tag_end(self.xml, name_end, start) for index in name_end.. + self.record_name_rewrite( + name_start, + name_end, + self.xml[value + 1:name_end].to_owned(), + ) + None => () + } + } ignore(self.open_name_starts.pop()) ignore(self.open_name_ends.pop()) + ignore(self.open_name_rewrites.pop()) self.pop_namespace_depth(self.depth) self.depth = self.depth - 1 if self.depth == 0 { @@ -809,12 +895,24 @@ pub fn XmlStartTagScanner::next( raise InvalidXml(msg="XML element nesting is too deep") } self.add_namespace_declarations(name_end, tag_end, scope_depth) - self.validate_attributes(name_end, tag_end) let prefix = match colon { Some(value) => self.xml[name_start:value] None => self.xml[name_start:name_start] } self.current_namespace_uri = self.resolve_prefix(prefix, false) + let rewrite_name = colon is Some(_) && + self.should_unprefix_element(self.current_namespace_uri) + if rewrite_name { + self.record_name_rewrite( + name_start, + name_end, + match colon { + Some(value) => self.xml[value + 1:name_end].to_owned() + None => self.xml[name_start:name_end].to_owned() + }, + ) + } + self.validate_attributes(name_end, tag_end) self.current_name_start = match colon { Some(value) => value + 1 None => name_start @@ -832,6 +930,7 @@ pub fn XmlStartTagScanner::next( self.depth = scope_depth self.open_name_starts.push(name_start) self.open_name_ends.push(name_end) + self.open_name_rewrites.push(rewrite_name) } return true } @@ -865,6 +964,34 @@ pub fn XmlStartTagScanner::depth(self : XmlStartTagScanner) -> Int { self.current_depth } +///| +/// Validates `xml` and rewrites selected expanded names into lexical forms +/// expected by a namespace-oblivious parser. Elements in `element_namespaces` +/// lose their prefix; attributes in `attribute_prefixes` receive the mapped +/// prefix. Text, comments, CDATA, values, and unselected names are unchanged. +pub fn canonicalize_xml_expanded_names( + xml : StringView, + element_namespaces : ArrayView[String], + attribute_prefixes : ArrayView[(String, String)], +) -> String raise ParseXmlError { + let elements : Array[String] = [] + elements.append(element_namespaces) + let attributes : Map[String, String] = Map([]) + for pair in attribute_prefixes { + let (namespace_uri, prefix) = pair + attributes[namespace_uri] = prefix + } + let scanner = xml_start_tag_scanner_with_rewrites(xml, elements, attributes) + while scanner.next() { + + } + if scanner.rewrite_count == 0 { + return xml.to_owned() + } + scanner.rewrite_output.write_view(xml[scanner.rewrite_cursor:]) + scanner.rewrite_output.to_string() +} + ///| /// Returns the raw, borrowed value of an attribute selected by expanded name. /// The default namespace never applies to unprefixed attributes. @@ -1024,6 +1151,21 @@ test "namespace duplicate checks retain namespace identities once" { assert_false(scanner.next()) } +///| +test "expanded-name canonicalization preserves data and rewrites both tag ends" { + let xml = + #|]]> + let canonical = canonicalize_xml_expanded_names(xml, ["urn:sheet"], [ + ("urn:rel", "r"), + ]) + inspect( + canonical, + content=( + #|]]> + ), + ) +} + ///| test "start tag scanner bounds qualified names" { let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") diff --git a/xlsx/read.mbt b/xlsx/read.mbt index b10ad946..19d266b1 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3158,7 +3158,7 @@ fn read_zip_archive_core_unchecked( ) -> Workbook raise XlsxError { let part_names = archive_part_name_index(archive) let read_budget = ReadBudget::new(limits, cancelled~) - let decode = (value : BytesView) => { + let decode_source = (value : BytesView) => { if value.length() > limits.max_xml_part_bytes { raise ResourceLimitExceeded( kind="xml_part_bytes", @@ -3170,6 +3170,11 @@ fn read_zip_archive_core_unchecked( read_budget.scan_xml(decoded) decoded } + let decode = (value : BytesView) => { + let decoded = decode_source(value) + read_budget.charge_work(decoded.length()) + canonicalize_xlsx_xml(decoded) + } let workbook_xml_part_path = actual_archive_part_path( part_names, resolve_workbook_xml_part_path(archive, part_names, decode), @@ -3178,7 +3183,9 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path=workbook_xml_part_path) } - let workbook_xml = decode(workbook_bytes) + let workbook_source_xml = decode_source(workbook_bytes) + read_budget.charge_work(workbook_source_xml.length()) + let workbook_xml = canonicalize_xlsx_xml(workbook_source_xml) let workbook_rels_path = actual_archive_part_path( part_names, rels_path_for_part(workbook_xml_part_path), @@ -3187,7 +3194,10 @@ fn read_zip_archive_core_unchecked( Some(value) => decode(value) None => raise MissingPart(path=workbook_rels_path) } - let sheets = parse_workbook_sheets(workbook_xml, limits.max_workbook_sheets) + let sheets = parse_workbook_sheets( + workbook_source_xml, + limits.max_workbook_sheets, + ) let sheet_names : Array[String] = [] for entry in sheets { sheet_names.push(entry.name) diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 7a236579..440b9d56 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -58,23 +58,36 @@ fn namespace_qualified_xlsx_fixture( " \n" + " \n" + " \n" + + " \n" + "" let root_relationships = "\n" + " \n" + "" let workbook = "\n" + + " \n" + + " \n" + " \n" + + " \n" + + " \n" + "" + let shared_strings_target = if entity_paths { + "../xl/sharedStrings.xml" + } else { + "sharedStrings.xml" + } let workbook_relationships = "\n" + " \n" + + " \n" + "" - let worksheet = "qualified-value" + let worksheet = "0" + let shared_strings = "qualified-value" let archive = @zip.Archive::new() archive.add("[Content_Types].xml", @encoding/utf8.encode(content_types)) archive.add("_rels/.rels", @encoding/utf8.encode(root_relationships)) archive.add(workbook_path, @encoding/utf8.encode(workbook)) archive.add(workbook_rels_path, @encoding/utf8.encode(workbook_relationships)) archive.add(worksheet_path, @encoding/utf8.encode(worksheet)) + archive.add("xl/sharedStrings.xml", @encoding/utf8.encode(shared_strings)) @zip.write(archive) } @@ -87,6 +100,20 @@ test "namespace-qualified Transitional and Strict workbook parts read end to end workbook.get_cell_value("Qualified", "A1"), content="Some(\"qualified-value\")", ) + assert_eq(workbook.get_workbook_props().date_1904, Some(true)) + assert_eq(workbook.get_workbook_props().code_name, Some("QualifiedBook")) + assert_eq(workbook.get_calc_props().calc_id, Some(123U)) + assert_eq(workbook.get_calc_props().ref_mode, Some("R1C1")) + assert_eq(workbook.get_merge_cells("Qualified"), ["A1:B1"]) + let rewritten = @zip.read(@xlsx.write(workbook)) + guard rewritten.get("xl/workbook.xml") is Some(workbook_xml) else { + fail("missing rewritten workbook") + } + let workbook_text = @encoding/utf8.decode(workbook_xml) catch { + _ => fail("rewritten workbook is not UTF-8") + } + assert_true(workbook_text.contains(" String raise XlsxError { + @ooxml.canonicalize_xml_expanded_names( + xml, + [transitional_spreadsheet_namespace, strict_spreadsheet_namespace], + [ + (transitional_relationship_attribute_namespace, "r"), + (strict_relationship_attribute_namespace, "r"), + ], + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + +///| +test "XLSX XML canonicalization handles Strict elements and relationship attributes" { + let source = + #|ok + inspect( + canonicalize_xlsx_xml(source), + content=( + #|ok + ), + ) +} From 8a58a08d822fe67075442ef5e8e5690dc7444dff Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 00:34:26 +0800 Subject: [PATCH 022/150] fix(xlsx): budget sqref normalization work --- xlsx/ignored_errors.mbt | 7 +++- xlsx/read.mbt | 62 ++++++++++++++++++++++------- xlsx/read_budget.mbt | 78 +++++++++++++++++++++++++++++++++++-- xlsx/read_limits_test.mbt | 34 ++++++++++++++++ xlsx/read_worksheet_xml.mbt | 3 +- xlsx/sqref.mbt | 57 +++++++++++++++++++++++---- 6 files changed, 216 insertions(+), 25 deletions(-) diff --git a/xlsx/ignored_errors.mbt b/xlsx/ignored_errors.mbt index 75ba077a..3146e617 100644 --- a/xlsx/ignored_errors.mbt +++ b/xlsx/ignored_errors.mbt @@ -114,6 +114,7 @@ fn write_ignored_errors_xml( ///| fn parse_ignored_errors( xml : StringView, + budget? : ReadBudget, ) -> Array[IgnoredError] raise XlsxError { let body = match extract_tag_body(xml, "ignoredErrors") { Some(value) => value @@ -138,7 +139,11 @@ fn parse_ignored_errors( let tag = text[:end] let sqref = match attr_value(tag, "sqref") { Some(value) => - normalize_sqref_for_read(unescape_xml_text(value), "ignoredError") + normalize_sqref_for_read( + unescape_xml_text(value), + "ignoredError", + budget?, + ) None => raise InvalidXml(msg="ignoredError sqref missing") } let error : IgnoredError = { diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 19d266b1..2a593b2e 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -1047,7 +1047,10 @@ fn parse_merge_cells(xml : StringView) -> Array[String] raise XlsxError { } ///| -fn parse_data_validations(xml : StringView) -> Array[String] raise XlsxError { +fn parse_data_validations( + xml : StringView, + budget? : ReadBudget, +) -> Array[String] raise XlsxError { let validations : Array[String] = [] let body = match extract_tag_body(xml, "dataValidations") { Some(value) => value @@ -1078,7 +1081,10 @@ fn parse_data_validations(xml : StringView) -> Array[String] raise XlsxError { } validations.push( normalize_xml_sqref_attr_for_read( - full, "dataValidation", "data validation", + full, + "dataValidation", + "data validation", + budget?, ), ) } @@ -1088,6 +1094,7 @@ fn parse_data_validations(xml : StringView) -> Array[String] raise XlsxError { ///| fn parse_data_validations_x14( xml : StringView, + budget? : ReadBudget, ) -> Array[String] raise XlsxError { let validations : Array[String] = [] let xml_str = xml.to_owned() @@ -1126,6 +1133,7 @@ fn parse_data_validations_x14( normalize_sqref_for_read( unescape_xml_text(value), "x14 data validation", + budget?, ) None => raise InvalidXml(msg="x14:dataValidation sqref missing") } @@ -1180,7 +1188,10 @@ fn parse_data_validations_x14( } ///| -fn parse_conditional_formats(xml : StringView) -> Array[String] raise XlsxError { +fn parse_conditional_formats( + xml : StringView, + budget? : ReadBudget, +) -> Array[String] raise XlsxError { let formats : Array[String] = [] let xml_str = xml.to_owned() if !xml_str.contains(" Array[String] raise XlsxError } formats.push( normalize_xml_sqref_attr_for_read( - full, "conditionalFormatting", "conditional formatting", + full, + "conditionalFormatting", + "conditional formatting", + budget?, ), ) } @@ -1419,6 +1433,7 @@ fn parse_x14_conditional_formatting_blocks( ///| fn parse_x14_data_bars( xml : StringView, + budget? : ReadBudget, ) -> Map[String, X14DataBarProps] raise XlsxError { let bars : Map[String, X14DataBarProps] = Map([]) let blocks = parse_x14_conditional_formatting_blocks(xml) @@ -1431,6 +1446,7 @@ fn parse_x14_data_bars( normalize_sqref_for_read( unescape_xml_text(value), "x14 conditional formatting", + budget?, ) None => match extract_tag_body_from(full, "sqref") { @@ -1438,6 +1454,7 @@ fn parse_x14_data_bars( normalize_sqref_for_read( unescape_xml_text(value), "x14 conditional formatting", + budget?, ) None => continue } @@ -1587,6 +1604,7 @@ fn parse_unknown_worksheet_ext_blocks( ///| fn parse_conditional_formats_x14( xml : StringView, + budget? : ReadBudget, ) -> Array[String] raise XlsxError { let formats : Array[String] = [] let blocks = parse_x14_conditional_formatting_blocks(xml) @@ -1599,6 +1617,7 @@ fn parse_conditional_formats_x14( normalize_sqref_for_read( unescape_xml_text(value), "x14 conditional formatting", + budget?, ) None => match extract_tag_body_from(full, "sqref") { @@ -1606,6 +1625,7 @@ fn parse_conditional_formats_x14( normalize_sqref_for_read( unescape_xml_text(value), "x14 conditional formatting", + budget?, ) None => raise InvalidXml(msg="x14:conditionalFormatting sqref missing") @@ -1789,6 +1809,7 @@ fn parse_sheet_view_pane(body : StringView) -> SheetPane? raise XlsxError { ///| fn parse_sheet_view_selections( body : StringView, + budget? : ReadBudget, ) -> Array[Selection] raise XlsxError { let selections : Array[Selection] = [] let mut first = true @@ -1813,6 +1834,7 @@ fn parse_sheet_view_selections( selection.sqref = normalize_sqref_for_read( unescape_xml_text(value), "selection", + budget?, ) None => () } @@ -1834,7 +1856,10 @@ fn parse_sheet_view_selections( } ///| -fn parse_sheet_views(xml : StringView) -> Array[SheetView] raise XlsxError { +fn parse_sheet_views( + xml : StringView, + budget? : ReadBudget, +) -> Array[SheetView] raise XlsxError { let views : Array[SheetView] = [] let body = match extract_tag_body(xml, "sheetViews") { Some(value) => value @@ -1925,7 +1950,7 @@ fn parse_sheet_views(xml : StringView) -> Array[SheetView] raise XlsxError { if close > body_start { let inner = text[body_start:close] view.pane = parse_sheet_view_pane(inner) - let selections = parse_sheet_view_selections(inner) + let selections = parse_sheet_view_selections(inner, budget?) view.selection.clear() view.selection.append(selections) } @@ -3466,7 +3491,12 @@ fn read_zip_archive_core_unchecked( sheet_props, dimension_ref, picture_rel_id, - ) = parse_worksheet(sheet_xml, shared_strings, style_count) + ) = parse_worksheet( + sheet_xml, + shared_strings, + style_count, + budget=read_budget, + ) let hyperlink_elements = parse_hyperlink_elements(sheet_xml) let table_part_ids = parse_table_part_ids(sheet_xml) let pivot_part_ids = parse_pivot_table_part_ids(sheet_xml) @@ -3479,17 +3509,23 @@ fn read_zip_archive_core_unchecked( sheet_xml, "legacyDrawingHF", ) let sparkline_groups = parse_sparkline_groups(sheet_xml) - let data_validations = parse_data_validations(sheet_xml) - for dv in parse_data_validations_x14(sheet_xml) { + let data_validations = parse_data_validations( + sheet_xml, + budget=read_budget, + ) + for dv in parse_data_validations_x14(sheet_xml, budget=read_budget) { data_validations.push(dv) } - let conditional_formats = parse_conditional_formats(sheet_xml) - for cf in parse_conditional_formats_x14(sheet_xml) { + let conditional_formats = parse_conditional_formats( + sheet_xml, + budget=read_budget, + ) + for cf in parse_conditional_formats_x14(sheet_xml, budget=read_budget) { conditional_formats.push(cf) } - let x14_data_bars = parse_x14_data_bars(sheet_xml) + let x14_data_bars = parse_x14_data_bars(sheet_xml, budget=read_budget) let unknown_ext_blocks = parse_unknown_worksheet_ext_blocks(sheet_xml) - let ignored_errors = parse_ignored_errors(sheet_xml) + let ignored_errors = parse_ignored_errors(sheet_xml, budget=read_budget) let mut needs_hyperlink_rels = false for link in hyperlink_elements { if link.r_id is Some(_) { diff --git a/xlsx/read_budget.mbt b/xlsx/read_budget.mbt index 45e98fa7..97c453a0 100644 --- a/xlsx/read_budget.mbt +++ b/xlsx/read_budget.mbt @@ -109,6 +109,45 @@ fn ReadBudget::tag_end( None } +///| +fn xml_sequence_at( + xml : StringView, + start : Int, + sequence : StringView, +) -> Bool { + if start < 0 || start > xml.length() - sequence.length() { + return false + } + for offset in 0.. Int? raise XlsxError { + let mut index = start + let mut next_checkpoint = start + while index <= xml.length() - sequence.length() { + if index >= next_checkpoint { + self.checkpoint() + next_checkpoint = index + 4096 + } + if xml_sequence_at(xml, index, sequence) { + return Some(index + sequence.length()) + } + index = index + 1 + } + None +} + ///| fn xml_name_local_start(xml : StringView, start : Int, end : Int) -> Int { let mut local_start = start @@ -182,12 +221,36 @@ fn ReadBudget::scan_xml( continue } let first = xml[index + 1] - if first == ('/' : UInt16) || - first == ('?' : UInt16) || - first == ('!' : UInt16) { + if first == ('/' : UInt16) { index = index + 2 continue } + if first == ('?' : UInt16) { + index = match self.sequence_end(xml, index + 2, "?>") { + Some(value) => value + None => xml.length() + } + continue + } + if first == ('!' : UInt16) { + index = if xml_sequence_at(xml, index, "") { + Some(value) => value + None => xml.length() + } + } else if xml_sequence_at(xml, index, "") { + Some(value) => value + None => xml.length() + } + } else { + match self.tag_end(xml, index + 2) { + Some(value) => value + 1 + None => xml.length() + } + } + continue + } let name_start = index + 1 let mut name_end = name_start while name_end < xml.length() && @@ -238,3 +301,12 @@ test "read budget checks cancellation inside a long start tag" { _ => fail("expected read cancellation") } } + +///| +test "read budget ignores element-like text in comments and CDATA" { + let budget = ReadBudget::new(ReadLimits::with_values(max_parser_items=2)) + budget.scan_xml( + "]]>", + ) + assert_eq(budget.parser_items, 2) +} diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index 6fee0be1..6ed31a50 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -317,6 +317,40 @@ test "semantic read limits reject XML items and derived column expansion" { } } +///| +test "semantic read limits charge sqref token materialization" { + let bytes = bounded_workbook_fixture() + let refs : Array[String] = [] + for _ in 0..<1200 { + refs.push("A1") + } + let joined_refs = refs.join(" ") + let worksheet = "\n" + + "\n" + + " \n" + + " \n" + + "\n" + let expanded = replace_xlsx_part(bytes, "xl/worksheets/sheet1.xml", worksheet) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read( + expanded, + limits=@xlsx.ReadLimits::with_values(max_parser_items=1000), + ), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="parser_items") + assert_eq(limit, 1000) + assert_true(actual > limit) + } + _ => fail("expected sqref parser item limit") + } +} + ///| test "semantic read limits bound cumulative parser work" { let bytes = bounded_workbook_fixture() diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index 53b61d89..cf46de49 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -3,6 +3,7 @@ fn parse_worksheet( xml : StringView, shared_strings : Array[SharedStringEntry], style_count : Int, + budget? : ReadBudget, ) -> ( Array[Cell], Array[String], @@ -249,7 +250,7 @@ fn parse_worksheet( let sheet_protection = parse_sheet_protection(xml) let row_breaks = parse_page_breaks(xml, "rowBreaks") let col_breaks = parse_page_breaks(xml, "colBreaks") - let sheet_views = parse_sheet_views(xml) + let sheet_views = parse_sheet_views(xml, budget?) let sheet_props = parse_sheet_props(xml) let dimension_ref = parse_sheet_dimension(xml) let picture_rel_id = parse_picture_rel_id(xml) diff --git a/xlsx/sqref.mbt b/xlsx/sqref.mbt index 093c578a..32c47cf3 100644 --- a/xlsx/sqref.mbt +++ b/xlsx/sqref.mbt @@ -60,15 +60,39 @@ fn validate_cell_or_range_ref_for_read( fn normalize_sqref_for_read( value : StringView, field : StringView, + budget? : ReadBudget, ) -> String raise XlsxError { - let tokens = xml_whitespace_tokens(value) - if tokens.length() == 0 { - invalid_coordinate_xml(field) + let normalized = StringBuilder::new() + let mut token_start = -1 + let mut token_count = 0 + for index in 0..<=value.length() { + let at_end = index == value.length() + let separator = at_end || is_xml_attr_space_unit(value[index]) + if !separator && token_start < 0 { + token_start = index + } else if separator && token_start >= 0 { + let token = value[token_start:index] + match budget { + Some(value) => { + value.checkpoint() + value.charge_items(1) + value.charge_work(token.length()) + } + None => () + } + ignore(validate_cell_or_range_ref_for_read(token, field)) + if token_count > 0 { + normalized.write_char(' ') + } + normalized.write_view(token) + token_count = token_count + 1 + token_start = -1 + } } - for token in tokens { - ignore(validate_cell_or_range_ref_for_read(token, field)) + if token_count == 0 { + invalid_coordinate_xml(field) } - tokens.join(" ") + normalized.to_string() } ///| @@ -98,6 +122,7 @@ fn normalize_xml_sqref_attr_for_read( xml : StringView, tag_name : StringView, field : StringView, + budget? : ReadBudget, ) -> String raise XlsxError { let tag = match tag_attributes_in(xml, tag_name) { Some(value) => value @@ -107,7 +132,7 @@ fn normalize_xml_sqref_attr_for_read( Some(value) => unescape_xml_text(value) None => raise InvalidXml(msg="\{field.to_owned()} sqref missing") } - let normalized = normalize_sqref_for_read(raw, field) + let normalized = normalize_sqref_for_read(raw, field, budget?) if normalized == raw { return xml.to_owned() } @@ -117,6 +142,24 @@ fn normalize_xml_sqref_attr_for_read( } } +///| +test "sqref read normalization charges tokens before materializing them" { + let budget = ReadBudget::new(ReadLimits::with_values(max_parser_items=2)) + let result : Result[String, Error] = Ok( + normalize_sqref_for_read("A1 B2 C3", "selection", budget~), + ) catch { + error => Err(error) + } + match result { + Err(ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="parser_items") + assert_eq(limit, 2) + assert_eq(actual, 3) + } + _ => fail("expected sqref parser-item limit") + } +} + ///| fn sort_and_merge_sqref_row_intervals(entries : Array[(Int, Int)]) -> Unit { if entries.length() < 2 { From e7279bac6de2a7a1ee098fc8bef78d5b8fcdd8e5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 00:36:14 +0800 Subject: [PATCH 023/150] test(xlsx): align discovery fixtures with root relationships --- xlsx/io_password_test.mbt | 2 +- xlsx/ooxml_rels_error_test.mbt | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/xlsx/io_password_test.mbt b/xlsx/io_password_test.mbt index 381832e5..6950fcad 100644 --- a/xlsx/io_password_test.mbt +++ b/xlsx/io_password_test.mbt @@ -100,7 +100,7 @@ test "standard encryption distinguishes wrong passwords from malformed XLSX" { error => Err(error) } match malformed { - Err(@xlsx.MissingPart(path~)) => inspect(path, content="xl/workbook.xml") + Err(@xlsx.MissingPart(path~)) => inspect(path, content="_rels/.rels") _ => fail("expected a parsed-package failure, not InvalidPassword") } diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index 0ab0eeab..d06bcbd5 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -220,7 +220,7 @@ test "ooxml rels: reader rejects case-equivalent part collisions" { } ///| -test "ooxml rels: workbook part path discovered from [Content_Types]" { +test "ooxml rels: relocated workbook path requires root relationship and content type" { let workbook = @xlsx.Workbook::new() ignore(workbook.add_sheet("Sheet1")) let bytes = @xlsx.write(workbook) @@ -235,6 +235,13 @@ test "ooxml rels: workbook part path discovered from [Content_Types]" { Some(value) => value None => fail("missing workbook.xml.rels") } + let root_rels_xml = match archive.get("_rels/.rels") { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("root relationships utf8 decode failed") + } + None => fail("missing _rels/.rels") + } let content_types_xml = match archive.get("[Content_Types].xml") { Some(value) => @encoding/utf8.decode(value) catch { @@ -246,10 +253,15 @@ test "ooxml rels: workbook part path discovered from [Content_Types]" { old="PartName=\"/xl/workbook.xml\"", new="PartName=\"/xl/workbook2.xml\"", ) + let patched_root_rels = root_rels_xml.replace_all( + old="Target=\"xl/workbook.xml\"", + new="Target=\"xl/workbook2.xml\"", + ) let updated = @zip.Archive::new() for entry in archive.entries() { if entry.name() == "xl/workbook.xml" || entry.name() == "xl/_rels/workbook.xml.rels" || + entry.name() == "_rels/.rels" || entry.name() == "[Content_Types].xml" { continue } @@ -262,6 +274,7 @@ test "ooxml rels: workbook part path discovered from [Content_Types]" { } updated.add("xl/workbook2.xml", workbook_xml) updated.add("xl/_rels/workbook2.xml.rels", workbook_rels) + updated.add("_rels/.rels", @encoding/utf8.encode(patched_root_rels)) updated.add( "[Content_Types].xml", @encoding/utf8.encode(patched_content_types), From e74b771608038ebd7d67de0654624c3a49406400 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 00:57:21 +0800 Subject: [PATCH 024/150] fix(xlsx): propagate cancellation through reads --- office/cmd/office/docx_commands.mbt | 16 ++- office/cmd/office/main.mbt | 2 +- office/cmd/office/read_package.mbt | 10 +- office/cmd/office/xlsx_read_model.mbt | 31 +++++- office/cmd/office/xlsx_read_output.mbt | 1 + office/cmd/office/xlsx_read_wbtest.mbt | 22 +++- xlsx/cfb.mbt | 25 ++++- xlsx/encryption.mbt | 142 ++++++++++++++++++++++--- xlsx/pkg.generated.mbti | 2 +- xlsx/read.mbt | 31 ++++-- xlsx/read_cancel.mbt | 9 ++ zip/deflate.mbt | 39 ++++++- zip/errors.mbt | 8 ++ zip/pkg.generated.mbti | 3 +- zip/reader.mbt | 20 +++- 15 files changed, 317 insertions(+), 44 deletions(-) create mode 100644 xlsx/read_cancel.mbt diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index 497acd0a..d9d0760f 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -132,7 +132,9 @@ async fn run_docx_outline(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let max_elements = cli_max_elements(matches) let max_output = cli_max_output_chars(matches) - let source = read_office_package(file) + let source = read_office_package(file, cancelled=() => { + @async.is_being_cancelled() + }) match source.format { Docx => { let projection = open_docx_projection_archive( @@ -184,7 +186,9 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { let selector = required_value(matches, "selector") let max_elements = cli_max_elements(matches) let max_output = cli_max_output_chars(matches) - let source = read_office_package(file) + let source = read_office_package(file, cancelled=() => { + @async.is_being_cancelled() + }) match source.format { Docx => { let projection = open_docx_projection_archive( @@ -253,7 +257,9 @@ async fn run_docx_text(matches : @argparse.Matches) -> Unit { let limit = bounded_decimal_argument( matches, "limit", docx_cli_default_text_limit, 0, docx_cli_hard_text_limit, ) - let source = read_office_package(file) + let source = read_office_package(file, cancelled=() => { + @async.is_being_cancelled() + }) match source.format { Docx => { let projection = open_docx_projection_archive( @@ -519,7 +525,9 @@ async fn run_docx_query(matches : @argparse.Matches) -> Unit { let limit = bounded_decimal_argument( matches, "limit", docx_cli_default_query_limit, 0, docx_cli_hard_query_limit, ) - let source = read_office_package(file) + let source = read_office_package(file, cancelled=() => { + @async.is_being_cancelled() + }) match source.format { Docx => { reject_docx_content_selector(matches) diff --git a/office/cmd/office/main.mbt b/office/cmd/office/main.mbt index a769f215..49a64eb1 100644 --- a/office/cmd/office/main.mbt +++ b/office/cmd/office/main.mbt @@ -198,8 +198,8 @@ async fn main { } run_cli(args) } catch { - CliFailure(error) => raise CliError(render_failure(error, mode)) error if @async.is_being_cancelled() => raise error + CliFailure(error) => raise CliError(render_failure(error, mode)) error => { let message = normalize_argument_error(error.to_string()) raise CliError( diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 433ecc2d..7a4fc672 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -95,7 +95,10 @@ fn office_xlsx_read_limits() -> @xlsx.ReadLimits raise @xlsx.XlsxError { } ///| -async fn read_office_package(file : String) -> OfficeReadPackage { +async fn read_office_package( + file : String, + cancelled? : () -> Bool = () => false, +) -> OfficeReadPackage { let expected = office_read_format_hint(file) let data = read_bounded_file( file, @@ -111,7 +114,9 @@ async fn read_office_package(file : String) -> OfficeReadPackage { max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, max_total_uncompressed_bytes=office_read_max_zip_total_bytes, max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + cancelled~, ) catch { + error if cancelled() => raise error error => raise office_zip_read_failure(error, file, expected) } let format = @lib.detect_archive_format(file, archive) catch { @@ -166,7 +171,7 @@ fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { fn open_xlsx_read_package( source : OfficeReadPackage, cancelled? : () -> Bool = () => false, -) -> @xlsx.Workbook raise CliFailure { +) -> @xlsx.Workbook raise { if source.format is Docx { raise CliFailure( @lib.protocol_error( @@ -178,6 +183,7 @@ fn open_xlsx_read_package( error => raise xlsx_read_failure(error, source.file) } @xlsx.read_bounded_archive(source.archive, limits~, cancelled~) catch { + ReadCancelled as error => raise error error => raise xlsx_read_failure(error, source.file) } } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 3abbbeed..fae214bf 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -39,6 +39,7 @@ struct XlsxProjection { sheets : Array[XlsxSheetTarget] sheet_index : Map[String, Int] max_scan_cells : Int + cancelled : () -> Bool } ///| @@ -90,6 +91,7 @@ struct XlsxScanBudget { strings : XlsxStringBudget formatting : XlsxFormatBudget mut scanned : Int + cancelled : () -> Bool } ///| @@ -184,7 +186,11 @@ fn make_xlsx_projection( file : String, workbook : @xlsx.Workbook, max_elements : Int, + cancelled? : () -> Bool = () => false, ) -> XlsxProjection raise CliFailure { + if cancelled() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } if max_elements < 1 || max_elements > docx_cli_hard_max_elements { raise xlsx_cli_failure( "office.invalid_arguments", @@ -198,15 +204,24 @@ fn make_xlsx_projection( } let worksheets : Map[String, @xlsx.Worksheet] = Map([]) for worksheet in workbook.sheets() { + if cancelled() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } worksheets[worksheet.name()] = worksheet } let chart_sheets : Map[String, @xlsx.ChartSheet] = Map([]) for chart_sheet in workbook.chart_sheets() { + if cancelled() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } chart_sheets[chart_sheet.name()] = chart_sheet } let sheets : Array[XlsxSheetTarget] = [] let sheet_index : Map[String, Int] = Map([]) for index, name in workbook.get_sheet_list() { + if cancelled() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } let normalized_name = name.to_lower() if sheet_index.contains(normalized_name) { raise xlsx_cli_failure( @@ -247,6 +262,7 @@ fn make_xlsx_projection( sheets, sheet_index, max_scan_cells: max_elements.min(xlsx_cli_hard_max_scan_cells), + cancelled, } } @@ -255,9 +271,9 @@ fn open_xlsx_projection( source : OfficeReadPackage, max_elements : Int, cancelled? : () -> Bool = () => false, -) -> XlsxProjection raise CliFailure { +) -> XlsxProjection raise { let workbook = open_xlsx_read_package(source, cancelled~) - make_xlsx_projection(source.file, workbook, max_elements) + make_xlsx_projection(source.file, workbook, max_elements, cancelled~) } ///| @@ -401,10 +417,16 @@ fn XlsxSheetTarget::shared_formula_master( shared_index : Int, ) -> @xlsx.SharedFormulaMaster raise CliFailure { if !self.shared_formulas_indexed { + if (projection.cancelled)() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } let masters = xlsx_call(projection.file, () => { worksheet.shared_formula_masters() }) for index, formula in masters { + if (projection.cancelled)() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } self.shared_formula_masters[index] = formula } self.shared_formulas_indexed = true @@ -597,17 +619,22 @@ fn XlsxFormatBudget::charge_style( fn XlsxScanBudget::new( maximum : Int, format_work_maximum? : Int = xlsx_cli_max_format_work_units, + cancelled? : () -> Bool = () => false, ) -> XlsxScanBudget { { maximum, strings: XlsxStringBudget::new(), formatting: XlsxFormatBudget::new(maximum=format_work_maximum), scanned: 0, + cancelled, } } ///| fn XlsxScanBudget::charge_cell(self : XlsxScanBudget) -> Unit raise CliFailure { + if (self.cancelled)() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } if self.scanned >= self.maximum { raise xlsx_resource_failure( "scanned cells", diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 0d7de313..5d4e1d9b 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -314,6 +314,7 @@ fn collect_xlsx_cell_page( let scan = XlsxScanBudget::new( projection.max_scan_cells, format_work_maximum~, + cancelled=projection.cancelled, ) let cells : Array[XlsxCellSnapshot] = [] let mut matched = 0 diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index b98351c6..5e71b8e0 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -10,11 +10,10 @@ fn xlsx_read_test_projection() -> XlsxProjection raise { } ///| -fn xlsx_cli_error( - run : () -> Unit raise CliFailure, -) -> @lib.ProtocolError raise { +fn xlsx_cli_error(run : () -> Unit raise) -> @lib.ProtocolError raise { try run() catch { CliFailure(error) => error + _ => fail("expected XLSX CLI failure") } noraise { _ => fail("expected XLSX CLI failure") } @@ -36,6 +35,23 @@ fn xlsx_bounded_test_source( { file, format: Xlsx, archive } } +///| +test "XLSX CLI preserves parser cancellation and checkpoints cell scans" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Data")) + let source = xlsx_bounded_test_source("cancel.xlsx", @xlsx.write(workbook)) + let parse_result : Result[@xlsx.Workbook, Error] = Ok( + open_xlsx_read_package(source, cancelled=() => true), + ) catch { + error => Err(error) + } + assert_true(parse_result is Err(@xlsx.ReadCancelled)) + + let scan = XlsxScanBudget::new(10, cancelled=() => true) + let scan_error = xlsx_cli_error(() => scan.charge_cell()) + assert_eq(scan_error.code, "office.cancelled") +} + ///| fn xlsx_strict_test_bytes() -> Bytes raise { let archive = @zip.Archive::new() diff --git a/xlsx/cfb.mbt b/xlsx/cfb.mbt index b65c7676..7a514505 100644 --- a/xlsx/cfb.mbt +++ b/xlsx/cfb.mbt @@ -592,11 +592,14 @@ fn read_sector_chain( start : Int, sector_count? : Int = fat.length(), maximum_length? : Int = fat.length(), + cancelled? : () -> Bool = () => false, ) -> Array[Int] raise XlsxError { + check_read_cancelled(cancelled) let chain : Array[Int] = [] let visited = Array::make(sector_count, false) let mut current = start while current != end_of_chain && current != free_sector { + check_read_cancelled(cancelled) if current < 0 || current >= sector_count || current >= fat.length() { raise InvalidEncryptionInfo(msg="cfb chain out of bounds") } @@ -614,7 +617,11 @@ fn read_sector_chain( } ///| -fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { +fn cfb_read( + bytes : BytesView, + cancelled? : () -> Bool = () => false, +) -> CfbReader raise XlsxError { + check_read_cancelled(cancelled) if bytes.length() < 512 { raise InvalidEncryptionInfo(msg="cfb header too short") } @@ -689,6 +696,7 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { let mut difat_count = 0 let difat_seen = Array::make(sector_count, false) while difat_chain_start != end_of_chain && difat_chain_start != free_sector { + check_read_cancelled(cancelled) if difat_count >= num_difat { raise InvalidEncryptionInfo(msg="cfb DIFAT chain longer than declared") } @@ -733,6 +741,7 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { } let fat : Array[Int] = [] for sector in fat_sectors { + check_read_cancelled(cancelled) let offset = sector_offset(sector_size, sector) let fat_bytes = bytes[offset:offset + sector_size] let mut idx = 0 @@ -746,9 +755,11 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { dir_start, sector_count~, maximum_length=sector_count, + cancelled~, ) let dir_data : Array[Byte] = [] for sector in dir_chain { + check_read_cancelled(cancelled) let offset = sector_offset(sector_size, sector) let chunk = bytes[offset:offset + sector_size] dir_data.append(chunk.to_array()) @@ -757,6 +768,9 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { let mut root_entry : CfbEntry = { name: "", type_id: 0, start: 0, size: 0 } let mut offset = 0 while offset + 128 <= dir_data.length() { + if offset % 4096 == 0 { + check_read_cancelled(cancelled) + } let entry_bytes = Bytes::from_array(dir_data[offset:offset + 128]) let name_len = read_u16_le(entry_bytes, 0x40) if name_len > 64 || name_len % 2 != 0 { @@ -791,11 +805,13 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { mini_fat_start, sector_count~, maximum_length=num_mini_fat, + cancelled~, ) if mini_chain.length() != num_mini_fat { raise InvalidEncryptionInfo(msg="cfb mini FAT sector count mismatch") } for sector in mini_chain { + check_read_cancelled(cancelled) let offset = sector_offset(sector_size, sector) let chunk = bytes[offset:offset + sector_size] let mut idx = 0 @@ -813,9 +829,11 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { root_entry.start, sector_count~, maximum_length=root_sector_count, + cancelled~, ) let data : Array[Byte] = [] for sector in root_chain { + check_read_cancelled(cancelled) let offset = sector_offset(sector_size, sector) let chunk = bytes[offset:offset + sector_size] data.append(chunk.to_array()) @@ -843,7 +861,9 @@ fn cfb_read(bytes : BytesView) -> CfbReader raise XlsxError { fn CfbReader::read_stream( self : CfbReader, name : StringView, + cancelled? : () -> Bool = () => false, ) -> Bytes raise XlsxError { + check_read_cancelled(cancelled) let mut entry : CfbEntry? = None let target = name.to_owned() for item in self.entries { @@ -870,6 +890,7 @@ fn CfbReader::read_stream( let mut remaining = item.size let mut guard_count = 0 while current != end_of_chain && current != free_sector && remaining > 0 { + check_read_cancelled(cancelled) if current < 0 || current >= self.mini_fat.length() { raise InvalidEncryptionInfo(msg="cfb mini chain out of bounds") } @@ -907,9 +928,11 @@ fn CfbReader::read_stream( item.start, sector_count~, maximum_length=stream_sector_count, + cancelled~, ) let data : Array[Byte] = [] for sector in chain { + check_read_cancelled(cancelled) let offset = sector_offset(self.sector_size, sector) let chunk = self.bytes[offset:offset + self.sector_size] data.append(chunk.to_array()) diff --git a/xlsx/encryption.mbt b/xlsx/encryption.mbt index ade9a9a7..8efc3d71 100644 --- a/xlsx/encryption.mbt +++ b/xlsx/encryption.mbt @@ -165,6 +165,27 @@ fn aes_cbc_decrypt( } } +///| +fn aes_ecb_decrypt_cancellable( + key : BytesView, + data : BytesView, + cancelled : () -> Bool, +) -> Bytes raise XlsxError { + if data.length() % 16 != 0 { + raise InvalidEncryptionInfo(msg="invalid AES data length") + } + let out : Array[Byte] = Array::new(capacity=data.length()) + let mut offset = 0 + while offset < data.length() { + check_read_cancelled(cancelled) + let end = (offset + 4096).min(data.length()) + out.append(aes_ecb_decrypt(key, data[offset:end]).to_array()) + offset = end + } + check_read_cancelled(cancelled) + Bytes::from_array(out) +} + ///| fn byte_xor(a : Byte, b : Byte) -> Byte { (a.to_int() ^ b.to_int()).to_byte() @@ -463,6 +484,7 @@ fn encryption_mechanism( fn derive_password_hash( password : String, encryption : EncryptionInfo, + cancelled? : () -> Bool = () => false, ) -> Bytes raise XlsxError { let key_data = encryption.encrypted_key.key_data let salt_value = key_data.salt_value @@ -473,9 +495,13 @@ fn derive_password_hash( ) let mut key = hash_parts(key_data.hash_algorithm, [salt_value, password_bytes]) for i in 0.. Bool = () => false, ) -> Bytes raise XlsxError { + check_read_cancelled(cancelled) let block_size = encryption.key_data.block_size if input.length() <= package_offset || (input.length() - package_offset) % block_size != 0 { @@ -582,6 +610,7 @@ fn decrypt_package( let mut offset = 0 let mut index = 0 while offset < payload.length() { + check_read_cancelled(cancelled) let end = if offset + package_encryption_chunk_size < payload.length() { offset + package_encryption_chunk_size } else { @@ -595,6 +624,7 @@ fn decrypt_package( offset = end index = index + 1 } + check_read_cancelled(cancelled) let raw_size_u64 = @crypto.u64_from_le(input, 0) if raw_size_u64 > out.length().to_uint64() { raise InvalidEncryptedPackage(msg="decrypted package size is invalid") @@ -609,7 +639,9 @@ fn agile_decrypt( encrypted_package_buf : BytesView, password : String, limits : ReadLimits, + cancelled? : () -> Bool = () => false, ) -> DecryptedPackage raise XlsxError { + check_read_cancelled(cancelled) if encryption_info_buf.length() > limits.max_xml_part_bytes { raise ResourceLimitExceeded( kind="xml_part_bytes", @@ -626,7 +658,7 @@ fn agile_decrypt( !is_aes_cipher(encryption.encrypted_key.key_data) { raise UnsupportedEncryption(msg="unsupported cipher settings") } - let password_hash = derive_password_hash(password, encryption) + let password_hash = derive_password_hash(password, encryption, cancelled~) let key = derive_password_key_from_hash(password_hash, block_key, encryption) let password_ok = agile_verify_password(password_hash, encryption) let salt = encryption.encrypted_key.key_data.salt_value @@ -636,7 +668,10 @@ fn agile_decrypt( encryption.encrypted_key.encrypted_key_value, ) let decrypted = decrypt_package( - package_key, encrypted_package_buf, encryption, + package_key, + encrypted_package_buf, + encryption, + cancelled~, ) { bytes: decrypted, password_verified: password_ok } } @@ -646,6 +681,7 @@ fn standard_convert_password_to_key( key_bits : Int, salt : BytesView, password : String, + cancelled? : () -> Bool = () => false, ) -> Bytes raise XlsxError { let password_bytes = @encoding/utf16.encode( password, @@ -654,9 +690,13 @@ fn standard_convert_password_to_key( ) let mut key = hash_parts("sha1", [salt, password_bytes]) for i in 0.. Bool = () => false, ) -> DecryptedPackage raise XlsxError { + check_read_cancelled(cancelled) if encryption_info_buf.length() < 12 { raise InvalidEncryptionInfo(msg="standard encryption header too short") } @@ -756,7 +798,12 @@ fn standard_decrypt( ) } let salt = verifier_block[4:4 + 16] - let secret_key = standard_convert_password_to_key(key_bits, salt, password) + let secret_key = standard_convert_password_to_key( + key_bits, + salt, + password, + cancelled~, + ) let password_verified = standard_password_verified( secret_key, verifier_block, algorithm, ) @@ -764,7 +811,11 @@ fn standard_decrypt( (encrypted_package_buf.length() - 8) % 16 != 0 { raise InvalidEncryptedPackage(msg="invalid encrypted package length") } - let decrypted = aes_ecb_decrypt(secret_key, encrypted_package_buf[8:]) + let decrypted = aes_ecb_decrypt_cancellable( + secret_key, + encrypted_package_buf[8:], + cancelled, + ) let raw_size_u64 = @crypto.u64_from_le(encrypted_package_buf, 0) if raw_size_u64 > decrypted.length().to_uint64() { if password_verified { @@ -794,18 +845,23 @@ fn decrypt_encrypted_package_result( bytes : BytesView, password : String, limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, ) -> DecryptedPackage raise XlsxError { + check_read_cancelled(cancelled) check_source_package_size(bytes, limits) if password.length() > max_password_length { raise InvalidPasswordLength(len=password.length()) } - let reader = cfb_read(bytes) catch { + let reader = cfb_read(bytes, cancelled~) catch { + ReadCancelled => raise ReadCancelled _ => raise InvalidEncryptedPackage(msg="invalid cfb package") } - let encryption_info = reader.read_stream("EncryptionInfo") catch { + let encryption_info = reader.read_stream("EncryptionInfo", cancelled~) catch { + ReadCancelled => raise ReadCancelled _ => raise InvalidEncryptedPackage(msg="encryption info missing") } - let encrypted_package = reader.read_stream("EncryptedPackage") catch { + let encrypted_package = reader.read_stream("EncryptedPackage", cancelled~) catch { + ReadCancelled => raise ReadCancelled _ => raise InvalidEncryptedPackage(msg="encrypted package missing") } if encryption_info.length() < 8 { @@ -816,10 +872,23 @@ fn decrypt_encrypted_package_result( } let decrypted = match encryption_mechanism(encryption_info) { Agile => - agile_decrypt(encryption_info[8:], encrypted_package, password, limits) + agile_decrypt( + encryption_info[8:], + encrypted_package, + password, + limits, + cancelled~, + ) Standard => - standard_decrypt(encryption_info, encrypted_package, password, limits~) + standard_decrypt( + encryption_info, + encrypted_package, + password, + limits~, + cancelled~, + ) } + check_read_cancelled(cancelled) check_source_package_size(decrypted.bytes, limits) decrypted } @@ -828,8 +897,9 @@ fn decrypt_encrypted_package_result( fn decrypted_package_archive( decrypted : DecryptedPackage, limits : ReadLimits, + cancelled? : () -> Bool = () => false, ) -> @zip.Archive raise XlsxError { - read_limited_archive(decrypted.bytes, limits) catch { + read_limited_archive(decrypted.bytes, limits, cancelled~) catch { ResourceLimitExceeded(kind~, limit~, actual~) => raise ResourceLimitExceeded(kind~, limit~, actual~) InvalidPackage(msg~) => @@ -847,10 +917,12 @@ fn decrypt_encrypted_archive( bytes : BytesView, password : String, limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, ) -> @zip.Archive raise XlsxError { decrypted_package_archive( - decrypt_encrypted_package_result(bytes, password, limits~), + decrypt_encrypted_package_result(bytes, password, limits~, cancelled~), limits, + cancelled~, ) } @@ -859,12 +931,18 @@ fn decrypt_encrypted_package( bytes : BytesView, password : String, limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Bytes raise XlsxError { - let decrypted = decrypt_encrypted_package_result(bytes, password, limits~) + let decrypted = decrypt_encrypted_package_result( + bytes, + password, + limits~, + cancelled~, + ) if !decrypted.password_verified { // Parse at most once to distinguish a tampered/legacy verifier from a // genuinely wrong password. Workbook reads reuse this archive directly. - ignore(decrypted_package_archive(decrypted, limits)) + ignore(decrypted_package_archive(decrypted, limits, cancelled~)) } decrypted.bytes } @@ -940,8 +1018,9 @@ pub fn decrypt( raw : BytesView, options? : Options = Options::new(), limits? : ReadLimits = ReadLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Bytes raise XlsxError { - decrypt_encrypted_package(raw, options.password, limits~) + decrypt_encrypted_package(raw, options.password, limits~, cancelled~) } ///| @@ -1569,6 +1648,41 @@ test "agile verify password" { inspect(agile_verify_password(wrong_hash, encryption), content="false") } +///| +test "encrypted read primitives observe cancellation during KDF and AES work" { + let bytes = decode_base64_multiline(encrypt_sha1_b64) + let reader = cfb_read(bytes) + let info = reader.read_stream("EncryptionInfo") + let encryption = parse_encryption_info(@encoding/utf8.decode(info[8:])) + let kdf_checks = [0] + let kdf_result : Result[Bytes, Error] = Ok( + derive_password_hash("password", encryption, cancelled=() => { + kdf_checks[0] += 1 + kdf_checks[0] >= 3 + }), + ) catch { + error => Err(error) + } + assert_true(kdf_result is Err(ReadCancelled)) + assert_true(kdf_checks[0] >= 3) + + let aes_checks = [0] + let aes_result : Result[Bytes, Error] = Ok( + aes_ecb_decrypt_cancellable( + Bytes::make(16, b'k'), + Bytes::make(16 * 1024, b'v'), + () => { + aes_checks[0] += 1 + aes_checks[0] >= 3 + }, + ), + ) catch { + error => Err(error) + } + assert_true(aes_result is Err(ReadCancelled)) + assert_true(aes_checks[0] >= 3) +} + ///| test "encryption KDF work and AES key sizes are validated before derivation" { let bytes = decode_base64_multiline(encrypt_sha1_b64) diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 38489e4b..1958e20c 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -17,7 +17,7 @@ pub fn column_number_to_name(Int) -> String raise XlsxError pub fn coordinates_to_cell_name(Int, Int, abs? : Bool) -> String raise XlsxError -pub fn decrypt(BytesView, options? : Options, limits? : ReadLimits) -> Bytes raise XlsxError +pub fn decrypt(BytesView, options? : Options, limits? : ReadLimits, cancelled? : () -> Bool) -> Bytes raise XlsxError pub fn encrypt(BytesView, options? : Options) -> Bytes raise XlsxError diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 2a593b2e..b183fcf3 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2959,9 +2959,12 @@ fn is_xml_package_part(name : StringView) -> Bool { fn enforce_archive_limits( archive : @zip.Archive, limits : ReadLimits, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { + check_read_cancelled(cancelled) let mut total_size = 0 for entry in archive.entries() { + check_read_cancelled(cancelled) let size = entry.data().length() if size > limits.max_entry_uncompressed_bytes { raise ResourceLimitExceeded( @@ -2988,6 +2991,7 @@ fn enforce_archive_limits( if @zip.crc32(entry.data()) != entry.crc32() { raise InvalidPackage(msg="ZIP entry CRC mismatch") } + check_read_cancelled(cancelled) } } @@ -2995,7 +2999,9 @@ fn enforce_archive_limits( fn require_bounded_archive( archive : @zip.Archive, limits : ReadLimits, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { + check_read_cancelled(cancelled) match archive.bounded_source_package_size( max_entries=limits.max_archive_entries, @@ -3020,14 +3026,16 @@ fn require_bounded_archive( actual=archive.entries().length(), ) } - enforce_archive_limits(archive, limits) + enforce_archive_limits(archive, limits, cancelled~) } ///| fn read_limited_archive( bytes : BytesView, limits : ReadLimits, + cancelled? : () -> Bool = () => false, ) -> @zip.Archive raise XlsxError { + check_read_cancelled(cancelled) @zip.read_limited( bytes, max_package_bytes=limits.max_package_bytes, @@ -3037,9 +3045,11 @@ fn read_limited_archive( // The ZIP package preserves exact source records for lossless rewrites. // Bound that second compressed representation independently from inflation. max_total_preserved_source_bytes=limits.max_total_preserved_source_bytes, + cancelled~, ) catch { ResourceLimitExceeded(kind~, limit~, actual~) => raise ResourceLimitExceeded(kind~, limit~, actual~) + ReadCancelled => raise ReadCancelled _ => raise InvalidPackage(msg="unreadable ZIP package") } } @@ -4150,8 +4160,9 @@ fn read_zip_bytes( cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { - let archive = read_limited_archive(bytes, limits) - require_bounded_archive(archive, limits) + check_read_cancelled(cancelled) + let archive = read_limited_archive(bytes, limits, cancelled~) + require_bounded_archive(archive, limits, cancelled~) match transcoder { Some(value) => read_zip_archive_core( @@ -4177,7 +4188,8 @@ pub fn read_bounded_archive( cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { - require_bounded_archive(archive, limits) + check_read_cancelled(cancelled) + require_bounded_archive(archive, limits, cancelled~) match transcoder { Some(value) => read_zip_archive_core( @@ -4204,6 +4216,7 @@ pub fn read( cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { + check_read_cancelled(cancelled) check_source_package_size(bytes, limits) if options.password != "" { match transcoder { @@ -4249,11 +4262,17 @@ pub fn read_with_password( cancelled? : () -> Bool = () => false, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { + check_read_cancelled(cancelled) check_source_package_size(bytes, limits) let resolved_options = options_with_password(options, password) if is_encrypted_package(bytes) { - let archive = decrypt_encrypted_archive(bytes, password, limits~) - require_bounded_archive(archive, limits) + let archive = decrypt_encrypted_archive( + bytes, + password, + limits~, + cancelled~, + ) + require_bounded_archive(archive, limits, cancelled~) match transcoder { Some(value) => read_zip_archive_core( diff --git a/xlsx/read_cancel.mbt b/xlsx/read_cancel.mbt new file mode 100644 index 00000000..e86a2864 --- /dev/null +++ b/xlsx/read_cancel.mbt @@ -0,0 +1,9 @@ +///| +/// Raises the stable XLSX cancellation error at synchronous work boundaries. +/// Callers supply a cheap task-state probe; no runtime- or platform-specific +/// stubs are required. +fn check_read_cancelled(cancelled : () -> Bool) -> Unit raise XlsxError { + if cancelled() { + raise ReadCancelled + } +} diff --git a/zip/deflate.mbt b/zip/deflate.mbt index 8e9cb08e..9c87dd21 100644 --- a/zip/deflate.mbt +++ b/zip/deflate.mbt @@ -411,11 +411,17 @@ fn decode_huffman_block( lit_table : HuffmanTable, dist_table : HuffmanTable, out : FixedByteOutput, + cancelled? : () -> Bool = () => false, ) -> Unit raise ZipError { + let mut next_checkpoint = out.length() for state = true { if !state { break } + if out.length() >= next_checkpoint { + check_zip_cancelled(cancelled) + next_checkpoint = out.length() + 4096 + } let symbol = decode_symbol(reader, lit_table) if symbol < 256 { out.write_byte(symbol.to_byte()) @@ -594,7 +600,9 @@ fn deflate_encode(bytes : BytesView) -> Bytes raise ZipError { fn deflate_decode_to_output( bytes : BytesView, out : FixedByteOutput, + cancelled : () -> Bool, ) -> Unit raise ZipError { + check_zip_cancelled(cancelled) let reader = BitReader::new(bytes) for state = false { if state { @@ -610,18 +618,21 @@ fn deflate_decode_to_output( if (len.reinterpret_as_uint() ^ nlen) != 0xFFFF { raise UnsupportedFeature(msg="invalid stored block length") } - for _ in 0.. { let lit = fixed_literal_table() let dist = fixed_distance_table() - decode_huffman_block(reader, lit, dist, out) + decode_huffman_block(reader, lit, dist, out, cancelled~) } 2 => { let (lit, dist) = read_dynamic_tables(reader) - decode_huffman_block(reader, lit, dist, out) + decode_huffman_block(reader, lit, dist, out, cancelled~) } _ => raise UnsupportedFeature(msg="invalid deflate block type") } @@ -640,12 +651,13 @@ fn deflate_decode_to_output( fn deflate_decode( bytes : BytesView, max_output? : Int = deflate_max_output_default, + cancelled? : () -> Bool = () => false, ) -> Bytes raise ZipError { let sizing = FixedByteOutput::counting(limit=max_output) - deflate_decode_to_output(bytes, sizing) + deflate_decode_to_output(bytes, sizing, cancelled) let exact_size = sizing.length() let out = FixedByteOutput::allocated(exact_size) - deflate_decode_to_output(bytes, out) + deflate_decode_to_output(bytes, out, cancelled) out.finish() } @@ -705,6 +717,23 @@ test "deflate dynamic huffman" { inspect(deflate_decode(stream) == expected, content="true") } +///| +test "deflate decoding observes cancellation while producing output" { + let input = Bytes::from_array(Array::make(32 * 1024, b'x')) + let compressed = deflate_encode(input) + let checks = [0] + let result : Result[Bytes, Error] = Ok( + deflate_decode(compressed, cancelled=() => { + checks[0] += 1 + checks[0] >= 3 + }), + ) catch { + error => Err(error) + } + assert_true(result is Err(ReadCancelled)) + assert_true(checks[0] >= 3) +} + ///| test "deflate encode fixed literals" { let data : Bytes = b"MoonBit deflate" diff --git a/zip/errors.mbt b/zip/errors.mbt index 2b04b390..41e96332 100644 --- a/zip/errors.mbt +++ b/zip/errors.mbt @@ -14,4 +14,12 @@ pub suberror ZipError { InvalidUtf8(offset~ : Int) OutputLimitExceeded(limit~ : Int) ResourceLimitExceeded(kind~ : String, limit~ : Int, actual~ : Int) + ReadCancelled } derive(Debug) + +///| +fn check_zip_cancelled(cancelled : () -> Bool) -> Unit raise ZipError { + if cancelled() { + raise ReadCancelled + } +} diff --git a/zip/pkg.generated.mbti b/zip/pkg.generated.mbti index 5b6a3feb..d91d11bc 100644 --- a/zip/pkg.generated.mbti +++ b/zip/pkg.generated.mbti @@ -14,7 +14,7 @@ pub fn gzip(BytesView) -> Bytes raise ZipError pub fn read(BytesView) -> Archive raise ZipError -pub fn read_limited(BytesView, max_package_bytes~ : Int, max_entries~ : Int, max_entry_uncompressed_bytes~ : Int, max_total_uncompressed_bytes~ : Int, max_total_preserved_source_bytes~ : Int) -> Archive raise ZipError +pub fn read_limited(BytesView, max_package_bytes~ : Int, max_entries~ : Int, max_entry_uncompressed_bytes~ : Int, max_total_uncompressed_bytes~ : Int, max_total_preserved_source_bytes~ : Int, cancelled? : () -> Bool) -> Archive raise ZipError pub fn write(Archive) -> Bytes raise ZipError @@ -30,6 +30,7 @@ pub suberror ZipError { InvalidUtf8(offset~ : Int) OutputLimitExceeded(limit~ : Int) ResourceLimitExceeded(kind~ : String, limit~ : Int, actual~ : Int) + ReadCancelled } derive(@debug.Debug) #deprecated pub fn ZipError::to_repr(Self) -> @debug.Repr diff --git a/zip/reader.mbt b/zip/reader.mbt index 864f8d4a..ee35c348 100644 --- a/zip/reader.mbt +++ b/zip/reader.mbt @@ -718,7 +718,9 @@ fn materialize_entry_data( limit_kind : String, reported_limit : Int, actual_base : Int, + cancelled : () -> Bool, ) -> Bytes raise ZipError { + check_zip_cancelled(cancelled) let data = match record.compression { Store => { match max_uncompressed_bytes { @@ -747,7 +749,7 @@ fn materialize_entry_data( deflate_max_output_default } } - deflate_decode(record.compressed, max_output~) catch { + deflate_decode(record.compressed, max_output~, cancelled~) catch { OutputLimitExceeded(limit~) => match max_uncompressed_bytes { Some(_) => @@ -781,7 +783,9 @@ fn read_impl( max_entry_uncompressed_bytes : Int?, max_total_uncompressed_bytes : Int?, max_total_preserved_source_bytes : Int?, + cancelled : () -> Bool, ) -> Archive raise ZipError { + check_zip_cancelled(cancelled) match max_package_bytes { Some(limit) if bytes.length() > limit => raise ResourceLimitExceeded( @@ -905,7 +909,10 @@ fn read_impl( let central_bytes = bytes[central_offset:central_boundary] let central_reader = Reader::new(central_bytes, 0) let mut declared_total = 0 - for _ in 0.. () } let data = materialize_entry_data( - record, meta, materialize_limit, limit_kind, reported_limit, actual_base, + record, meta, materialize_limit, limit_kind, reported_limit, actual_base, cancelled, ) match max_total_uncompressed_bytes { Some(limit) if data.length() > limit - actual_total => @@ -1194,7 +1204,7 @@ fn read_impl( ///| /// Reads a ZIP archive using the library's compatibility defaults. pub fn read(bytes : BytesView) -> Archive raise ZipError { - read_impl(bytes, None, None, None, None, None) + read_impl(bytes, None, None, None, None, None, () => false) } ///| @@ -1208,6 +1218,7 @@ pub fn read_limited( max_entry_uncompressed_bytes~ : Int, max_total_uncompressed_bytes~ : Int, max_total_preserved_source_bytes~ : Int, + cancelled? : () -> Bool = () => false, ) -> Archive raise ZipError { if max_package_bytes < 0 || max_entries < 0 || @@ -1223,6 +1234,7 @@ pub fn read_limited( Some(max_entry_uncompressed_bytes), Some(max_total_uncompressed_bytes), Some(max_total_preserved_source_bytes), + cancelled, ) } From 6d42fdcda04025bd811e0519159022d739296284 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 01:20:15 +0800 Subject: [PATCH 025/150] fix(xlsx): validate shared formula groups --- office/cmd/office/xlsx_read_model.mbt | 4 +- xlsx/iterators_excelize_parity_test.mbt | 21 ++- xlsx/pkg.generated.mbti | 6 +- xlsx/read_worksheet_xml.mbt | 9 +- xlsx/shared_formula_ranges.mbt | 189 ++++++++++++++++++++++++ xlsx/shared_formula_validation_test.mbt | 98 ++++++++++++ xlsx/worksheet.mbt | 87 +++++------ xlsx/worksheet_types.mbt | 13 +- 8 files changed, 371 insertions(+), 56 deletions(-) create mode 100644 xlsx/shared_formula_ranges.mbt diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index fae214bf..fc1a81ce 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -28,7 +28,7 @@ struct XlsxSheetTarget { index : Int path : String content : XlsxSheetContent - shared_formula_masters : Map[Int, @xlsx.SharedFormulaMaster] + shared_formula_masters : Map[UInt, @xlsx.SharedFormulaMaster] mut shared_formulas_indexed : Bool } @@ -414,7 +414,7 @@ fn XlsxSheetTarget::shared_formula_master( self : XlsxSheetTarget, projection : XlsxProjection, worksheet : @xlsx.Worksheet, - shared_index : Int, + shared_index : UInt, ) -> @xlsx.SharedFormulaMaster raise CliFailure { if !self.shared_formulas_indexed { if (projection.cancelled)() { diff --git a/xlsx/iterators_excelize_parity_test.mbt b/xlsx/iterators_excelize_parity_test.mbt index 445d5f59..d900017a 100644 --- a/xlsx/iterators_excelize_parity_test.mbt +++ b/xlsx/iterators_excelize_parity_test.mbt @@ -88,7 +88,26 @@ test "rows/cols/get_rows/get_cols validate sheet name" { #cfg(target="native") async test "excelize Book1 iterators match get_rows/get_cols" { let path = "fixtures/excelize/test/Book1.xlsx" - let workbook = @xlsx.open_file(path) + let archive = @zip.read(@async/fs.read_file(path).binary()) + let sheet_path = "xl/worksheets/sheet2.xml" + let sheet_xml = match archive.get(sheet_path) { + Some(bytes) => @encoding/utf8.decode(bytes) + None => fail("missing Book1 worksheet") + } + // Book1 predates strict shared-formula validation and contains intersecting + // si=0/si=1 ranges. Normalize that unrelated defect in memory so this test + // continues to exercise the original third-party iterator fixture. + let normalized_sheet_xml = sheet_xml + .replace(old="ref=\"F11:H11\"", new="ref=\"F11:F11\"") + .replace( + old="", + new="", + ) + assert_true(normalized_sheet_xml != sheet_xml) + assert_true( + archive.replace(sheet_path, @encoding/utf8.encode(normalized_sheet_xml)), + ) + let workbook = @xlsx.read(@zip.write(archive)) let total = workbook.get_cell_value("Sheet1", "A19") catch { err => fail("get_cell_value failed: \{repr(err)}") } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 1958e20c..8d631c15 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -229,7 +229,7 @@ pub struct Cell { formula : String? formula_type : FormulaType? formula_ref : String? - formula_shared_index : Int? + formula_shared_index : UInt? formula_value_present : Bool style_id : Int } @@ -238,7 +238,7 @@ pub struct CellFormulaInfo { formula : String formula_type : FormulaType? range_ref : String? - shared_index : Int? + shared_index : UInt? cached_value_present : Bool } @@ -2179,7 +2179,7 @@ pub fn Worksheet::set_row_style(Self, Int, Int) -> Unit raise XlsxError pub fn Worksheet::set_row_visible(Self, Int, Bool) -> Unit raise XlsxError pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxError pub fn Worksheet::shapes(Self) -> ArrayView[Shape] -pub fn Worksheet::shared_formula_masters(Self) -> Map[Int, SharedFormulaMaster] raise XlsxError +pub fn Worksheet::shared_formula_masters(Self) -> Map[UInt, SharedFormulaMaster] raise XlsxError pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? pub fn Worksheet::slicers(Self) -> ArrayView[Slicer] diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index cf46de49..c82565a1 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -98,7 +98,7 @@ fn parse_worksheet( let si_value = match attr_value(open_tag, "si") { Some(value) => Some( - @string.parse_int(value, base=10) catch { + @string.parse_uint(value, base=10) catch { _ => raise InvalidXml(msg="formula si invalid") }, ) @@ -118,6 +118,13 @@ fn parse_worksheet( } None => (None, None, None, None) } + // ECMA-376 defines the expression on non-master shared formulas as + // ignored. Canonicalize followers immediately so downstream consumers do + // not accidentally evaluate producer-specific garbage from that node. + let formula = match (formula_type, formula_ref, formula) { + (Some(Shared), None, Some(_)) => Some("") + _ => formula + } let raw_value_opt = match find_xml_open_tag_start(rest, "v") { Some(pos) => { let v_rest = rest[pos:] diff --git a/xlsx/shared_formula_ranges.mbt b/xlsx/shared_formula_ranges.mbt new file mode 100644 index 00000000..4aa04a71 --- /dev/null +++ b/xlsx/shared_formula_ranges.mbt @@ -0,0 +1,189 @@ +///| +priv struct SharedFormulaRangeEvent { + row : Int + column_lo : Int + column_hi : Int + delta : Int +} + +///| +fn shared_formula_range_tree_push( + maximums : Array[Int], + pending_additions : Array[Int], + node : Int, +) -> Unit { + let pending = pending_additions[node] + if pending == 0 { + return + } + let left = node * 2 + let right = left + 1 + maximums[left] += pending + maximums[right] += pending + pending_additions[left] += pending + pending_additions[right] += pending + pending_additions[node] = 0 +} + +///| +fn shared_formula_range_tree_add( + maximums : Array[Int], + pending_additions : Array[Int], + node : Int, + lo : Int, + hi : Int, + query_lo : Int, + query_hi : Int, + delta : Int, +) -> Unit { + if query_lo <= lo && hi <= query_hi { + maximums[node] += delta + pending_additions[node] += delta + return + } + shared_formula_range_tree_push(maximums, pending_additions, node) + let middle = lo + (hi - lo) / 2 + if query_lo <= middle { + shared_formula_range_tree_add( + maximums, + pending_additions, + node * 2, + lo, + middle, + query_lo, + query_hi, + delta, + ) + } + if query_hi > middle { + shared_formula_range_tree_add( + maximums, + pending_additions, + node * 2 + 1, + middle + 1, + hi, + query_lo, + query_hi, + delta, + ) + } + let left_maximum = maximums[node * 2] + let right_maximum = maximums[node * 2 + 1] + maximums[node] = if left_maximum > right_maximum { + left_maximum + } else { + right_maximum + } +} + +///| +fn shared_formula_range_tree_maximum( + maximums : Array[Int], + pending_additions : Array[Int], + node : Int, + lo : Int, + hi : Int, + query_lo : Int, + query_hi : Int, +) -> Int { + if query_lo <= lo && hi <= query_hi { + return maximums[node] + } + shared_formula_range_tree_push(maximums, pending_additions, node) + let middle = lo + (hi - lo) / 2 + let mut result = 0 + if query_lo <= middle { + result = shared_formula_range_tree_maximum( + maximums, + pending_additions, + node * 2, + lo, + middle, + query_lo, + query_hi, + ) + } + if query_hi > middle { + let right = shared_formula_range_tree_maximum( + maximums, + pending_additions, + node * 2 + 1, + middle + 1, + hi, + query_lo, + query_hi, + ) + if right > result { + result = right + } + } + result +} + +///| +/// Validates two-dimensional master rectangles in O(m log columns) time. End +/// events sort before starts on the following row, so touching but disjoint +/// ranges remain valid. The column tree has a fixed 16,384-column footprint. +fn validate_shared_formula_master_ranges( + masters : Map[UInt, SharedFormulaMaster], +) -> Unit raise XlsxError { + if masters.length() < 2 { + return + } + let events : Array[SharedFormulaRangeEvent] = [] + for _, master in masters { + events.push({ + row: master.row_lo, + column_lo: master.column_lo, + column_hi: master.column_hi, + delta: 1, + }) + events.push({ + row: master.row_hi + 1, + column_lo: master.column_lo, + column_hi: master.column_hi, + delta: -1, + }) + } + events.sort_by((left, right) => { + if left.row != right.row { + left.row.compare(right.row) + } else if left.delta != right.delta { + // Remove rectangles that ended on the preceding row before inserting + // rectangles that begin on this row. + left.delta.compare(right.delta) + } else if left.column_lo != right.column_lo { + left.column_lo.compare(right.column_lo) + } else { + left.column_hi.compare(right.column_hi) + } + }) + let tree_size = cell_ref_max_cols * 4 + 8 + let maximums = Array::make(tree_size, 0) + let pending_additions = Array::make(tree_size, 0) + for event in events { + if event.delta > 0 && + shared_formula_range_tree_maximum( + maximums, + pending_additions, + 1, + 1, + cell_ref_max_cols, + event.column_lo, + event.column_hi, + ) > + 0 { + raise InvalidXml(msg="shared formula master ranges overlap") + } + shared_formula_range_tree_add( + maximums, + pending_additions, + 1, + 1, + cell_ref_max_cols, + event.column_lo, + event.column_hi, + event.delta, + ) + } +} diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt index 08ee8523..3a7d987e 100644 --- a/xlsx/shared_formula_validation_test.mbt +++ b/xlsx/shared_formula_validation_test.mbt @@ -69,3 +69,101 @@ test "XLSX read rejects missing, out-of-range, and conflicting shared masters" { "shared formula index has conflicting masters", ) } + +///| +test "XLSX read ignores expressions stored on shared-formula followers" { + let bytes = malformed_shared_formula_package( + "", "PRODUCER_GARBAGE+1", + ) + let workbook = @xlsx.read(bytes) + guard workbook.sheet("Data") is Some(sheet) else { + fail("missing shared-formula worksheet") + } + guard sheet.get_cell_formula_info_rc(1, 2) is Some(follower) else { + fail("missing shared-formula follower metadata") + } + assert_eq(follower.formula, "") + assert_eq(follower.shared_index, Some(0)) + guard sheet.shared_formula_masters().get(0) is Some(master) else { + fail("missing shared-formula master") + } + assert_eq(master.translate_to(1, 2), "E1+1") +} + +///| +test "XLSX read rejects overlapping shared-formula master ranges" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "B1+1", + opts=@xlsx.FormulaOpts::shared("A1:A2"), + ) + sheet.set_cell_formula_opts( + "C1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("C1:C2"), + ) + let archive = @zip.read(@xlsx.write(workbook)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("shared-formula worksheet is not UTF-8") + } + None => fail("missing shared-formula worksheet") + } + let updated = xml.replace(old="ref=\"A1:A2\"", new="ref=\"A1:C1\"") + assert_true(updated != xml) + assert_true(archive.replace(path, @encoding/utf8.encode(updated))) + expect_shared_formula_read_error( + "overlapping masters", + @zip.write(archive), + "shared formula master ranges overlap", + ) +} + +///| +test "XLSX shared-formula indices cover the OOXML unsigned range" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("A1:C1"), + ) + let archive = @zip.read(@xlsx.write(workbook)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("shared-formula worksheet is not UTF-8") + } + None => fail("missing shared-formula worksheet") + } + let updated = xml.replace_all(old="si=\"0\"", new="si=\"4294967295\"") + assert_true(updated != xml) + assert_true(archive.replace(path, @encoding/utf8.encode(updated))) + let parsed = @xlsx.read(@zip.write(archive)) + guard parsed.sheet("Data") is Some(parsed_sheet) else { + fail("missing parsed shared-formula worksheet") + } + guard parsed_sheet.shared_formula_masters().get(0xFFFFFFFF) is Some(master) else { + fail("missing maximum unsigned shared-formula index") + } + assert_eq(master.translate_to(1, 3), "F1+1") + let roundtripped = @xlsx.read(@xlsx.write(parsed)) + guard roundtripped.sheet("Data") is Some(roundtripped_sheet) else { + fail("missing roundtripped shared-formula worksheet") + } + assert_true(roundtripped_sheet.shared_formula_masters().contains(0xFFFFFFFF)) +} + +///| +test "XLSX read rejects negative shared-formula indices" { + expect_shared_formula_read_error( + "negative index", + malformed_shared_formula_package("si=\"0\"", "si=\"-1\""), + "formula si invalid", + ) +} diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 6df57d90..cc85d872 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -390,14 +390,9 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { /// are retained only once even for large shared ranges. fn validated_shared_formula_masters( cells : ArrayView[Cell], -) -> Map[Int, SharedFormulaMaster] raise XlsxError { - let masters : Map[Int, SharedFormulaMaster] = Map([]) +) -> Map[UInt, SharedFormulaMaster] raise XlsxError { + let masters : Map[UInt, SharedFormulaMaster] = Map([]) for cell in cells { - match (cell.formula_type, cell.formula_shared_index) { - (Some(Shared), Some(shared_index)) if shared_index < 0 => - raise InvalidXml(msg="shared formula index is negative") - _ => () - } match ( cell.formula_type, @@ -405,36 +400,32 @@ fn validated_shared_formula_masters( cell.formula_shared_index, cell.formula_ref, ) { - (Some(Shared), Some(formula), Some(shared_index), Some(range_ref)) => - if formula != "" { - let (row_lo, column_lo, row_hi, column_hi) = parse_range_ref( - range_ref, - ) - if cell.row < row_lo || - cell.row > row_hi || - cell.col < column_lo || - cell.col > column_hi { - raise InvalidXml(msg="shared formula master is outside its range") - } - let master : SharedFormulaMaster = { - formula, - row: cell.row, - column: cell.col, - row_lo, - column_lo, - row_hi, - column_hi, - } - match masters.get(shared_index) { - Some(_) => - raise InvalidXml( - msg="shared formula index has conflicting masters", - ) - None => masters[shared_index] = master - } + (Some(Shared), Some(formula), Some(shared_index), Some(range_ref)) => { + if formula == "" { + raise InvalidXml(msg="shared formula master text missing") } - (Some(Shared), Some(formula), Some(_), None) if formula != "" => - raise InvalidXml(msg="shared formula master range missing") + let (row_lo, column_lo, row_hi, column_hi) = parse_range_ref(range_ref) + if cell.row < row_lo || + cell.row > row_hi || + cell.col < column_lo || + cell.col > column_hi { + raise InvalidXml(msg="shared formula master is outside its range") + } + let master : SharedFormulaMaster = { + formula, + row: cell.row, + column: cell.col, + row_lo, + column_lo, + row_hi, + column_hi, + } + match masters.get(shared_index) { + Some(_) => + raise InvalidXml(msg="shared formula index has conflicting masters") + None => masters[shared_index] = master + } + } (Some(Shared), Some(_), None, _) => raise InvalidXml(msg="shared formula index missing") (Some(Shared), None, _, _) => @@ -442,9 +433,10 @@ fn validated_shared_formula_masters( _ => () } } + validate_shared_formula_master_ranges(masters) for cell in cells { - match (cell.formula_type, cell.formula, cell.formula_shared_index) { - (Some(Shared), Some(""), Some(shared_index)) => + match (cell.formula_type, cell.formula_shared_index, cell.formula_ref) { + (Some(Shared), Some(shared_index), None) => match masters.get(shared_index) { Some(master) => if !master.contains_coordinate(cell.row, cell.col) { @@ -466,7 +458,7 @@ fn validated_shared_formula_masters( /// master's declared range. Conflicting or incomplete groups are rejected. pub fn Worksheet::shared_formula_masters( self : Worksheet, -) -> Map[Int, SharedFormulaMaster] raise XlsxError { +) -> Map[UInt, SharedFormulaMaster] raise XlsxError { validated_shared_formula_masters(self.cells) } @@ -663,21 +655,30 @@ pub fn Worksheet::set_cell_formula_opts( msg="shared formula master must be top-left of ref", ) } - let mut max_si = -1 + let mut max_si : UInt? = None for cell in self.cells { match cell.formula_shared_index { - Some(si) => if si > max_si { max_si = si } + Some(si) => + match max_si { + Some(current) => if si > current { max_si = Some(si) } + None => max_si = Some(si) + } None => () } } - let shared_index = max_si + 1 + let shared_index = match max_si { + Some(0xFFFFFFFF) => + raise InvalidSheetOperation(msg="shared formula index exhausted") + Some(value) => value + 1 + None => 0 + } fn set_shared_cell( sheet : Worksheet, row : Int, col : Int, formula : String, formula_ref : String?, - shared_index : Int, + shared_index : UInt, value : String, ) -> Unit raise XlsxError { let canonical = cell_ref_from(row, col) diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 14b79068..fa36f93d 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -9,7 +9,7 @@ pub struct Cell { formula : String? formula_type : FormulaType? formula_ref : String? - formula_shared_index : Int? + formula_shared_index : UInt? /// Whether this formula cell contained an OOXML `` cached-result node. /// Empty cached strings are semantically distinct from a missing cache. formula_value_present : Bool @@ -18,15 +18,16 @@ pub struct Cell { ///| /// Read-only formula metadata for one worksheet coordinate. `formula` is the -/// exact text stored in that cell; OOXML shared-formula followers therefore -/// carry an empty string while `formula_type` and `shared_index` preserve -/// their formula-bearing state. Callers that need display/query text can use -/// the shared index to resolve the corresponding non-empty master formula. +/// cell's semantic formula text; OOXML says shared-formula follower text is +/// ignored, so followers are canonicalized to an empty string while +/// `formula_type` and `shared_index` preserve their formula-bearing state. +/// Callers that need display/query text can use the shared index to resolve +/// the corresponding non-empty master formula. pub struct CellFormulaInfo { formula : String formula_type : FormulaType? range_ref : String? - shared_index : Int? + shared_index : UInt? /// True when the formula has a cached `` result, including ``. cached_value_present : Bool } From 6abb5a640508881df1cff45d23f1eefd78c493f0 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 01:29:33 +0800 Subject: [PATCH 026/150] fix(office): publish exact format capabilities --- docs/agent-json-schemas.md | 6 +- office/capabilities.mbt | 255 ++++++++++++++++++++++++++++++++++- office/capabilities_test.mbt | 82 +++++++++-- office/cmd/office/cram/cli.t | 28 ++-- 4 files changed, 347 insertions(+), 24 deletions(-) diff --git a/docs/agent-json-schemas.md b/docs/agent-json-schemas.md index 53fc7427..3266b3ca 100644 --- a/docs/agent-json-schemas.md +++ b/docs/agent-json-schemas.md @@ -33,8 +33,10 @@ disagree, the tests are the source of truth and this document has a bug. `images`, `pivot_tables`, `defined_names`, `cells`) are always present (possibly `[]`); optional sub-object fields — including list-valued ones like `fill.colors` or a style's `border` — may be omitted entirely. -- **Index bases**: sheet `index` is 0-based tab order. Rows and columns are - never sent as bare numbers; cell and range references are A1-style strings. +- **Index bases**: standalone `xlsx.*` payloads use 0-based sheet `index` + values; unified `office.xlsx.*` payloads use 1-based sheet `index` values. + Rows and columns are never sent as bare numbers; cell and range references + are A1-style strings. - Output is pretty-printed (2-space indent) UTF-8 with a trailing newline, deterministic for a given workbook. diff --git a/office/capabilities.mbt b/office/capabilities.mbt index 57d4c120..956736b1 100644 --- a/office/capabilities.mbt +++ b/office/capabilities.mbt @@ -185,7 +185,7 @@ pub fn capability_formats() -> Array[CapabilityFormat] { /// Returns implemented command declarations in stable order. An empty /// `formats` array marks a format-neutral command. pub fn capability_commands() -> Array[CapabilityCommand] { - [ + let commands : Array[CapabilityCommand] = [ { name: "help", summary: "Show the implemented Office capability registry", @@ -854,6 +854,7 @@ pub fn capability_commands() -> Array[CapabilityCommand] { ], }, ] + commands.map(hydrate_capability_command_variants) } ///| @@ -1156,6 +1157,245 @@ fn command_supports_format( command.formats.contains(format.name()) } +///| +fn capability_variant_document_format( + variant : CapabilityVariant, +) -> DocumentFormat? { + for constraint in variant.constraints { + match constraint { + "format=docx" => return Some(Docx) + "format=xlsx" => return Some(Xlsx) + _ => () + } + } + None +} + +///| +fn capability_field_applies_to_format( + command_name : String, + field_name : String, + output : Bool, + format : DocumentFormat, +) -> Bool { + let docx_only : Array[String] = if output { + match command_name { + "outline" => + [ + "scanned_elements", "counts", "stories", "headings", "styles_in_use", "images", + "sections", "diagnostics", + ] + "get" => + ["role", "source", "id", "children", "properties", "metadata", "text"] + "text" => ["scanned_elements"] + "query" => ["filters", "scanned_elements"] + _ => [] + } + } else { + match command_name { + "query" => ["kind", "text", "id", "property", "ignore-case"] + _ => [] + } + } + let xlsx_only : Array[String] = if output { + match command_name { + "outline" => + [ + "path", "sheet_count", "active_sheet", "sheets", "defined_names", "limits", + ] + "get" => + [ + "sheet_count", "sheets", "defined_names", "sheet", "cell", "cells", "reference", + "styles", "scanned_cells", "returned", + ] + "text" => ["scanned_cells"] + "query" => ["selector", "styles", "scanned_cells"] + _ => [] + } + } else { + match command_name { + "query" => ["selector"] + _ => [] + } + } + if docx_only.contains(field_name) { + return format is Docx + } + if xlsx_only.contains(field_name) { + return format is Xlsx + } + true +} + +///| +fn exact_capability_field( + field : CapabilityField, + format : DocumentFormat, + result_schema : String?, +) -> CapabilityField { + let type_name = if field.name == "schema" { + match result_schema { + Some(schema) => "literal(\{schema})" + None => field.type_name + } + } else if field.name == "format" { + "literal(\{format.name()})" + } else { + field.type_name + } + { + name: field.name, + type_name, + required: field.required, + description: field.description, + } +} + +///| +fn exact_capability_fields( + command_name : String, + fields : Array[CapabilityField], + output : Bool, + format : DocumentFormat, + result_schema : String?, +) -> Array[CapabilityField] { + let exact : Array[CapabilityField] = [] + for field in fields { + if capability_field_applies_to_format( + command_name, + field.name, + output, + format, + ) { + exact.push(exact_capability_field(field, format, result_schema)) + } + } + exact +} + +///| +fn hydrate_capability_command_variants( + command : CapabilityCommand, +) -> CapabilityCommand { + let variants : Array[CapabilityVariant] = [] + for variant in command.variants { + match capability_variant_document_format(variant) { + Some(format) => + variants.push({ + name: variant.name, + usage: variant.usage, + result_schema: variant.result_schema, + inputs: if variant.inputs.is_empty() { + exact_capability_fields( + command.name, + command.inputs, + false, + format, + Some(variant.result_schema), + ) + } else { + variant.inputs + }, + outputs: if variant.outputs.is_empty() { + exact_capability_fields( + command.name, + command.outputs, + true, + format, + Some(variant.result_schema), + ) + } else { + variant.outputs + }, + constraints: variant.constraints, + actions: variant.actions, + output_modes: variant.output_modes, + }) + None => variants.push(variant) + } + } + { + name: command.name, + summary: command.summary, + usage: command.usage, + formats: command.formats, + aliases: command.aliases, + inputs: command.inputs, + outputs: command.outputs, + output_modes: command.output_modes, + variants, + } +} + +///| +fn capability_command_for_format( + command : CapabilityCommand, + format : DocumentFormat, +) -> CapabilityCommand { + let mut result_schema : String? = None + for variant in command.variants { + match capability_variant_document_format(variant) { + Some(candidate) if candidate.name() == format.name() => + result_schema = Some(variant.result_schema) + _ => () + } + } + let variants : Array[CapabilityVariant] = [] + for variant in command.variants { + let should_include = match capability_variant_document_format(variant) { + Some(candidate) => candidate.name() == format.name() + None => true + } + if should_include { + variants.push({ + name: variant.name, + usage: variant.usage, + result_schema: variant.result_schema, + inputs: exact_capability_fields( + command.name, + variant.inputs, + false, + format, + Some(variant.result_schema), + ), + outputs: exact_capability_fields( + command.name, + variant.outputs, + true, + format, + Some(variant.result_schema), + ), + constraints: variant.constraints, + actions: variant.actions, + output_modes: variant.output_modes, + }) + } + } + { + name: command.name, + summary: command.summary, + usage: command.usage, + formats: [format.name()], + aliases: command.aliases, + inputs: exact_capability_fields( + command.name, + command.inputs, + false, + format, + result_schema, + ), + outputs: exact_capability_fields( + command.name, + command.outputs, + true, + format, + result_schema, + ), + output_modes: command.output_modes, + variants, + } +} + ///| /// Returns self-contained registry records in stable order. With a format /// filter, format-neutral commands are omitted and only commands that operate @@ -1176,7 +1416,11 @@ pub fn capability_records( } if format_matches && (command.name == normalized || command.aliases.contains(normalized)) { - records.push(capability_command_json(command, fingerprint)) + let exact_command = match format { + Some(selected) => capability_command_for_format(command, selected) + None => command + } + records.push(capability_command_json(exact_command, fingerprint)) } } return records @@ -1192,7 +1436,12 @@ pub fn capability_records( } for command in capability_commands() { if command_supports_format(command, selected) { - records.push(capability_command_json(command, fingerprint)) + records.push( + capability_command_json( + capability_command_for_format(command, selected), + fingerprint, + ), + ) } } } diff --git a/office/capabilities_test.mbt b/office/capabilities_test.mbt index 6f0c7701..0ff836c0 100644 --- a/office/capabilities_test.mbt +++ b/office/capabilities_test.mbt @@ -211,6 +211,70 @@ test "structured read capabilities publish both format schemas and enforced boun } } +///| +test "structured capability variants are self-contained and format exact" { + for + case in [ + ("outline", @office.SCHEMA_DOCX_OUTLINE, @office.SCHEMA_XLSX_OUTLINE), + ("get", @office.SCHEMA_DOCX_ELEMENT, @office.SCHEMA_XLSX_ELEMENT), + ("text", @office.SCHEMA_DOCX_TEXT, @office.SCHEMA_XLSX_TEXT), + ("query", @office.SCHEMA_DOCX_QUERY, @office.SCHEMA_XLSX_QUERY), + ] { + let (name, docx_schema, xlsx_schema) = case + guard @office.find_capability_command(name) is Some(command) else { + fail("missing command: \{name}") + } + assert_eq(command.variants.length(), 2) + let docx = command.variants[0] + let xlsx = command.variants[1] + assert_eq(docx.name, "docx") + assert_eq(xlsx.name, "xlsx") + assert_true(!docx.inputs.is_empty()) + assert_true(!docx.outputs.is_empty()) + assert_true(!xlsx.inputs.is_empty()) + assert_true(!xlsx.outputs.is_empty()) + guard docx.outputs.filter(field => field.name == "schema").get(0) + is Some(docx_field) else { + fail("missing DOCX schema field: \{name}") + } + guard xlsx.outputs.filter(field => field.name == "schema").get(0) + is Some(xlsx_field) else { + fail("missing XLSX schema field: \{name}") + } + assert_eq(docx_field.type_name, "literal(\{docx_schema})") + assert_eq(xlsx_field.type_name, "literal(\{xlsx_schema})") + assert_false( + docx.outputs.any(field => field.description.contains("XLSX only")), + ) + assert_false( + xlsx.outputs.any(field => field.description.contains("DOCX only")), + ) + assert_false( + docx.inputs.any(field => field.description.contains("XLSX only")), + ) + assert_false( + xlsx.inputs.any(field => field.description.contains("DOCX only")), + ) + } +} + +///| +test "format-filtered capability records remove cross-format unions" { + let docx_records = @office.capability_records(format=Docx).map(record => { + record.stringify() + }) + let joined = docx_records.join("\n") + assert_false(joined.contains("\"formats\":[\"docx\",\"xlsx\"]")) + assert_false(joined.contains("\"name\":\"xlsx\",\"usage\":")) + assert_false(joined.contains("XLSX only")) + assert_true(joined.contains("\"type\":\"literal(docx)\"")) + for record in docx_records { + if record.contains("\"kind\":\"command\"") { + assert_true(record.contains("\"formats\":[\"docx\"]")) + } + } +} + ///| test "capability registry format filters remain truthful" { let records = @office.capability_records(format=Docx) @@ -219,13 +283,13 @@ test "capability registry format filters remain truthful" { inspect( records, content=( - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"outline","summary":"Summarize bounded XLSX or DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.outline/1|office.xlsx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"},{"name":"path","type":"literal(/xlsx/workbook)","required":false,"description":"XLSX only: canonical workbook selector"},{"name":"sheet_count","type":"integer","required":false,"description":"XLSX only: workbook tab count"},{"name":"active_sheet","type":"object","required":false,"description":"XLSX only: canonical active-sheet summary when the workbook has tabs"},{"name":"sheets","type":"array","required":false,"description":"XLSX only: bounded tab-order sheet summaries with canonical paths and used ranges"},{"name":"defined_names","type":"array","required":false,"description":"XLSX only: bounded workbook defined-name inventory"},{"name":"limits","type":"object","required":false,"description":"XLSX only: effective scan and metadata limits"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office outline FILE","result_schema":"office.docx.outline/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office outline FILE","result_schema":"office.xlsx.outline/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"get","summary":"Resolve one canonical XLSX or DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.element/1|office.xlsx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"},{"name":"sheet_count","type":"integer","required":false,"description":"XLSX workbook selectors only: workbook tab count"},{"name":"sheets","type":"array","required":false,"description":"XLSX workbook selectors only: bounded tab-order sheet summaries"},{"name":"defined_names","type":"array","required":false,"description":"XLSX workbook selectors only: bounded defined-name inventory"},{"name":"sheet","type":"object","required":false,"description":"XLSX sheet selectors only: bounded sheet summary"},{"name":"cell","type":"object","required":false,"description":"XLSX cell selectors only: typed value, formula, style, and canonical path"},{"name":"cells","type":"array","required":false,"description":"XLSX range selectors only: populated cells in row-major order"},{"name":"reference","type":"a1-range","required":false,"description":"XLSX range selectors only: normalized A1 rectangle"},{"name":"styles","type":"object","required":false,"description":"XLSX coordinate selectors only: deduplicated referenced style definitions"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX coordinate selectors only: exact rectangle scan count"},{"name":"returned","type":"integer","required":false,"description":"XLSX range selectors only: populated cell count"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office get FILE /docx/...","result_schema":"office.docx.element/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office get FILE /xlsx/...","result_schema":"office.xlsx.element/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.text/1|office.xlsx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office text FILE [--under /xlsx/...]","result_schema":"office.xlsx.text/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: cell followed by up to 16 bounded type, value, formula, or text predicates; default cell"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"enum(office.docx.query/1|office.xlsx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"selector","type":"xlsx-cell-selector","required":false,"description":"XLSX only: supplied bounded cell selector"},{"name":"styles","type":"object","required":false,"description":"XLSX only: deduplicated styles referenced by returned matches"},{"name":"scanned_cells","type":"integer","required":false,"description":"XLSX only: exact bounded cell scan count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[],"outputs":[],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]},{"name":"xlsx","usage":"office query FILE [cell[predicate]...]","result_schema":"office.xlsx.query/1","inputs":[],"outputs":[],"constraints":["format=xlsx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"outline","summary":"Summarize bounded XLSX or DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office outline FILE","result_schema":"office.docx.outline/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"get","summary":"Resolve one canonical XLSX or DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office get FILE /docx/...","result_schema":"office.docx.element/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} ), ) } @@ -235,7 +299,7 @@ test "capability fingerprint and inventory ordering are deterministic" { inspect( @office.capability_fingerprint(), content=( - #|crc32:e9c5b34f + #|crc32:3f8b2b34 ), ) let first = @office.capabilities_data().stringify() @@ -251,7 +315,7 @@ test "capability operation queries do not leak unrelated records" { inspect( data.stringify(), content=( - #|{"schema":"office.capabilities/2","fingerprint":"crc32:e9c5b34f","records":[{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx","xlsx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"enum(docx|xlsx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} + #|{"schema":"office.capabilities/2","fingerprint":"crc32:3f8b2b34","records":[{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} ), ) } diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index d2afc580..30eaccef 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -17,7 +17,7 @@ and JSONL inventories without deferred PowerPoint or MCP entries. $ office.exe help | sed -n '1,8p' Office capability registry Schema: office.capabilities/2 - Fingerprint: crc32:e9c5b34f + Fingerprint: crc32:3f8b2b34 Formats: docx (aliases: word) — WordprocessingML documents xlsx (aliases: excel) — SpreadsheetML workbooks @@ -37,13 +37,13 @@ and JSONL inventories without deferred PowerPoint or MCP entries. {"kind":"format","name":"docx","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} $ office.exe help xlsx query --json | jq -c '.data.records[0] | {formats,variants:[.variants[]|{name,result_schema,constraints}]}' - {"formats":["docx","xlsx"],"variants":[{"name":"docx","result_schema":"office.docx.query/1","constraints":["format=docx"]},{"name":"xlsx","result_schema":"office.xlsx.query/1","constraints":["format=xlsx"]}]} + {"formats":["xlsx"],"variants":[{"name":"xlsx","result_schema":"office.xlsx.query/1","constraints":["format=xlsx"]}]} $ office.exe help all --json | jq -c '{schema,success,capability_schema:.data.schema,fingerprint:.data.fingerprint,names:[.data.records[].name]}' - {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:e9c5b34f","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} + {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:3f8b2b34","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} $ office.exe help all --jsonl | jq -s -c 'map({schema,fingerprint,kind,name})' - [{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:e9c5b34f","kind":"command","name":"raw"}] + [{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"raw"}] The raw command publishes explicit subcommand schemas, including every edit input and its conditional constraints. @@ -124,7 +124,15 @@ Structured XLSX reads use the same commands and envelope. Positional sheet input canonicalizes to stable name paths; ranges, text, and query scan in tab/row/column order with exact totals. - $ office.exe outline "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --json | jq -c '{success,schema:.data.schema,path:.data.path,sheet_count:.data.sheet_count,active:.data.active_sheet.path,sheets:[.data.sheets[]|{path,kind,state,used:.used_range.reference}]}' +The third-party Book1 fixture contains two intersecting shared-formula ranges. +Normalize that unrelated invalid metadata through the raw transaction surface +before exercising its original contents with the strict structured reader. + + $ office.exe raw read "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" /Sheet2 --output book1-sheet2.xml >/dev/null + $ sed -e 's/ref="F11:H11"/ref="F11:F11"/' -e 's/<\/f>/<\/f>/' book1-sheet2.xml > book1-sheet2-valid.xml + $ office.exe raw replace "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" /Sheet2 --xml-file book1-sheet2-valid.xml --out Book1-valid.xlsx --json >/dev/null + + $ office.exe outline Book1-valid.xlsx --json | jq -c '{success,schema:.data.schema,path:.data.path,sheet_count:.data.sheet_count,active:.data.active_sheet.path,sheets:[.data.sheets[]|{path,kind,state,used:.used_range.reference}]}' {"success":true,"schema":"office.xlsx.outline/1","path":"/xlsx/workbook","sheet_count":2,"active":"/xlsx/sheet[name=\"Sheet1\"]","sheets":[{"path":"/xlsx/sheet[name=\"Sheet1\"]","kind":"worksheet","state":"visible","used":"A1:D22"},{"path":"/xlsx/sheet[name=\"Sheet2\"]","kind":"worksheet","state":"visible","used":"A1:I11"}]} Singleton extents retain two endpoints, so every emitted range path parses and @@ -135,24 +143,24 @@ round-trips through the public selector grammar. $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/OverflowNumericCell.xlsx" '/xlsx/sheet[1]/range[A1:A1]' --json | jq -c '{path:.data.path,reference:.data.reference,refs:[.data.cells[].reference]}' {"path":"/xlsx/sheet[name=\"Sheet1\"]/range[A1:A1]","reference":"A1:A1","refs":["A1"]} - $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" '/xlsx/sheet[1]/range[A19:B19]' --json | jq -c '{schema:.data.schema,path:.data.path,kind:.data.kind,refs:[.data.cells[].reference],raw:[.data.cells[].raw],formulas:[.data.cells[]|(.formula // null)],returned:.data.returned}' + $ office.exe get Book1-valid.xlsx '/xlsx/sheet[1]/range[A19:B19]' --json | jq -c '{schema:.data.schema,path:.data.path,kind:.data.kind,refs:[.data.cells[].reference],raw:[.data.cells[].raw],formulas:[.data.cells[]|(.formula // null)],returned:.data.returned}' {"schema":"office.xlsx.element/1","path":"/xlsx/sheet[name=\"Sheet1\"]/range[A19:B19]","kind":"range","refs":["A19","B19"],"raw":[{"type":"string","value":"Total:"},{"type":"number","value":237}],"formulas":[null,"SUM(Sheet2!D2,Sheet2!D11)"],"returned":2} - $ office.exe text "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --under '/xlsx/sheet[name="Sheet1"]' --offset 1 --limit 2 --json | jq -c '{schema:.data.schema,under:.data.under,paths:[.data.entries[].path],texts:[.data.entries[].text],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' + $ office.exe text Book1-valid.xlsx --under '/xlsx/sheet[name="Sheet1"]' --offset 1 --limit 2 --json | jq -c '{schema:.data.schema,under:.data.under,paths:[.data.entries[].path],texts:[.data.entries[].text],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' {"schema":"office.xlsx.text/1","under":"/xlsx/sheet[name=\"Sheet1\"]","paths":["/xlsx/sheet[name=\"Sheet1\"]/cell[B19]","/xlsx/sheet[name=\"Sheet1\"]/cell[C21]"],"texts":["237","Column1"],"matched_total":5,"returned":2,"truncated":true,"scanned":88} - $ office.exe query "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json | jq -c '{schema:.data.schema,selector:.data.selector,under:.data.under,paths:[.data.matches[].path],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' + $ office.exe query Book1-valid.xlsx 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json | jq -c '{schema:.data.schema,selector:.data.selector,under:.data.under,paths:[.data.matches[].path],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' {"schema":"office.xlsx.query/1","selector":"cell[type=formula][formula~=IF]","under":"/xlsx/sheet[name=\"Sheet2\"]","paths":["/xlsx/sheet[name=\"Sheet2\"]/cell[F11]","/xlsx/sheet[name=\"Sheet2\"]/cell[G11]","/xlsx/sheet[name=\"Sheet2\"]/cell[H11]","/xlsx/sheet[name=\"Sheet2\"]/cell[I11]"],"matched_total":4,"returned":4,"truncated":false,"scanned":99} Cross-format selectors and DOCX-only XLSX query flags fail with XLSX-specific, machine-correctable codes. - $ office.exe get "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" '/docx/body' --json > xlsx-selector-mismatch.json 2>&1; echo $? + $ office.exe get Book1-valid.xlsx '/docx/body' --json > xlsx-selector-mismatch.json 2>&1; echo $? 1 $ jq -c '{success,code:.error.code,expected:.error.details.expected_format,actual:.error.details.actual_format}' xlsx-selector-mismatch.json {"success":false,"code":"office.xlsx.selector_format_mismatch","expected":"xlsx","actual":"docx"} - $ office.exe query "$TESTDIR/../../../../fixtures/excelize/test/Book1.xlsx" --kind cell --json > xlsx-query-options.json 2>&1; echo $? + $ office.exe query Book1-valid.xlsx --kind cell --json > xlsx-query-options.json 2>&1; echo $? 1 $ jq -c '{success,code:.error.code,options:.error.details.options}' xlsx-query-options.json {"success":false,"code":"office.xlsx.unsupported_query_options","options":["--kind"]} From 75a4071b4a022611198313f3e3139c5c140b59b3 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 01:29:52 +0800 Subject: [PATCH 027/150] fix(office): detect bounded input size races --- office/cmd/office/raw.mbt | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/office/cmd/office/raw.mbt b/office/cmd/office/raw.mbt index 38be4ce8..bd7e2468 100644 --- a/office/cmd/office/raw.mbt +++ b/office/cmd/office/raw.mbt @@ -144,6 +144,35 @@ async fn read_bounded_file( } offset = offset + count } + let final_size = file.size() catch { + error if @async.is_being_cancelled() => raise error + _ => + raise cli_file_error( + code, + "could not recheck \{description} size: ", + path, + ) + } + if final_size != size { + if final_size < 0L || final_size > maximum.to_int64() { + raise CliFailure( + @lib.protocol_error( + limit_code, + "\{description} exceeds the \{maximum}-byte limit", + details=Json::object({ + "file": Json::string(bounded_text(path, 160)), + "limit": Json::number(maximum.to_double()), + "actual": Json::number(final_size.to_double()), + }), + ), + ) + } + raise cli_file_error( + code, + "\{description} changed while being read: ", + path, + ) + } complete_bounded_file_read(buffer) } From 3f4bb81e5ed9faf48f94e0c86fcff87af961f329 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 01:30:08 +0800 Subject: [PATCH 028/150] chore(ooxml): remove redundant view conversion --- ooxml/read_parse.mbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 4a14542c..fef8539f 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -316,7 +316,7 @@ pub fn find_part_content_type( None => { let filename = match normalized.rev_find("/") { Some(index) => normalized[index + 1:] - None => normalized[:] + None => normalized } let extension = match filename.rev_find(".") { Some(index) if index + 1 < filename.length() => From 9db55f6fae5a3c27dae11e3481222e8c6123728c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 02:38:59 +0800 Subject: [PATCH 029/150] fix: harden XLSX XML and relationship parsing --- ooxml/read_parse.mbt | 73 +++++++++++- ooxml/read_parse_test.mbt | 65 ++++++++-- ooxml/start_tag_scanner.mbt | 190 +++++++++++++++++++++++++++--- xlsx/cell_images.mbt | 2 +- xlsx/header_footer_image_read.mbt | 2 +- xlsx/ooxml_rels.mbt | 84 +++++++++++-- xlsx/picture_ops_test.mbt | 32 +++++ xlsx/read.mbt | 57 ++++++--- xlsx/read_drawing_xml.mbt | 12 +- xlsx/read_limits_test.mbt | 33 ++++++ xlsx/read_package_parts.mbt | 26 +++- xlsx/read_sheet_rel_parts.mbt | 4 +- xlsx/read_xml_namespaces.mbt | 6 +- xlsx/rich_value_images.mbt | 4 +- 14 files changed, 513 insertions(+), 77 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index fef8539f..e61c18ab 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -137,13 +137,41 @@ fn relationship_attribute( value } +///| +priv enum ParsedRelationshipTargetMode { + InternalTarget + ExternalTarget +} + +///| +fn relationship_target_mode( + scanner : XmlStartTagScanner, +) -> ParsedRelationshipTargetMode raise ParseXmlError { + match scanner.attribute("", "TargetMode") { + None => InternalTarget + Some(value) => + match collapse_schema_whitespace(value) { + "Internal" => InternalTarget + "External" => ExternalTarget + _ => raise InvalidXml(msg="relationship target mode invalid") + } + } +} + +///| +priv enum RelationshipTargetSelection { + InternalTargets + ExternalTargets +} + ///| /// Reads relationship targets whose decoded `Type` is in `rel_types`. Element /// names are matched by expanded namespace name, so the package namespace may /// use either a default namespace or any declared prefix. -pub fn parse_relationship_targets_by_types( +fn parse_relationship_targets_by_types_selected( xml : StringView, rel_types : ArrayView[String], + selection : RelationshipTargetSelection, ) -> Map[String, String] raise ParseXmlError { let targets : Map[String, String] = Map([]) let ids : Set[String] = Set([]) @@ -171,6 +199,7 @@ pub fn parse_relationship_targets_by_types( let rel = relationship_attribute(scanner, "Type") let id = relationship_attribute(scanner, "Id") let target = relationship_attribute(scanner, "Target") + let target_mode = relationship_target_mode(scanner) if rel.length() > max_relationship_type_chars { raise InvalidXml(msg="relationship type is too long") } @@ -199,17 +228,53 @@ pub fn parse_relationship_targets_by_types( if !wanted { continue } + let selected = match (selection, target_mode) { + (InternalTargets, InternalTarget) | (ExternalTargets, ExternalTarget) => + true + _ => false + } + if !selected { + continue + } targets[id] = target } targets } ///| -pub fn parse_relationship_targets( +/// Reads internal relationship targets whose decoded `Type` is in +/// `rel_types`. Explicitly external records are validated but excluded. +pub fn parse_internal_relationship_targets_by_types( + xml : StringView, + rel_types : ArrayView[String], +) -> Map[String, String] raise ParseXmlError { + parse_relationship_targets_by_types_selected(xml, rel_types, InternalTargets) +} + +///| +pub fn parse_internal_relationship_targets( + xml : StringView, + rel_type : StringView, +) -> Map[String, String] raise ParseXmlError { + parse_internal_relationship_targets_by_types(xml, [rel_type.to_owned()]) +} + +///| +/// Reads explicitly external relationship targets whose decoded `Type` is in +/// `rel_types`. Internal records are validated but excluded. +pub fn parse_external_relationship_targets_by_types( + xml : StringView, + rel_types : ArrayView[String], +) -> Map[String, String] raise ParseXmlError { + parse_relationship_targets_by_types_selected(xml, rel_types, ExternalTargets) +} + +///| +pub fn parse_external_relationship_targets( xml : StringView, rel_type : StringView, ) -> Map[String, String] raise ParseXmlError { - parse_relationship_targets_by_types(xml, [rel_type.to_owned()]) + parse_external_relationship_targets_by_types(xml, [rel_type.to_owned()]) } ///| @@ -357,7 +422,7 @@ test "ooxml read_parse wb: every relationship record is validated" { #| #| #| - try parse_relationship_targets(xml, "t1") catch { + try parse_internal_relationship_targets(xml, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship type missing") } noraise { _ => fail("expected relationship type missing") diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 3c399546..11714ff7 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -1,5 +1,5 @@ ///| -test "ooxml read_parse: parse_relationship_targets supports self-closing and expanded tags" { +test "ooxml read_parse: internal relationships support self-closing and expanded tags" { let xml = #| #| @@ -8,7 +8,7 @@ test "ooxml read_parse: parse_relationship_targets supports self-closing and exp #| #| #| - let targets = @ooxml.parse_relationship_targets(xml, "t1") + let targets = @ooxml.parse_internal_relationship_targets(xml, "t1") debug_inspect( targets, content=( @@ -18,14 +18,14 @@ test "ooxml read_parse: parse_relationship_targets supports self-closing and exp } ///| -test "ooxml read_parse: parse_relationship_targets reports missing id/target and malformed tag" { +test "ooxml read_parse: internal relationships report missing id/target and malformed tag" { let missing_id = #| #| #| #| #| - try @ooxml.parse_relationship_targets(missing_id, "t1") catch { + try @ooxml.parse_internal_relationship_targets(missing_id, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship id missing") } noraise { _ => fail("expected relationship id missing") @@ -37,7 +37,7 @@ test "ooxml read_parse: parse_relationship_targets reports missing id/target and #| #| #| - try @ooxml.parse_relationship_targets(missing_target, "t1") catch { + try @ooxml.parse_internal_relationship_targets(missing_target, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship target missing") } noraise { _ => fail("expected relationship target missing") @@ -49,7 +49,7 @@ test "ooxml read_parse: parse_relationship_targets reports missing id/target and #| #| - try @ooxml.parse_relationship_targets(malformed, "t1") catch { + try @ooxml.parse_internal_relationship_targets(malformed, "t1") catch { InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") } noraise { _ => fail("expected relationship tag not closed") @@ -62,7 +62,7 @@ test "ooxml read_parse: prefixed relationships decode entities exactly once" { #| #| #| - let targets = @ooxml.parse_relationship_targets(xml, "urn:type1") + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:type1") debug_inspect( targets, content=( @@ -77,10 +77,52 @@ test "ooxml read_parse: relationship schema whitespace is collapsed" { #| #| #| - let targets = @ooxml.parse_relationship_targets(xml, "urn:type one") + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:type one") debug_inspect(targets, content="{ \"r 1\": \"custom/a b.xml\" }") } +///| +test "ooxml read_parse: relationship target modes are explicit and filtered" { + let xml = + #| + #| + #| + #| + #| + debug_inspect( + @ooxml.parse_internal_relationship_targets(xml, "wanted"), + content=( + #|{ "default": "inside.xml", "internal": "also-inside.xml" } + ), + ) + debug_inspect( + @ooxml.parse_external_relationship_targets(xml, "wanted"), + content=( + #|{ "external": "https://example.invalid" } + ), + ) +} + +///| +test "ooxml read_parse: invalid target modes fail closed" { + let xml = + #| + for external in [false, true] { + try { + if external { + ignore(@ooxml.parse_external_relationship_targets(xml, "wanted")) + } else { + ignore(@ooxml.parse_internal_relationship_targets(xml, "wanted")) + } + } catch { + InvalidXml(msg~) => + inspect(msg, content="relationship target mode invalid") + } noraise { + _ => fail("expected invalid relationship target mode") + } + } +} + ///| test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay unique" { for @@ -92,7 +134,7 @@ test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay #| ), ] { - try @ooxml.parse_relationship_targets(xml, "wanted") catch { + try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { InvalidXml(_) => () } noraise { _ => fail("expected invalid relationship document") @@ -116,7 +158,10 @@ test "ooxml read_parse: relationship limits accept writer boundaries" { "", ) xml.write_string("") - let targets = @ooxml.parse_relationship_targets(xml.to_string(), "wanted") + let targets = @ooxml.parse_internal_relationship_targets( + xml.to_string(), + "wanted", + ) assert_eq(targets.get("wanted"), Some(long_target)) } diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 570e4264..0723c7b8 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -25,6 +25,7 @@ priv struct XmlNamespaceBinding { uri : String namespace_id : Int depth : Int + previous_index : Int? } ///| @@ -45,6 +46,7 @@ pub struct XmlStartTagScanner { priv mut index : Int priv mut depth : Int priv namespace_bindings : Array[XmlNamespaceBinding] + priv namespace_prefix_indices : Map[String, Int] priv namespace_ids : Map[String, Int] priv namespace_ref_counts : Map[Int, Int] priv mut next_namespace_id : Int @@ -54,6 +56,9 @@ pub struct XmlStartTagScanner { priv rewrite_element_namespaces : Array[String] priv rewrite_attribute_prefixes : Map[String, String] priv rewrite_output : StringBuilder + priv canonicalize_markup : Bool + priv rewrite_output_limit : Int + priv mut rewrite_output_chars : Int priv mut rewrite_cursor : Int priv mut rewrite_count : Int priv mut pending_namespace_pop : Int? @@ -497,6 +502,10 @@ fn XmlStartTagScanner::pop_namespace_depth( } let binding = self.namespace_bindings[last] ignore(self.namespace_bindings.pop()) + match binding.previous_index { + Some(index) => self.namespace_prefix_indices[binding.prefix] = index + None => ignore(self.namespace_prefix_indices.remove(binding.prefix)) + } self.release_namespace_identity(binding) } } @@ -513,12 +522,12 @@ fn XmlStartTagScanner::resolve_prefix_identity( if prefix.length() == 0 && attribute { return ("", 0) } - for offset in 0.. { + let binding = self.namespace_bindings[index] return (binding.uri, binding.namespace_id) } + None => () } if prefix.length() == 0 { ("", 0) @@ -598,12 +607,16 @@ fn XmlStartTagScanner::add_namespace_declarations( raise InvalidXml(msg="XML has too many active namespace bindings") } let namespace_id = self.namespace_identity(uri) + let previous_index = self.namespace_prefix_indices.get(prefix) + let binding_index = self.namespace_bindings.length() self.namespace_bindings.push({ prefix, uri, namespace_id, depth: scope_depth, + previous_index, }) + self.namespace_prefix_indices[prefix] = binding_index } } @@ -614,6 +627,7 @@ fn XmlStartTagScanner::validate_attributes( tag_end : Int, ) -> Unit raise ParseXmlError { let expanded_names : Map[Int, Set[String]] = Map([]) + let canonical_names : Set[String] = Set([]) let mut next = attributes_start let mut count = 0 while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { @@ -659,47 +673,102 @@ fn XmlStartTagScanner::validate_attributes( raise InvalidXml(msg="XML attribute is duplicated") } local_names.add(local_name) - match self.rewrite_attribute_prefixes.get(uri) { - Some(canonical_prefix) => { - let replacement = if canonical_prefix == "" { + let replacement = match self.rewrite_attribute_prefixes.get(uri) { + Some(canonical_prefix) => + if canonical_prefix == "" { local_name } else { canonical_prefix + ":" + local_name } + None => self.xml[attribute.name_start:attribute.name_end].to_owned() + } + if canonical_names.contains(replacement) { + raise InvalidXml(msg="XML canonical attribute name collision") + } + canonical_names.add(replacement) + match self.rewrite_attribute_prefixes.get(uri) { + Some(_) => if !xml_span_equals( self.xml, attribute.name_start, attribute.name_end, replacement, ) { - self.record_name_rewrite( + self.record_rewrite( attribute.name_start, attribute.name_end, replacement, ) } - } None => () } } } ///| -fn XmlStartTagScanner::record_name_rewrite( +fn XmlStartTagScanner::record_rewrite( self : XmlStartTagScanner, start : Int, end : Int, replacement : String, ) -> Unit raise ParseXmlError { if start < self.rewrite_cursor || end < start { - raise InvalidXml(msg="XML name rewrites overlap") + raise InvalidXml(msg="XML canonical rewrites overlap") } + self.charge_rewrite_output(start - self.rewrite_cursor) + self.charge_rewrite_output(replacement.length()) self.rewrite_output.write_view(self.xml[self.rewrite_cursor:start]) self.rewrite_output.write_string(replacement) self.rewrite_cursor = end self.rewrite_count = self.rewrite_count + 1 } +///| +fn XmlStartTagScanner::charge_rewrite_output( + self : XmlStartTagScanner, + count : Int, +) -> Unit raise ParseXmlError { + if count < 0 || count > self.rewrite_output_limit - self.rewrite_output_chars { + raise InvalidXml(msg="XML canonical output limit exceeded") + } + self.rewrite_output_chars += count +} + +///| +fn XmlStartTagScanner::escaped_character_data( + self : XmlStartTagScanner, + rewrite_start : Int, + value_start : Int, + end : Int, +) -> String raise ParseXmlError { + let unchanged = rewrite_start - self.rewrite_cursor + if unchanged < 0 || + unchanged > self.rewrite_output_limit - self.rewrite_output_chars { + raise InvalidXml(msg="XML canonical output limit exceeded") + } + let available = self.rewrite_output_limit - + self.rewrite_output_chars - + unchanged + let value = self.xml[value_start:end] + let output = StringBuilder::new(size_hint=value.length().min(available)) + let mut used = 0 + for unit in value { + let needed = if unit == '&' { 5 } else if unit == '<' { 4 } else { 1 } + if needed > available - used { + raise InvalidXml(msg="XML canonical output limit exceeded") + } + used += needed + if unit == '&' { + output.write_string("&") + } else if unit == '<' { + output.write_string("<") + } else { + output.write_char(unit) + } + } + output.to_string() +} + ///| fn XmlStartTagScanner::should_unprefix_element( self : XmlStartTagScanner, @@ -718,12 +787,15 @@ fn xml_start_tag_scanner_with_rewrites( xml : StringView, element_namespaces : Array[String], attribute_prefixes : Map[String, String], + canonicalize_markup : Bool, + rewrite_output_limit : Int, ) -> XmlStartTagScanner { { xml, index: 0, depth: 0, namespace_bindings: [], + namespace_prefix_indices: Map([]), namespace_ids: Map([]), namespace_ref_counts: Map([]), next_namespace_id: 2, @@ -733,6 +805,9 @@ fn xml_start_tag_scanner_with_rewrites( rewrite_element_namespaces: element_namespaces, rewrite_attribute_prefixes: attribute_prefixes, rewrite_output: StringBuilder::new(), + canonicalize_markup, + rewrite_output_limit, + rewrite_output_chars: 0, rewrite_cursor: 0, rewrite_count: 0, pending_namespace_pop: None, @@ -750,7 +825,7 @@ fn xml_start_tag_scanner_with_rewrites( /// Creates a namespace-aware cursor over `xml`. The source is borrowed; no /// copy of the full XML part is made. pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { - xml_start_tag_scanner_with_rewrites(xml, [], Map([])) + xml_start_tag_scanner_with_rewrites(xml, [], Map([]), false, 0) } ///| @@ -785,7 +860,7 @@ fn XmlStartTagScanner::consume_end_tag( if rewrite_name { match colon { Some(value) => - self.record_name_rewrite( + self.record_rewrite( name_start, name_end, self.xml[value + 1:name_end].to_owned(), @@ -837,6 +912,9 @@ pub fn XmlStartTagScanner::next( Some(value) => value None => raise InvalidXml(msg="XML processing instruction is not closed") } + if self.canonicalize_markup { + self.record_rewrite(start, end + 2, "") + } self.index = end + 2 continue } @@ -847,6 +925,9 @@ pub fn XmlStartTagScanner::next( Some(value) => value None => raise InvalidXml(msg="XML comment is not closed") } + if self.canonicalize_markup { + self.record_rewrite(start, end + 3, "") + } self.index = end + 3 continue } @@ -859,6 +940,13 @@ pub fn XmlStartTagScanner::next( Some(value) => value None => raise InvalidXml(msg="XML CDATA section is not closed") } + if self.canonicalize_markup { + self.record_rewrite( + start, + end + 3, + self.escaped_character_data(start, start + 9, end), + ) + } self.index = end + 3 continue } @@ -903,7 +991,7 @@ pub fn XmlStartTagScanner::next( let rewrite_name = colon is Some(_) && self.should_unprefix_element(self.current_namespace_uri) if rewrite_name { - self.record_name_rewrite( + self.record_rewrite( name_start, name_end, match colon { @@ -968,12 +1056,19 @@ pub fn XmlStartTagScanner::depth(self : XmlStartTagScanner) -> Int { /// Validates `xml` and rewrites selected expanded names into lexical forms /// expected by a namespace-oblivious parser. Elements in `element_namespaces` /// lose their prefix; attributes in `attribute_prefixes` receive the mapped -/// prefix. Text, comments, CDATA, values, and unselected names are unchanged. +/// prefix. Comments and processing instructions are removed, while CDATA is +/// converted to escaped character data so namespace-oblivious parsers cannot +/// interpret markup-looking content as elements. Values and unselected names +/// are unchanged. pub fn canonicalize_xml_expanded_names( xml : StringView, element_namespaces : ArrayView[String], attribute_prefixes : ArrayView[(String, String)], + max_output_chars? : Int = max_xml_start_tag_chars, ) -> String raise ParseXmlError { + if max_output_chars < 0 { + raise InvalidXml(msg="XML canonical output limit is invalid") + } let elements : Array[String] = [] elements.append(element_namespaces) let attributes : Map[String, String] = Map([]) @@ -981,13 +1076,19 @@ pub fn canonicalize_xml_expanded_names( let (namespace_uri, prefix) = pair attributes[namespace_uri] = prefix } - let scanner = xml_start_tag_scanner_with_rewrites(xml, elements, attributes) + let scanner = xml_start_tag_scanner_with_rewrites( + xml, elements, attributes, true, max_output_chars, + ) while scanner.next() { } if scanner.rewrite_count == 0 { + if xml.length() > max_output_chars { + raise InvalidXml(msg="XML canonical output limit exceeded") + } return xml.to_owned() } + scanner.charge_rewrite_output(xml.length() - scanner.rewrite_cursor) scanner.rewrite_output.write_view(xml[scanner.rewrite_cursor:]) scanner.rewrite_output.to_string() } @@ -1152,20 +1253,71 @@ test "namespace duplicate checks retain namespace identities once" { } ///| -test "expanded-name canonicalization preserves data and rewrites both tag ends" { +test "namespace lookup stays bounded with many bindings and prefixed elements" { + let xml = StringBuilder::new() + xml.write_string("') + for _ in 0..<20_000 { + xml.write_string("") + } + xml.write_string("") + let scanner = XmlStartTagScanner::new(xml.to_string()) + let mut elements = 0 + while scanner.next() { + elements += 1 + } + assert_eq(elements, 20_001) +} + +///| +test "expanded-name canonicalization neutralizes non-element markup" { let xml = - #|]]> + #|"?>&value]]> let canonical = canonicalize_xml_expanded_names(xml, ["urn:sheet"], [ ("urn:rel", "r"), ]) inspect( canonical, content=( - #|]]> + #|<s:literal/>&value ), ) } +///| +test "expanded-name canonicalization rejects lexical attribute collisions" { + let xml = + #| + try + canonicalize_xml_expanded_names(xml, ["urn:sheet"], [("urn:rel", "r")]) + catch { + InvalidXml(msg~) => + inspect(msg, content="XML canonical attribute name collision") + } noraise { + _ => fail("expected canonical attribute collision") + } +} + +///| +test "expanded-name canonicalization bounds escaped CDATA output" { + try + canonicalize_xml_expanded_names( + "", + [], + [], + max_output_chars=20, + ) + catch { + InvalidXml(msg~) => + inspect(msg, content="XML canonical output limit exceeded") + } noraise { + _ => fail("expected canonical output limit") + } +} + ///| test "start tag scanner bounds qualified names" { let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index 453486b3..3d7bcbb1 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -27,7 +27,7 @@ fn parse_cell_images( Map([]) } else { // cellimages relationships are always image relationships - parse_relationship_targets(rels_xml, rel_image) + parse_internal_relationship_targets(rels_xml, rel_image) } let xml_str = cell_images_xml.to_owned() // Each embedded image is one <...pic> element; cellimages.xml contains diff --git a/xlsx/header_footer_image_read.mbt b/xlsx/header_footer_image_read.mbt index 05dd958a..5555c296 100644 --- a/xlsx/header_footer_image_read.mbt +++ b/xlsx/header_footer_image_read.mbt @@ -86,7 +86,7 @@ fn parse_header_footer_images_from_vml( ) -> Array[HeaderFooterImage] raise XlsxError { let shapes = extract_vml_shapes(vml_xml) let rel_targets = match vml_rels_xml { - Some(xml) => parse_relationship_targets(xml, rel_image) + Some(xml) => parse_internal_relationship_targets(xml, rel_image) None => Map([]) } let images : Array[HeaderFooterImage] = [] diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index a6d9c625..a9a999c2 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -5,21 +5,41 @@ let transitional_office_relationship_prefix = "http://schemas.openxmlformats.org let strict_office_relationship_prefix = "http://purl.oclc.org/ooxml/officeDocument/relationships/" ///| -fn parse_relationship_targets_exact( +fn parse_internal_relationship_targets_exact( xml : StringView, rel_type : StringView, ) -> Map[String, String] raise XlsxError { - @ooxml.parse_relationship_targets(xml, rel_type) catch { + @ooxml.parse_internal_relationship_targets(xml, rel_type) catch { InvalidXml(msg~) => raise InvalidXml(msg~) } } ///| -fn parse_relationship_targets_exact_types( +fn parse_internal_relationship_targets_exact_types( xml : StringView, rel_types : ArrayView[String], ) -> Map[String, String] raise XlsxError { - @ooxml.parse_relationship_targets_by_types(xml, rel_types) catch { + @ooxml.parse_internal_relationship_targets_by_types(xml, rel_types) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + +///| +fn parse_external_relationship_targets_exact( + xml : StringView, + rel_type : StringView, +) -> Map[String, String] raise XlsxError { + @ooxml.parse_external_relationship_targets(xml, rel_type) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } +} + +///| +fn parse_external_relationship_targets_exact_types( + xml : StringView, + rel_types : ArrayView[String], +) -> Map[String, String] raise XlsxError { + @ooxml.parse_external_relationship_targets_by_types(xml, rel_types) catch { InvalidXml(msg~) => raise InvalidXml(msg~) } } @@ -28,7 +48,26 @@ fn parse_relationship_targets_exact_types( /// Reads both ISO Strict and ECMA Transitional aliases for relationship types /// from the Office relationship family. Microsoft extension relationships use /// their own namespaces and therefore remain exact matches. -fn parse_relationship_targets( +fn parse_internal_relationship_targets( + xml : StringView, + rel_type : StringView, +) -> Map[String, String] raise XlsxError { + let requested = rel_type.to_owned() + match requested.strip_prefix(transitional_office_relationship_prefix) { + Some(suffix) => { + let strict_type = strict_office_relationship_prefix + suffix.to_owned() + parse_internal_relationship_targets_exact_types(xml, [ + requested, strict_type, + ]) + } + None => parse_internal_relationship_targets_exact(xml, requested) + } +} + +///| +/// Reads only explicitly external targets, accepting the same Strict aliases +/// as internal Office relationships. +fn parse_external_relationship_targets( xml : StringView, rel_type : StringView, ) -> Map[String, String] raise XlsxError { @@ -36,9 +75,11 @@ fn parse_relationship_targets( match requested.strip_prefix(transitional_office_relationship_prefix) { Some(suffix) => { let strict_type = strict_office_relationship_prefix + suffix.to_owned() - parse_relationship_targets_exact_types(xml, [requested, strict_type]) + parse_external_relationship_targets_exact_types(xml, [ + requested, strict_type, + ]) } - None => parse_relationship_targets_exact(xml, requested) + None => parse_external_relationship_targets_exact(xml, requested) } } @@ -167,7 +208,7 @@ fn resolve_workbook_rel_target(target : StringView) -> String raise XlsxError { } ///| -test "ooxml_rels: parse_relationship_targets filters by Type and maps Id->Target" { +test "ooxml_rels: internal targets filter by Type and map Id->Target" { let xml = #| #| @@ -176,7 +217,7 @@ test "ooxml_rels: parse_relationship_targets filters by Type and maps Id->Target #| #| #| - let targets = parse_relationship_targets(xml, "t1") + let targets = parse_internal_relationship_targets(xml, "t1") debug_inspect( targets, content=( @@ -185,6 +226,27 @@ test "ooxml_rels: parse_relationship_targets filters by Type and maps Id->Target ) } +///| +test "ooxml_rels: target modes separate package parts from external resources" { + let xml = + #| + #| + #| + #| + debug_inspect( + parse_internal_relationship_targets(xml, "t1"), + content=( + #|{ "inside": "worksheets/sheet1.xml" } + ), + ) + debug_inspect( + parse_external_relationship_targets(xml, "t1"), + content=( + #|{ "outside": "https://example.invalid/sheet.xml" } + ), + ) +} + ///| test "ooxml_rels: Office relationship types accept Strict aliases" { let transitional = transitional_office_relationship_prefix + "worksheet" @@ -194,12 +256,12 @@ test "ooxml_rels: Office relationship types accept Strict aliases" { #| #| #| - let targets = parse_relationship_targets(xml, transitional) + let targets = parse_internal_relationship_targets(xml, transitional) assert_eq(targets.get("strict"), Some("worksheets/strict.xml")) assert_eq(targets.get("transitional"), Some("worksheets/transitional.xml")) // Asking for an already-Strict URI remains exact and does not broaden to // the Transitional family in the opposite direction. - let strict_only = parse_relationship_targets(xml, strict) + let strict_only = parse_internal_relationship_targets(xml, strict) assert_eq(strict_only.length(), 1) assert_eq(strict_only.get("strict"), Some("worksheets/strict.xml")) } diff --git a/xlsx/picture_ops_test.mbt b/xlsx/picture_ops_test.mbt index a7f56373..f5184410 100644 --- a/xlsx/picture_ops_test.mbt +++ b/xlsx/picture_ops_test.mbt @@ -54,6 +54,38 @@ test "picture from bytes add and write" { inspect(types_xml.contains("Extension=\"png\""), content="true") } +///| +test "external drawing relationships are never dereferenced as package parts" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + workbook.add_picture_from_bytes("Sheet1", "A1", [1, 2, 3], ".png") + let archive = @zip.read(@xlsx.write(workbook)) + let rels_path = "xl/worksheets/_rels/sheet1.xml.rels" + let rels = match archive.get(rels_path) { + Some(value) => @encoding/utf8.decode(value) + None => fail("missing worksheet relationships") + } + let internal_target = "Target=\"../drawings/drawing1.xml\"" + assert_true(rels.contains(internal_target)) + assert_true( + archive.replace( + rels_path, + @encoding/utf8.encode( + rels.replace( + old=internal_target, + new=internal_target + " TargetMode=\"External\"", + ), + ), + ), + ) + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(@zip.write(archive)), + ) catch { + error => Err(error) + } + assert_true(result is Err(@xlsx.XlsxError::InvalidXml(_))) +} + ///| test "picture dimension parsing (bmp/tiff/ico)" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read.mbt b/xlsx/read.mbt index b183fcf3..75e88f0c 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3165,7 +3165,9 @@ fn workbook_related_part_path( fallback : StringView, part_names : Map[String, String], ) -> String? raise XlsxError { - let targets = parse_relationship_targets(workbook_rels, relationship_type) + let targets = parse_internal_relationship_targets( + workbook_rels, relationship_type, + ) match first_existing_workbook_rel_target_path(targets, workbook_part, part_names) { Some(path) => Some(path) @@ -3208,7 +3210,12 @@ fn read_zip_archive_core_unchecked( let decode = (value : BytesView) => { let decoded = decode_source(value) read_budget.charge_work(decoded.length()) - canonicalize_xlsx_xml(decoded) + let canonical = canonicalize_xlsx_xml( + decoded, + max_output_chars=limits.max_xml_part_bytes, + ) + read_budget.charge_work(canonical.length()) + canonical } let workbook_xml_part_path = actual_archive_part_path( part_names, @@ -3220,7 +3227,11 @@ fn read_zip_archive_core_unchecked( } let workbook_source_xml = decode_source(workbook_bytes) read_budget.charge_work(workbook_source_xml.length()) - let workbook_xml = canonicalize_xlsx_xml(workbook_source_xml) + let workbook_xml = canonicalize_xlsx_xml( + workbook_source_xml, + max_output_chars=limits.max_xml_part_bytes, + ) + read_budget.charge_work(workbook_xml.length()) let workbook_rels_path = actual_archive_part_path( part_names, rels_path_for_part(workbook_xml_part_path), @@ -3405,7 +3416,9 @@ fn read_zip_archive_core_unchecked( let slicer_cache_defs = parse_slicer_cache_definitions( workbook_rels, workbook_xml_part_path, part_names, archive, decode, ) - let vba_targets = parse_relationship_targets(workbook_rels, rel_vba_project) + let vba_targets = parse_internal_relationship_targets( + workbook_rels, rel_vba_project, + ) let vba_path = match first_existing_workbook_rel_target_path( vba_targets, workbook_xml_part_path, part_names, @@ -3433,10 +3446,10 @@ fn read_zip_archive_core_unchecked( } None => () } - let worksheet_targets = parse_relationship_targets( + let worksheet_targets = parse_internal_relationship_targets( workbook_rels, rel_worksheet, ) - let chartsheet_targets = parse_relationship_targets( + let chartsheet_targets = parse_internal_relationship_targets( workbook_rels, rel_chartsheet, ) let seen_sheet_parts : Set[String] = Set([]) @@ -3562,7 +3575,9 @@ fn read_zip_archive_core_unchecked( if rels_xml == "" { raise InvalidXml(msg="slicer relationship missing") } - let rel_targets = parse_relationship_targets(rels_xml, rel_slicer) + let rel_targets = parse_internal_relationship_targets( + rels_xml, rel_slicer, + ) let parsed : Array[ParsedSlicerPartEntry] = [] let seen_paths : Map[String, Bool] = Map([]) for rel_id in slicer_rel_ids { @@ -3591,7 +3606,9 @@ fn read_zip_archive_core_unchecked( if rels_xml == "" { raise InvalidXml(msg="picture relationship missing") } - let image_targets = parse_relationship_targets(rels_xml, rel_image) + let image_targets = parse_internal_relationship_targets( + rels_xml, rel_image, + ) let target = match image_targets.get(rel_id) { Some(value) => value None => raise InvalidXml(msg="picture target missing") @@ -3627,7 +3644,7 @@ fn read_zip_archive_core_unchecked( [] } else { let rel_targets = if needs_hyperlink_rels { - parse_relationship_targets(rels_xml, rel_hyperlink) + parse_external_relationship_targets(rels_xml, rel_hyperlink) } else { Map([]) } @@ -3668,7 +3685,9 @@ fn read_zip_archive_core_unchecked( let tables = if table_part_ids.length() == 0 { [] } else { - let rel_targets = parse_relationship_targets(rels_xml, rel_table) + let rel_targets = parse_internal_relationship_targets( + rels_xml, rel_table, + ) let parsed_tables : Array[Table] = [] for rel_id in table_part_ids { let target = match rel_targets.get(rel_id) { @@ -3691,7 +3710,7 @@ fn read_zip_archive_core_unchecked( let rel_targets = if rels_xml == "" { Map([]) } else { - parse_relationship_targets(rels_xml, rel_pivot_table) + parse_internal_relationship_targets(rels_xml, rel_pivot_table) } let pivot_rel_ids : Array[String] = [] if pivot_part_ids.length() > 0 { @@ -3728,7 +3747,7 @@ fn read_zip_archive_core_unchecked( None => raise MissingPart(path=pivot_rels_path) } let pivot_rels_xml = decode(pivot_rels_bytes) - let cache_targets = parse_relationship_targets( + let cache_targets = parse_internal_relationship_targets( pivot_rels_xml, rel_pivot_cache, ) let cache_path = match @@ -3763,7 +3782,7 @@ fn read_zip_archive_core_unchecked( archive.get(actual_relationship_part_path(cache_path, part_names)) { Some(value) => { let cache_rels_xml = decode(value) - let record_targets = parse_relationship_targets( + let record_targets = parse_internal_relationship_targets( cache_rels_xml, rel_pivot_cache_records, ) let record_path = match @@ -3809,7 +3828,7 @@ fn read_zip_archive_core_unchecked( } let vml_targets = if legacy_drawing_rel_id is Some(_) || legacy_drawing_hf_rel_id is Some(_) { - parse_relationship_targets(rels_xml, rel_vml) + parse_internal_relationship_targets(rels_xml, rel_vml) } else { Map([]) } @@ -3868,7 +3887,9 @@ fn read_zip_archive_core_unchecked( let comments = if rels_xml == "" { [] } else { - let comment_targets = parse_relationship_targets(rels_xml, rel_comments) + let comment_targets = parse_internal_relationship_targets( + rels_xml, rel_comments, + ) let parsed_comments : Array[Comment] = [] for _rel_id, target in comment_targets { let comment_path = actual_relationship_target_path( @@ -3940,7 +3961,7 @@ fn read_zip_archive_core_unchecked( if rels_xml == "" { raise InvalidXml(msg="drawing relationship missing") } - let drawing_targets = parse_relationship_targets( + let drawing_targets = parse_internal_relationship_targets( rels_xml, rel_drawing, ) let drawing_target = match drawing_targets.get(rel_id) { @@ -4073,7 +4094,7 @@ fn read_zip_archive_core_unchecked( None => raise MissingPart(path=chartsheet_rels_path) } let chartsheet_rels_xml = decode(chartsheet_rels_bytes) - let drawing_targets = parse_relationship_targets( + let drawing_targets = parse_internal_relationship_targets( chartsheet_rels_xml, rel_drawing, ) let drawing_target = match drawing_targets.get(drawing_rel) { @@ -4091,7 +4112,7 @@ fn read_zip_archive_core_unchecked( None => raise MissingPart(path=drawing_rels_path) } let drawing_rels_xml = decode(drawing_rels_bytes) - let chart_targets = parse_relationship_targets( + let chart_targets = parse_internal_relationship_targets( drawing_rels_xml, rel_chart, ) let chart_path = match diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 0c1e6ba8..40973aaf 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -171,12 +171,12 @@ fn parse_drawing_images( let image_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_relationship_targets(drawing_rels_xml, rel_image) + parse_internal_relationship_targets(drawing_rels_xml, rel_image) } let hyperlink_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_relationship_targets(drawing_rels_xml, rel_hyperlink) + parse_external_relationship_targets(drawing_rels_xml, rel_hyperlink) } let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { @@ -331,11 +331,7 @@ fn parse_drawing_images( match hyperlink_targets.get(rel_id.to_string()) { Some(value) => { hyperlink = value - if hyperlink.contains("://") { - hyperlink_type = External - } else { - hyperlink_type = Location - } + hyperlink_type = External } None => () } @@ -381,7 +377,7 @@ fn parse_drawing_charts( let chart_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_relationship_targets(drawing_rels_xml, rel_chart) + parse_internal_relationship_targets(drawing_rels_xml, rel_chart) } let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index 6ed31a50..4cfed9b5 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -317,6 +317,39 @@ test "semantic read limits reject XML items and derived column expansion" { } } +///| +test "comments and CDATA cannot inject worksheet cells or column expansion" { + let bytes = bounded_workbook_fixture() + let fake_columns = "".repeat(8) + let fake_cell = + #|phantom + let worksheet = "\n" + + "\n" + + " \n" + + " \n" + + " real" + + "&value]]>\n" + + "\n" + let parsed = @xlsx.read( + replace_xlsx_part(bytes, "xl/worksheets/sheet1.xml", worksheet), + limits=@xlsx.ReadLimits::with_values(max_parser_items=1000), + ) + debug_inspect(parsed.get_cell_value("Data", "A1"), content="Some(\"real\")") + debug_inspect( + parsed.get_cell_value("Data", "B1"), + content="Some(\"&value\")", + ) + debug_inspect(parsed.get_cell_value("Data", "Z99"), content="None") +} + ///| test "semantic read limits charge sqref token materialization" { let bytes = bounded_workbook_fixture() diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index 6ce384b7..610f457f 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -19,7 +19,9 @@ fn workbook_part_from_root_relationships( } let office_document_type = transitional_office_relationship_prefix + "officeDocument" - let targets = parse_relationship_targets(root_rels_xml, office_document_type) + let targets = parse_internal_relationship_targets( + root_rels_xml, office_document_type, + ) if targets.length() > 1 { raise InvalidXml(msg="root relationships contain multiple workbook targets") } @@ -120,6 +122,28 @@ test "read package parts: workbook path follows the root relationship" { ) } +///| +test "read package parts: external workbook relationship is never resolved as a part" { + let archive = @zip.Archive::new() + let content_types = + #| + #| + #| + let root_relationships = + #| + #| + #| + archive.add(content_types_part_path, @encoding/utf8.encode(content_types)) + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + try resolve_workbook_xml_part_path_for_test(archive) catch { + InvalidXml(msg~) => + inspect(msg, content="root workbook relationship missing") + _ => fail("unexpected external-root error") + } noraise { + _ => fail("expected external root relationship rejection") + } +} + ///| test "read package parts: root target must have a workbook content type" { let archive = @zip.Archive::new() diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index b21ad6b7..b3177605 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -346,7 +346,9 @@ fn parse_slicer_cache_definitions( decode : (BytesView) -> String raise XlsxError, ) -> Map[String, SlicerCacheDef] raise XlsxError { let defs : Map[String, SlicerCacheDef] = Map([]) - let targets = parse_relationship_targets(workbook_rels_xml, rel_slicer_cache) + let targets = parse_internal_relationship_targets( + workbook_rels_xml, rel_slicer_cache, + ) for _rel_id, target in targets { let cache_path = actual_archive_part_path( part_names, diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 854c504e..5b002a1c 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -2,7 +2,10 @@ /// Converts namespace-qualified OOXML element names into the lexical form used /// by the existing bounded feature parsers. The source is fully namespace- and /// structure-validated before rewriting; only names change, never values. -fn canonicalize_xlsx_xml(xml : StringView) -> String raise XlsxError { +fn canonicalize_xlsx_xml( + xml : StringView, + max_output_chars? : Int = default_max_xml_part_bytes, +) -> String raise XlsxError { @ooxml.canonicalize_xml_expanded_names( xml, [transitional_spreadsheet_namespace, strict_spreadsheet_namespace], @@ -10,6 +13,7 @@ fn canonicalize_xlsx_xml(xml : StringView) -> String raise XlsxError { (transitional_relationship_attribute_namespace, "r"), (strict_relationship_attribute_namespace, "r"), ], + max_output_chars~, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) } diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index 140eb62e..c2b14215 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -49,7 +49,7 @@ fn parse_rich_value_images( } let rel_targets = match archive.get("xl/richData/_rels/richValueRel.xml.rels") { - Some(value) => parse_relationship_targets(decode(value), rel_image) + Some(value) => parse_internal_relationship_targets(decode(value), rel_image) None => Map([]) } let web_blip_ids = match archive.get("xl/richData/rdRichValueWebImage.xml") { @@ -58,7 +58,7 @@ fn parse_rich_value_images( } let web_rel_targets = match archive.get("xl/richData/_rels/rdRichValueWebImage.xml.rels") { - Some(value) => parse_relationship_targets(decode(value), rel_image) + Some(value) => parse_internal_relationship_targets(decode(value), rel_image) None => Map([]) } Some({ From 2c634c844c2b694d81f7a78131d2c0ea399d6fda Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 02:39:05 +0800 Subject: [PATCH 030/150] fix: honor cancellation throughout XLSX output --- office/cmd/office/main.mbt | 2 + office/cmd/office/read_package.mbt | 28 +++++++- office/cmd/office/xlsx_read_model.mbt | 32 +++++++-- office/cmd/office/xlsx_read_output.mbt | 90 +++++++++++++++++++++++--- office/cmd/office/xlsx_read_wbtest.mbt | 52 +++++++++++++++ 5 files changed, 184 insertions(+), 20 deletions(-) diff --git a/office/cmd/office/main.mbt b/office/cmd/office/main.mbt index 49a64eb1..afa50f17 100644 --- a/office/cmd/office/main.mbt +++ b/office/cmd/office/main.mbt @@ -85,9 +85,11 @@ async fn run_identify(matches : @argparse.Matches) -> Unit { ), ) } + check_office_read_cancelled(() => @async.is_being_cancelled()) let format = @lib.detect_format(file, data) catch { error => raise CliFailure(identify_error(error, file)) } + check_office_read_cancelled(() => @async.is_being_cancelled()) if matches.flags.get_or_default("json", false) { println( @lib.output_success( diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 7a4fc672..8a7228e6 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -32,6 +32,29 @@ struct OfficeReadPackage { archive : @zip.Archive } +///| +fn check_office_read_cancelled(cancelled : () -> Bool) -> Unit raise CliFailure { + if cancelled() { + raise CliFailure( + @lib.protocol_error("office.cancelled", "Office read was cancelled"), + ) + } +} + +///| +fn detect_office_archive_format( + file : String, + archive : @zip.Archive, + cancelled : () -> Bool, +) -> @lib.DocumentFormat raise CliFailure { + check_office_read_cancelled(cancelled) + let format = @lib.detect_archive_format(file, archive) catch { + error => raise CliFailure(identify_error(error, file)) + } + check_office_read_cancelled(cancelled) + format +} + ///| fn office_read_format_hint( file : String, @@ -99,6 +122,7 @@ async fn read_office_package( file : String, cancelled? : () -> Bool = () => false, ) -> OfficeReadPackage { + check_office_read_cancelled(cancelled) let expected = office_read_format_hint(file) let data = read_bounded_file( file, @@ -119,9 +143,7 @@ async fn read_office_package( error if cancelled() => raise error error => raise office_zip_read_failure(error, file, expected) } - let format = @lib.detect_archive_format(file, archive) catch { - error => raise CliFailure(identify_error(error, file)) - } + let format = detect_office_archive_format(file, archive, cancelled) { file, format, archive } } diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index fc1a81ce..73e00c59 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -126,6 +126,15 @@ fn xlsx_resource_failure( format_resource_failure("xlsx", resource, limit, actual?) } +///| +/// Checks the retained command cancellation callback at synchronous +/// projection boundaries and reports the stable protocol error. +fn XlsxProjection::checkpoint(self : XlsxProjection) -> Unit raise CliFailure { + if (self.cancelled)() { + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + } +} + ///| fn xlsx_scan_resource_failure( resource : String, @@ -338,6 +347,7 @@ fn XlsxProjection::check_scan_rect( self : XlsxProjection, rect : XlsxRect, ) -> Unit raise CliFailure { + self.checkpoint() let actual = rect.cell_count() if actual > self.max_scan_cells.to_int64() { raise xlsx_scan_resource_failure( @@ -416,17 +426,13 @@ fn XlsxSheetTarget::shared_formula_master( worksheet : @xlsx.Worksheet, shared_index : UInt, ) -> @xlsx.SharedFormulaMaster raise CliFailure { + projection.checkpoint() if !self.shared_formulas_indexed { - if (projection.cancelled)() { - raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") - } let masters = xlsx_call(projection.file, () => { worksheet.shared_formula_masters() }) for index, formula in masters { - if (projection.cancelled)() { - raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") - } + projection.checkpoint() self.shared_formula_masters[index] = formula } self.shared_formulas_indexed = true @@ -450,6 +456,7 @@ fn resolve_xlsx_selector( projection : XlsxProjection, input : String, ) -> XlsxResolvedSelector raise CliFailure { + projection.checkpoint() let selector = @lib.parse_selector(input) catch { error => raise selector_failure(error) } @@ -511,6 +518,7 @@ fn xlsx_scan_regions( projection : XlsxProjection, under : XlsxResolvedSelector?, ) -> Array[XlsxScanRegion] raise CliFailure { + projection.checkpoint() let regions : Array[XlsxScanRegion] = [] match under { Some(Cell(sheet, rect, ..)) | Some(Range(sheet, rect, ..)) => @@ -526,6 +534,7 @@ fn xlsx_scan_regions( } Some(Workbook(..)) | None => for sheet in projection.sheets { + projection.checkpoint() match sheet.content { Worksheet(worksheet) => match xlsx_used_rect(worksheet) { @@ -538,6 +547,7 @@ fn xlsx_scan_regions( } let mut total = 0L for region in regions { + projection.checkpoint() total += region.rect.cell_count() if total > projection.max_scan_cells.to_int64() { raise xlsx_scan_resource_failure( @@ -598,6 +608,7 @@ fn XlsxFormatBudget::charge_style( projection : XlsxProjection, style_id : Int, ) -> Unit raise CliFailure { + projection.checkpoint() let style = xlsx_call(projection.file, () => { projection.workbook.get_style(style_id) }) @@ -613,6 +624,7 @@ fn XlsxFormatBudget::charge_style( ) } self.used += cost + projection.checkpoint() } ///| @@ -669,7 +681,8 @@ fn xlsx_cell_formula( row : Int, column : Int, ) -> (String?, Bool) raise CliFailure { - match + projection.checkpoint() + let result = match xlsx_call(projection.file, () => { worksheet.get_cell_formula_info_rc(row, column) }) { @@ -701,6 +714,8 @@ fn xlsx_cell_formula( info.cached_value_present, ) } + projection.checkpoint() + result } ///| @@ -730,6 +745,7 @@ fn xlsx_cell_snapshot( let style_id = xlsx_call(projection.file, () => { worksheet.effective_style_id_rc(row, column) }) + projection.checkpoint() let snapshot : XlsxCellSnapshot = { path: canonical_xlsx_cell_path(sheet.name, reference), reference, @@ -756,6 +772,7 @@ fn XlsxCellSnapshot::ensure_formatted( projection : XlsxProjection, budget : XlsxScanBudget, ) -> Unit raise CliFailure { + projection.checkpoint() if self.formatted_ready { return } @@ -784,6 +801,7 @@ fn XlsxCellSnapshot::ensure_formatted( } self.formatted = formatted self.formatted_ready = true + projection.checkpoint() } ///| diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 5d4e1d9b..e1886f30 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -25,18 +25,24 @@ fn xlsx_sheet_state_name(state : @xlsx.SheetState) -> String { ///| fn xlsx_conditional_format_range_count( - file : String, + projection : XlsxProjection, worksheet : @xlsx.Worksheet, ) -> Int raise CliFailure { - let formats = xlsx_call(file, () => worksheet.get_conditional_formats()) + projection.checkpoint() + let formats = xlsx_call(projection.file, () => { + worksheet.get_conditional_formats() + }) let mut count = 0 for ranges, _ in formats { + projection.checkpoint() for part in ranges.split(" ") { + projection.checkpoint() if part.length() > 0 { count += 1 } } } + projection.checkpoint() count } @@ -46,6 +52,7 @@ fn xlsx_sheet_summary_json( sheet : XlsxSheetTarget, metadata : XlsxMetadataBudget, ) -> Json raise CliFailure { + projection.checkpoint() metadata.charge_item([sheet.name, sheet.path]) let fields : Map[String, Json] = { "path": Json::string(sheet.path), @@ -79,12 +86,19 @@ fn xlsx_sheet_summary_json( None => () } let merges = worksheet.merged_cells() + projection.checkpoint() let tables = worksheet.tables() + projection.checkpoint() let comments = worksheet.comments() + projection.checkpoint() let hyperlinks = worksheet.get_hyperlink_cells() + projection.checkpoint() let validations = worksheet.data_validations() + projection.checkpoint() let pivots = worksheet.pivot_tables() + projection.checkpoint() let slicers = worksheet.slicers() + projection.checkpoint() fields["counts"] = Json::object({ "merges": Json::number(merges.length().to_double()), "tables": Json::number(tables.length().to_double()), @@ -95,24 +109,28 @@ fn xlsx_sheet_summary_json( "hyperlinks": Json::number(hyperlinks.length().to_double()), "data_validations": Json::number(validations.length().to_double()), "conditional_format_ranges": Json::number( - xlsx_conditional_format_range_count(projection.file, worksheet).to_double(), + xlsx_conditional_format_range_count(projection, worksheet).to_double(), ), "slicers": Json::number(slicers.length().to_double()), }) } } + projection.checkpoint() Json::object(fields) } ///| fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { + projection.checkpoint() let metadata = XlsxMetadataBudget::new() let sheets : Array[Json] = [] for sheet in projection.sheets { + projection.checkpoint() sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) } let defined_names : Array[Json] = [] for defined in projection.workbook.get_defined_names() { + projection.checkpoint() metadata.charge_item([ defined.name, defined.refers_to, @@ -148,6 +166,7 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { }), } if !projection.sheets.is_empty() { + projection.checkpoint() let active_index = projection.workbook.get_active_sheet_index() guard projection.sheets.get(active_index) is Some(sheet) else { raise xlsx_cli_failure( @@ -165,6 +184,7 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { "index": Json::number((sheet.index + 1).to_double()), }) } + projection.checkpoint() Json::object(fields) } @@ -173,12 +193,15 @@ fn xlsx_outline_payload( projection : XlsxProjection, max_output_chars : Int, ) -> BoundedOfficePayload raise CliFailure { - bounded_office_payload( + projection.checkpoint() + let payload = bounded_office_payload( xlsx_outline_data(projection), [], max_output_chars, "xlsx", ) + projection.checkpoint() + payload } ///| @@ -238,8 +261,10 @@ fn xlsx_styles_json( projection : XlsxProjection, cells : Array[XlsxCellSnapshot], ) -> Json raise CliFailure { + projection.checkpoint() let fields : Map[String, Json] = Map([]) for cell in cells { + projection.checkpoint() if cell.style_id != 0 { let key = cell.style_id.to_string() if !fields.contains(key) { @@ -250,9 +275,24 @@ fn xlsx_styles_json( } } } + projection.checkpoint() Json::object(fields) } +///| +fn xlsx_cells_json( + projection : XlsxProjection, + cells : Array[XlsxCellSnapshot], +) -> Array[Json] raise CliFailure { + let values : Array[Json] = [] + for cell in cells { + projection.checkpoint() + values.push(xlsx_cell_json(cell)) + } + projection.checkpoint() + values +} + ///| fn xlsx_cell_text(cell : XlsxCellSnapshot) -> String? { match cell.formatted { @@ -311,6 +351,7 @@ fn collect_xlsx_cell_page( limit : Int, format_work_maximum? : Int = xlsx_cli_max_format_work_units, ) -> XlsxCellPage raise CliFailure { + projection.checkpoint() let scan = XlsxScanBudget::new( projection.max_scan_cells, format_work_maximum~, @@ -319,6 +360,7 @@ fn collect_xlsx_cell_page( let cells : Array[XlsxCellSnapshot] = [] let mut matched = 0 for region in regions { + projection.checkpoint() for row in region.rect.row_lo..<=region.rect.row_hi { for column in region.rect.col_lo..<=region.rect.col_hi { let cell = xlsx_cell_snapshot( @@ -341,6 +383,7 @@ fn collect_xlsx_cell_page( } } } + projection.checkpoint() { cells, matched_total: matched, scanned_cells: scan.scanned, offset, limit } } @@ -363,6 +406,7 @@ fn xlsx_get_data( projection : XlsxProjection, resolved : XlsxResolvedSelector, ) -> Json raise CliFailure { + projection.checkpoint() match resolved { Workbook(path~) => { let outline = xlsx_outline_data(projection) @@ -448,7 +492,7 @@ fn xlsx_get_data( "stability": Json::string("snapshot-relative"), "parent": Json::string(sheet.path), "reference": Json::string(rect.range_reference(projection.file)), - "cells": Json::array(page.cells.map(xlsx_cell_json)), + "cells": Json::array(xlsx_cells_json(projection, page.cells)), "styles": xlsx_styles_json(projection, page.cells), "scanned_cells": Json::number(page.scanned_cells.to_double()), "returned": Json::number(page.cells.length().to_double()), @@ -463,12 +507,15 @@ fn xlsx_get_payload( resolved : XlsxResolvedSelector, max_output_chars : Int, ) -> BoundedOfficePayload raise CliFailure { - bounded_office_payload( + projection.checkpoint() + let payload = bounded_office_payload( xlsx_get_data(projection, resolved), [], max_output_chars, "xlsx", ) + projection.checkpoint() + payload } ///| @@ -478,6 +525,7 @@ fn xlsx_text_data( offset : Int, limit : Int, ) -> Json raise CliFailure { + projection.checkpoint() let page = collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), @@ -487,6 +535,7 @@ fn xlsx_text_data( ) let entries : Array[Json] = [] for cell in page.cells { + projection.checkpoint() guard xlsx_cell_text(cell) is Some(text) else { continue } entries.push( Json::object({ @@ -496,6 +545,7 @@ fn xlsx_text_data( }), ) } + projection.checkpoint() let fields = xlsx_page_fields(page) fields["schema"] = Json::string(@lib.SCHEMA_XLSX_TEXT) fields["file"] = Json::string(projection.file) @@ -516,12 +566,15 @@ fn xlsx_text_payload( limit : Int, max_output_chars : Int, ) -> BoundedOfficePayload raise CliFailure { - bounded_office_payload( + projection.checkpoint() + let payload = bounded_office_payload( xlsx_text_data(projection, under, offset, limit), [], max_output_chars, "xlsx", ) + projection.checkpoint() + payload } ///| @@ -530,6 +583,7 @@ fn xlsx_query_data( under : XlsxResolvedSelector?, spec : XlsxQuerySpec, ) -> Json raise CliFailure { + projection.checkpoint() let page = collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), @@ -542,7 +596,7 @@ fn xlsx_query_data( fields["file"] = Json::string(projection.file) fields["format"] = Json::string("xlsx") fields["selector"] = Json::string(spec.selector) - fields["matches"] = Json::array(page.cells.map(xlsx_cell_json)) + fields["matches"] = Json::array(xlsx_cells_json(projection, page.cells)) fields["styles"] = xlsx_styles_json(projection, page.cells) match under { Some(value) => fields["under"] = Json::string(value.path()) @@ -558,12 +612,15 @@ fn xlsx_query_payload( spec : XlsxQuerySpec, max_output_chars : Int, ) -> BoundedOfficePayload raise CliFailure { - bounded_office_payload( + projection.checkpoint() + let payload = bounded_office_payload( xlsx_query_data(projection, under, spec), [], max_output_chars, "xlsx", ) + projection.checkpoint() + payload } ///| @@ -571,6 +628,7 @@ fn xlsx_outline_human( projection : XlsxProjection, maximum : Int, ) -> String raise CliFailure { + projection.checkpoint() let output = DocxHumanWriter::new(maximum, format="xlsx") output.begin_line() output.write_plain("xlsx: ") @@ -579,6 +637,7 @@ fn xlsx_outline_human( output.write_plain("workbook\t/xlsx/workbook\t") output.write_plain("\{projection.sheets.length()} sheets") for sheet in projection.sheets { + projection.checkpoint() output.begin_line() output.write_terminal_safe(sheet.path) output.write_char('\t') @@ -597,17 +656,21 @@ fn xlsx_outline_human( } } } + projection.checkpoint() output.finish() } ///| fn xlsx_page_human( + projection : XlsxProjection, page : XlsxCellPage, include_kind : Bool, maximum : Int, ) -> String raise CliFailure { + projection.checkpoint() let output = DocxHumanWriter::new(maximum, format="xlsx") for cell in page.cells { + projection.checkpoint() output.begin_line() output.write_terminal_safe(cell.path) if include_kind { @@ -625,6 +688,7 @@ fn xlsx_page_human( output.write_plain( "# matched \{page.matched_total}; returned \{page.cells.length()}; scanned \{page.scanned_cells}; offset \{page.offset}", ) + projection.checkpoint() output.finish() } @@ -634,14 +698,17 @@ fn xlsx_get_human( resolved : XlsxResolvedSelector, maximum : Int, ) -> String raise CliFailure { + projection.checkpoint() match resolved { Workbook(..) => xlsx_outline_human(projection, maximum) Sheet(sheet) => { + projection.checkpoint() let output = DocxHumanWriter::new(maximum, format="xlsx") output.begin_line() output.write_terminal_safe(sheet.path) output.write_char('\t') output.write_terminal_safe(sheet.name, single_line=true) + projection.checkpoint() output.finish() } Cell(sheet, rect, path~) => { @@ -659,10 +726,11 @@ fn xlsx_get_human( details=Json::object({ "selector": Json::string(path) }), ) } - xlsx_page_human(page, false, maximum) + xlsx_page_human(projection, page, false, maximum) } Range(sheet, rect, ..) => xlsx_page_human( + projection, collect_xlsx_cell_page( projection, [{ sheet, rect }], @@ -685,6 +753,7 @@ fn xlsx_text_human( maximum : Int, ) -> String raise CliFailure { xlsx_page_human( + projection, collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), @@ -705,6 +774,7 @@ fn xlsx_query_human( maximum : Int, ) -> String raise CliFailure { xlsx_page_human( + projection, collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 5e71b8e0..5fa785bc 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -52,6 +52,58 @@ test "XLSX CLI preserves parser cancellation and checkpoints cell scans" { assert_eq(scan_error.code, "office.cancelled") } +///| +test "XLSX post-projection metadata, styles, and human output observe cancellation" { + let projection = { ..xlsx_read_test_projection(), cancelled: () => true } + guard projection.sheets[0].content is Worksheet(worksheet) else { + fail("expected worksheet fixture") + } + for + action in [ + () => ignore(xlsx_outline_data(projection)), + () => ignore(xlsx_conditional_format_range_count(projection, worksheet)), + () => ignore(xlsx_styles_json(projection, [])), + () => ignore(xlsx_outline_human(projection, 10_000)), + () => { + ignore( + xlsx_page_human( + projection, + { + cells: [], + matched_total: 0, + scanned_cells: 0, + offset: 0, + limit: 0, + }, + false, + 10_000, + ), + ) + }, + ] { + let error = xlsx_cli_error(() => action()) + assert_eq(error.code, "office.cancelled") + } +} + +///| +test "Office format detection probes cancellation before and after validation" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Data")) + let source = xlsx_bounded_test_source("cancel.xlsx", @xlsx.write(workbook)) + let mut checks = 0 + let error = xlsx_cli_error(() => { + ignore( + detect_office_archive_format(source.file, source.archive, () => { + checks += 1 + checks > 1 + }), + ) + }) + assert_eq(error.code, "office.cancelled") + assert_eq(checks, 2) +} + ///| fn xlsx_strict_test_bytes() -> Bytes raise { let archive = @zip.Archive::new() From 3a97af743999977337a33d57f50575aa31ff940e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 02:39:39 +0800 Subject: [PATCH 031/150] docs: refresh OOXML relationship interface --- ooxml/pkg.generated.mbti | 10 +++++++--- ooxml/read_parse.mbt | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index bd48ee07..f4c94bfe 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -8,7 +8,7 @@ import { // Values pub fn attr_value(StringView, StringView) -> String? raise ParseXmlError -pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)]) -> String raise ParseXmlError +pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)], max_output_chars? : Int) -> String raise ParseXmlError pub fn decode_xml_attribute(StringView) -> String raise ParseXmlError @@ -20,9 +20,13 @@ pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> Str pub fn find_part_content_type(StringView, StringView) -> String? raise ParseXmlError -pub fn parse_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError +pub fn parse_external_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError -pub fn parse_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError +pub fn parse_external_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError + +pub fn parse_internal_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError + +pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError pub fn xml_needs_escape(StringView, attr~ : Bool) -> Bool diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index e61c18ab..59fbd8a7 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -252,6 +252,7 @@ pub fn parse_internal_relationship_targets_by_types( } ///| +/// Reads internal relationship targets whose decoded `Type` is `rel_type`. pub fn parse_internal_relationship_targets( xml : StringView, rel_type : StringView, @@ -270,6 +271,8 @@ pub fn parse_external_relationship_targets_by_types( } ///| +/// Reads explicitly external relationship targets whose decoded `Type` is +/// `rel_type`. pub fn parse_external_relationship_targets( xml : StringView, rel_type : StringView, From c3b5d3a806d1616775a17bb23bec46e07e2d0f47 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 02:44:44 +0800 Subject: [PATCH 032/150] ci: normalize XLSX smoke fixture --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8496e6f1..9cc4335c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -283,7 +283,16 @@ jobs: and .error.details.suggestions == ["xlsx"]' office-wasm-error.json # XLSX structured reads share the validated bounded archive and use # canonical name-keyed paths even when the input selector is positional. - xlsx_fixture=fixtures/excelize/test/Book1.xlsx + # The third-party Book1 fixture has intersecting shared-formula ranges; + # normalize that unrelated invalid metadata through the raw transaction + # surface before exercising the fixture's original contents. + xlsx_source=fixtures/excelize/test/Book1.xlsx + xlsx_sheet="$RUNNER_TEMP/office-book1-sheet2.xml" + xlsx_sheet_valid="$RUNNER_TEMP/office-book1-sheet2-valid.xml" + xlsx_fixture="$RUNNER_TEMP/office-Book1-valid.xlsx" + moon run --target wasm office/cmd/office -- raw read "$xlsx_source" /Sheet2 --output "$xlsx_sheet" >/dev/null + sed -e 's/ref="F11:H11"/ref="F11:F11"/' -e 's/<\/f>/<\/f>/' "$xlsx_sheet" >"$xlsx_sheet_valid" + moon run --target wasm office/cmd/office -- raw replace "$xlsx_source" /Sheet2 --xml-file "$xlsx_sheet_valid" --out "$xlsx_fixture" --json >/dev/null xo=$(moon run --target wasm office/cmd/office -- outline "$xlsx_fixture" --json 2>office-wasm.err) || { echo "office XLSX outline failed; stderr:"; cat office-wasm.err; exit 1; } echo "$xo" | jq -e '.success == true and .data.schema == "office.xlsx.outline/1" From 6ac6fcf6225269264986ffae6ddf5829256bb220 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 04:14:39 +0800 Subject: [PATCH 033/150] fix: isolate XLSX core XML from extensions --- ooxml/pkg.generated.mbti | 4 + ooxml/start_tag_scanner.mbt | 158 +++++++++++++++++++++++++++++++++++ xlsx/ooxml_utils.mbt | 11 +++ xlsx/read.mbt | 48 ++++++++--- xlsx/read_namespace_test.mbt | 46 ++++++++++ xlsx/read_workbook_xml.mbt | 21 +++++ xlsx/read_worksheet_xml.mbt | 13 ++- xlsx/read_xml_namespaces.mbt | 91 ++++++++++++++++++++ 8 files changed, 379 insertions(+), 13 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index f4c94bfe..71837fcb 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -28,6 +28,8 @@ pub fn parse_internal_relationship_targets(StringView, StringView) -> Map[String pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError +pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String) -> String? raise ParseXmlError + pub fn xml_needs_escape(StringView, attr~ : Bool) -> Bool // Errors @@ -60,6 +62,8 @@ pub fn XmlStartTagScanner::local_name(Self) -> StringView pub fn XmlStartTagScanner::namespace_uri(Self) -> StringView pub fn XmlStartTagScanner::new(StringView) -> Self pub fn XmlStartTagScanner::next(Self) -> Bool raise ParseXmlError +pub fn XmlStartTagScanner::parent_local_name(Self) -> StringView? +pub fn XmlStartTagScanner::parent_namespace_uri(Self) -> String? // Type aliases diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 0723c7b8..2310ce9f 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -53,6 +53,9 @@ pub struct XmlStartTagScanner { priv open_name_starts : Array[Int] priv open_name_ends : Array[Int] priv open_name_rewrites : Array[Bool] + priv open_local_name_starts : Array[Int] + priv open_local_name_ends : Array[Int] + priv open_namespace_uris : Array[String] priv rewrite_element_namespaces : Array[String] priv rewrite_attribute_prefixes : Map[String, String] priv rewrite_output : StringBuilder @@ -69,6 +72,10 @@ pub struct XmlStartTagScanner { priv mut current_tag_end : Int priv mut current_depth : Int priv mut current_namespace_uri : String + priv capture_element_namespaces : Array[String] + priv capture_element_local_name : String + priv mut capture_start : Int? + priv mut capture_depth : Int } ///| @@ -789,6 +796,8 @@ fn xml_start_tag_scanner_with_rewrites( attribute_prefixes : Map[String, String], canonicalize_markup : Bool, rewrite_output_limit : Int, + capture_element_namespaces? : Array[String] = [], + capture_element_local_name? : String = "", ) -> XmlStartTagScanner { { xml, @@ -802,6 +811,9 @@ fn xml_start_tag_scanner_with_rewrites( open_name_starts: [], open_name_ends: [], open_name_rewrites: [], + open_local_name_starts: [], + open_local_name_ends: [], + open_namespace_uris: [], rewrite_element_namespaces: element_namespaces, rewrite_attribute_prefixes: attribute_prefixes, rewrite_output: StringBuilder::new(), @@ -818,6 +830,10 @@ fn xml_start_tag_scanner_with_rewrites( current_tag_end: 0, current_depth: 0, current_namespace_uri: "", + capture_element_namespaces, + capture_element_local_name, + capture_start: None, + capture_depth: 0, } } @@ -868,9 +884,21 @@ fn XmlStartTagScanner::consume_end_tag( None => () } } + match self.capture_start { + Some(capture_start) => + if self.capture_depth == self.depth { + self.record_rewrite(capture_start, tag_end + 1, "") + self.capture_start = None + self.capture_depth = 0 + } + None => () + } ignore(self.open_name_starts.pop()) ignore(self.open_name_ends.pop()) ignore(self.open_name_rewrites.pop()) + ignore(self.open_local_name_starts.pop()) + ignore(self.open_local_name_ends.pop()) + ignore(self.open_namespace_uris.pop()) self.pop_namespace_depth(self.depth) self.depth = self.depth - 1 if self.depth == 0 { @@ -1008,8 +1036,28 @@ pub fn XmlStartTagScanner::next( self.current_name_end = name_end self.current_tag_end = tag_end self.current_depth = scope_depth + if self.capture_start is None && + self.capture_element_local_name != "" && + self.local_name() == self.capture_element_local_name { + for wanted in self.capture_element_namespaces { + if xml_string_equals_view(wanted, self.current_namespace_uri) { + self.capture_start = Some(start) + self.capture_depth = scope_depth + break + } + } + } self.index = tag_end + 1 if self_closing { + match self.capture_start { + Some(capture_start) => + if self.capture_depth == scope_depth { + self.record_rewrite(capture_start, tag_end + 1, "") + self.capture_start = None + self.capture_depth = 0 + } + None => () + } self.pending_namespace_pop = Some(scope_depth) if self.depth == 0 { self.root_closed = true @@ -1019,6 +1067,9 @@ pub fn XmlStartTagScanner::next( self.open_name_starts.push(name_start) self.open_name_ends.push(name_end) self.open_name_rewrites.push(rewrite_name) + self.open_local_name_starts.push(self.current_name_start) + self.open_local_name_ends.push(self.current_name_end) + self.open_namespace_uris.push(self.current_namespace_uri) } return true } @@ -1052,6 +1103,71 @@ pub fn XmlStartTagScanner::depth(self : XmlStartTagScanner) -> Int { self.current_depth } +///| +/// Returns the local name of the current element's parent, or `None` for the +/// document element. +pub fn XmlStartTagScanner::parent_local_name( + self : XmlStartTagScanner, +) -> StringView? { + let index = self.current_depth - 2 + if index < 0 || index >= self.open_local_name_starts.length() { + None + } else { + Some( + self.xml[self.open_local_name_starts[index]:self.open_local_name_ends[index]], + ) + } +} + +///| +/// Returns the namespace URI of the current element's parent, or `None` for +/// the document element. +pub fn XmlStartTagScanner::parent_namespace_uri( + self : XmlStartTagScanner, +) -> String? { + let index = self.current_depth - 2 + if index < 0 || index >= self.open_namespace_uris.length() { + None + } else { + Some(self.open_namespace_uris[index]) + } +} + +///| +/// Removes every outermost subtree selected by expanded element name. `None` +/// means the document was valid but contained no matching subtree. Namespace +/// resolution and nesting are handled by the same bounded scanner used for +/// canonicalization, so markup-looking text cannot create a removal span. +pub fn remove_xml_expanded_name_subtrees( + xml : StringView, + element_namespaces : ArrayView[String], + local_name : String, +) -> String? raise ParseXmlError { + if local_name == "" { + raise InvalidXml(msg="XML removal local name is empty") + } + let namespaces : Array[String] = [] + namespaces.append(element_namespaces) + let scanner = xml_start_tag_scanner_with_rewrites( + xml, + [], + Map([]), + false, + xml.length(), + capture_element_namespaces=namespaces, + capture_element_local_name=local_name, + ) + while scanner.next() { + + } + if scanner.rewrite_count == 0 { + return None + } + scanner.charge_rewrite_output(xml.length() - scanner.rewrite_cursor) + scanner.rewrite_output.write_view(xml[scanner.rewrite_cursor:]) + Some(scanner.rewrite_output.to_string()) +} + ///| /// Validates `xml` and rewrites selected expanded names into lexical forms /// expected by a namespace-oblivious parser. Elements in `element_namespaces` @@ -1161,6 +1277,48 @@ test "start tag scanner resolves namespaces and decodes attributes once" { assert_false(scanner.next()) } +///| +test "start tag scanner exposes expanded parent names" { + let scanner = XmlStartTagScanner::new( + ( + #| + ), + ) + assert_true(scanner.next()) + assert_true(scanner.parent_local_name() is None) + assert_true(scanner.next()) + assert_true(scanner.parent_local_name() is Some("root")) + debug_inspect(scanner.parent_namespace_uri(), content="Some(\"urn:sheet\")") + assert_true(scanner.next()) + assert_true(scanner.parent_local_name() is Some("items")) + assert_true(scanner.next()) + assert_true(scanner.parent_local_name() is Some("items")) + assert_true(scanner.next()) + assert_true(scanner.parent_local_name() is Some("item")) +} + +///| +test "expanded-name subtree removal preserves text and other namespaces" { + let xml = + #|]]> + debug_inspect( + remove_xml_expanded_name_subtrees(xml, ["urn:sheet"], "extLst"), + content=( + #|Some("]]>") + ), + ) + let without_extension = + #| + debug_inspect( + remove_xml_expanded_name_subtrees( + without_extension, + ["urn:sheet"], + "extLst", + ), + content="None", + ) +} + ///| test "start tag scanner rejects undeclared and duplicate expanded attributes" { for diff --git a/xlsx/ooxml_utils.mbt b/xlsx/ooxml_utils.mbt index 74ec51ec..d4ef38a1 100644 --- a/xlsx/ooxml_utils.mbt +++ b/xlsx/ooxml_utils.mbt @@ -106,6 +106,9 @@ fn extract_tag_body( Some(pos) => pos None => raise InvalidXml(msg="tag open not closed") } + if xml_str[start + 1:open_end].trim().has_suffix("/") { + return Some("") + } let body_start = open_end + 1 let end = match xml_str.find(close_tag) { Some(pos) => pos @@ -233,6 +236,14 @@ test "ooxml_utils: extract_tag_body_from extracts body" { debug_inspect(extract_tag_body_from(xml, "a"), content="Some(\"hello\")") } +///| +test "ooxml_utils: extract_tag_body accepts an empty self-closing container" { + debug_inspect( + extract_tag_body("", "items"), + content="Some(\"\")", + ) +} + ///| test "ooxml_utils: extract_tag_body_from returns None when missing" { let xml = "" diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 75e88f0c..2ef44d98 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2870,8 +2870,12 @@ fn parse_col_dimensions( style_count : Int, ) -> Map[Int, ColDimension] raise XlsxError { let dims : Map[Int, ColDimension] = Map([]) + let body = match extract_tag_body(xml, "cols") { + Some(value) => value + None => return dims + } let mut first = true - for chunk in xml.split(" value None => raise MissingPart(path=sheet_path) } - let sheet_xml = decode(sheet_bytes) + let sheet_source_xml = decode_source(sheet_bytes) + read_budget.charge_work(sheet_source_xml.length()) + let sheet_xml = canonicalize_xlsx_xml( + sheet_source_xml, + max_output_chars=limits.max_xml_part_bytes, + ) + read_budget.charge_work(sheet_xml.length()) + let sheet_core_xml = canonicalize_xlsx_worksheet_core( + sheet_source_xml, + sheet_xml, + limits.max_xml_part_bytes, + ) + if sheet_core_xml != sheet_xml { + read_budget.charge_work(sheet_core_xml.length()) + } let ( cells, merged_cells, @@ -3515,31 +3533,36 @@ fn read_zip_archive_core_unchecked( dimension_ref, picture_rel_id, ) = parse_worksheet( - sheet_xml, + sheet_core_xml, shared_strings, style_count, budget=read_budget, ) - let hyperlink_elements = parse_hyperlink_elements(sheet_xml) - let table_part_ids = parse_table_part_ids(sheet_xml) - let pivot_part_ids = parse_pivot_table_part_ids(sheet_xml) + let hyperlink_elements = parse_hyperlink_elements(sheet_core_xml) + let table_part_ids = parse_table_part_ids(sheet_core_xml) + let pivot_part_ids = parse_pivot_table_part_ids(sheet_core_xml) let slicer_rel_ids = parse_sheet_slicer_rel_ids(sheet_xml) - let drawing_rel_id = parse_legacy_drawing_rel_id(sheet_xml, "drawing") + let drawing_rel_id = parse_legacy_drawing_rel_id( + sheet_core_xml, "drawing", + ) let legacy_drawing_rel_id = parse_legacy_drawing_rel_id( - sheet_xml, "legacyDrawing", + sheet_core_xml, "legacyDrawing", ) let legacy_drawing_hf_rel_id = parse_legacy_drawing_rel_id( - sheet_xml, "legacyDrawingHF", + sheet_core_xml, "legacyDrawingHF", ) let sparkline_groups = parse_sparkline_groups(sheet_xml) let data_validations = parse_data_validations( - sheet_xml, + sheet_core_xml, budget=read_budget, ) for dv in parse_data_validations_x14(sheet_xml, budget=read_budget) { data_validations.push(dv) } let conditional_formats = parse_conditional_formats( + // Base data-bar rules carry their x14 correlation id in a nested + // extLst, so this parser needs the preserved extension view. Cell and + // row parsing above still uses the extension-free structural view. sheet_xml, budget=read_budget, ) @@ -3548,7 +3571,10 @@ fn read_zip_archive_core_unchecked( } let x14_data_bars = parse_x14_data_bars(sheet_xml, budget=read_budget) let unknown_ext_blocks = parse_unknown_worksheet_ext_blocks(sheet_xml) - let ignored_errors = parse_ignored_errors(sheet_xml, budget=read_budget) + let ignored_errors = parse_ignored_errors( + sheet_core_xml, + budget=read_budget, + ) let mut needs_hyperlink_rels = false for link in hyperlink_elements { if link.r_id is Some(_) { diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 440b9d56..8ba91895 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -126,3 +126,49 @@ test "main-part names, relationship targets, and ids decode XML entities" { content="Some(\"qualified-value\")", ) } + +///| +test "SpreadsheetML-looking extension payload cannot create sheets or cells" { + for strict in [false, true] { + let bytes = namespace_qualified_xlsx_fixture(strict, false) + let archive = @zip.read(bytes) + guard archive.get("xl/workbook.xml") is Some(workbook_bytes) else { + fail("missing workbook fixture part") + } + let workbook_xml = @encoding/utf8.decode(workbook_bytes) catch { + _ => fail("workbook fixture is not UTF-8") + } + let workbook_with_decoy = workbook_xml.replace( + old=" ", + new=" \n ", + ) + if workbook_with_decoy == workbook_xml { + fail("workbook injection marker was not found") + } + archive.add("xl/workbook.xml", @encoding/utf8.encode(workbook_with_decoy)) + guard archive.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { + fail("missing worksheet fixture part") + } + let worksheet_xml = @encoding/utf8.decode(worksheet_bytes) catch { + _ => fail("worksheet fixture is not UTF-8") + } + let worksheet_with_decoy = worksheet_xml.replace( + old="", + new="phantom", + ) + if worksheet_with_decoy == worksheet_xml { + fail("worksheet injection marker was not found") + } + archive.add( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode(worksheet_with_decoy), + ) + let parsed = @xlsx.read(@zip.write(archive)) + debug_inspect(parsed.get_sheet_list(), content="[\"Qualified\"]") + debug_inspect(parsed.get_cell_value("Qualified", "Z99"), content="None") + debug_inspect( + parsed.get_cell_value("Qualified", "A1"), + content="Some(\"qualified-value\")", + ) + } +} diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index 134cc425..a6ca3fb5 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -44,8 +44,20 @@ fn parse_workbook_sheets( ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] let scanner = @ooxml.XmlStartTagScanner::new(xml) + let mut workbook_namespace : String? = None while workbook_scanner_next(scanner) { let namespace_uri = scanner.namespace_uri() + if scanner.depth() == 1 { + if scanner.local_name() != "workbook" || + ( + namespace_uri != transitional_spreadsheet_namespace && + namespace_uri != strict_spreadsheet_namespace + ) { + raise InvalidXml(msg="workbook document element is invalid") + } + workbook_namespace = Some(namespace_uri.to_owned()) + continue + } if ( namespace_uri != transitional_spreadsheet_namespace && namespace_uri != strict_spreadsheet_namespace @@ -53,6 +65,15 @@ fn parse_workbook_sheets( scanner.local_name() != "sheet" { continue } + let correct_parent = scanner.parent_local_name() is Some("sheets") && + scanner.parent_namespace_uri() == workbook_namespace + if scanner.depth() != 3 || + namespace_uri.to_owned() != workbook_namespace.unwrap_or("") || + !correct_parent { + // SpreadsheetML elements outside the schema-defined sheet collection + // are extension payload, not logical workbook tabs. + continue + } if sheets.length() >= max_sheets { raise ResourceLimitExceeded( kind="workbook_sheets", diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index c82565a1..bc6b9ca7 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -1,3 +1,11 @@ +///| +fn worksheet_sheet_data_body(xml : StringView) -> String raise XlsxError { + match extract_tag_body(xml, "sheetData") { + Some(value) => value + None => "" + } +} + ///| fn parse_worksheet( xml : StringView, @@ -22,8 +30,9 @@ fn parse_worksheet( String?, ) raise XlsxError { let cells : Array[Cell] = [] + let sheet_data = worksheet_sheet_data_body(xml) let mut first = true - for chunk in xml.split(" Unit raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(xml) + let mut worksheet_namespace : String? = None + let mut sheet_data_seen = false + while workbook_scanner_next(scanner) { + let local_name = scanner.local_name() + let namespace_uri = scanner.namespace_uri() + if scanner.depth() == 1 { + if local_name != "worksheet" || + ( + namespace_uri != transitional_spreadsheet_namespace && + namespace_uri != strict_spreadsheet_namespace + ) { + raise InvalidXml(msg="worksheet document element is invalid") + } + worksheet_namespace = Some(namespace_uri.to_owned()) + continue + } + let dialect = worksheet_namespace.unwrap_or("") + if local_name == "sheetData" { + if namespace_uri != dialect || + scanner.depth() != 2 || + scanner.parent_local_name() != Some("worksheet") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet sheetData element is misplaced") + } + if sheet_data_seen { + raise InvalidXml(msg="worksheet has multiple sheetData elements") + } + sheet_data_seen = true + } else if local_name == "row" { + if namespace_uri != dialect || + scanner.depth() != 3 || + scanner.parent_local_name() != Some("sheetData") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet row element is misplaced") + } + } else if local_name == "c" { + if namespace_uri != dialect || + scanner.depth() != 4 || + scanner.parent_local_name() != Some("row") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet cell element is misplaced") + } + } else if local_name == "cols" { + if namespace_uri != dialect || + scanner.depth() != 2 || + scanner.parent_local_name() != Some("worksheet") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet cols element is misplaced") + } + } else if local_name == "col" { + if namespace_uri != dialect || + scanner.depth() != 3 || + scanner.parent_local_name() != Some("cols") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet col element is misplaced") + } + } + } +} + +///| +/// Produces the namespace-oblivious view used by core worksheet parsers. +/// Extension payload stays available in the full canonical view, but is +/// removed before core elements are validated and flattened. +fn canonicalize_xlsx_worksheet_core( + source : StringView, + full_canonical : String, + max_output_chars : Int, +) -> String raise XlsxError { + let without_extensions = @ooxml.remove_xml_expanded_name_subtrees( + source, + [transitional_spreadsheet_namespace, strict_spreadsheet_namespace], + "extLst", + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + } + match without_extensions { + Some(core_source) => { + validate_xlsx_worksheet_core(core_source) + canonicalize_xlsx_xml(core_source, max_output_chars~) + } + None => { + validate_xlsx_worksheet_core(source) + full_canonical + } + } +} + ///| test "XLSX XML canonicalization handles Strict elements and relationship attributes" { let source = From c44cdb08175ba52fd7be1a706473b150b0f17744 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 04:20:44 +0800 Subject: [PATCH 034/150] fix: bound XLSX read cancellation latency --- xlsx/io.mbt | 10 ++++-- xlsx/read.mbt | 62 ++++++++++++++++++++++---------- xlsx/read_limits_test.mbt | 24 +++++++++++++ zip/deflate.mbt | 61 ++++++++++++++++++++++++++++--- zip/reader.mbt | 75 ++++++++++++++++++++++++++++++++++----- 5 files changed, 199 insertions(+), 33 deletions(-) diff --git a/xlsx/io.mbt b/xlsx/io.mbt index 4b2df3d6..8aaa3f60 100644 --- a/xlsx/io.mbt +++ b/xlsx/io.mbt @@ -250,7 +250,7 @@ fn read_workbook_from_bytes( password } let resolved_options = options_with_password(options, resolved_password) - if resolved_password == "" { + let workbook = if resolved_password == "" { match io_context.charset_transcoder { Some(value) => read_zip_bytes( @@ -284,6 +284,7 @@ fn read_workbook_from_bytes( ) } } + checked_read_workbook(workbook, cancelled) } ///| @@ -311,6 +312,7 @@ pub async fn[R : @async/io.Reader] open_reader( () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) + @async.pause() workbook } @@ -323,11 +325,13 @@ pub async fn[R : @async/io.Reader] read_zip_reader( limits? : ReadLimits = ReadLimits::new(), transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook { - match transcoder { + let workbook = match transcoder { Some(value) => open_reader(reader, password~, options~, limits~, transcoder=value) None => open_reader(reader, password~, options~, limits~) } + @async.pause() + workbook } ///| @@ -365,6 +369,7 @@ pub async fn[R : @async/io.Reader] Workbook::read_zip_reader( () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) + @async.pause() workbook } @@ -398,6 +403,7 @@ pub async fn open_file( () => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) + @async.pause() workbook } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 2ef44d98..5466384c 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3218,6 +3218,7 @@ fn read_zip_archive_core_unchecked( decoded, max_output_chars=limits.max_xml_part_bytes, ) + read_budget.checkpoint() read_budget.charge_work(canonical.length()) canonical } @@ -3235,6 +3236,7 @@ fn read_zip_archive_core_unchecked( workbook_source_xml, max_output_chars=limits.max_xml_part_bytes, ) + read_budget.checkpoint() read_budget.charge_work(workbook_xml.length()) let workbook_rels_path = actual_archive_part_path( part_names, @@ -3507,12 +3509,14 @@ fn read_zip_archive_core_unchecked( sheet_source_xml, max_output_chars=limits.max_xml_part_bytes, ) + read_budget.checkpoint() read_budget.charge_work(sheet_xml.length()) let sheet_core_xml = canonicalize_xlsx_worksheet_core( sheet_source_xml, sheet_xml, limits.max_xml_part_bytes, ) + read_budget.checkpoint() if sheet_core_xml != sheet_xml { read_budget.charge_work(sheet_core_xml.length()) } @@ -4166,6 +4170,7 @@ fn read_zip_archive_core_unchecked( } } resolve_slicer_sources_after_read(workbook, slicer_cache_defs) + read_budget.checkpoint() workbook } @@ -4177,7 +4182,7 @@ fn read_zip_archive_core( cancelled : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { - match transcoder { + let workbook = match transcoder { Some(decode) => read_zip_archive_core_unchecked( archive, @@ -4197,6 +4202,17 @@ fn read_zip_archive_core( other => raise other } } + check_read_cancelled(cancelled) + workbook +} + +///| +fn checked_read_workbook( + workbook : Workbook, + cancelled : () -> Bool, +) -> Workbook raise XlsxError { + check_read_cancelled(cancelled) + workbook } ///| @@ -4210,7 +4226,7 @@ fn read_zip_bytes( check_read_cancelled(cancelled) let archive = read_limited_archive(bytes, limits, cancelled~) require_bounded_archive(archive, limits, cancelled~) - match transcoder { + let workbook = match transcoder { Some(value) => read_zip_archive_core( archive, @@ -4221,6 +4237,7 @@ fn read_zip_bytes( ) None => read_zip_archive_core(archive, options, limits, cancelled) } + checked_read_workbook(workbook, cancelled) } ///| @@ -4237,7 +4254,7 @@ pub fn read_bounded_archive( ) -> Workbook raise XlsxError { check_read_cancelled(cancelled) require_bounded_archive(archive, limits, cancelled~) - match transcoder { + let workbook = match transcoder { Some(value) => read_zip_archive_core( archive, @@ -4248,6 +4265,7 @@ pub fn read_bounded_archive( ) None => read_zip_archive_core(archive, options, limits, cancelled) } + checked_read_workbook(workbook, cancelled) } ///| @@ -4268,32 +4286,39 @@ pub fn read( if options.password != "" { match transcoder { Some(value) => - return read_with_password( - bytes, - options.password, - options~, - limits~, - cancelled~, - transcoder=value, + return checked_read_workbook( + read_with_password( + bytes, + options.password, + options~, + limits~, + cancelled~, + transcoder=value, + ), + cancelled, ) None => - return read_with_password( - bytes, - options.password, - options~, - limits~, - cancelled~, + return checked_read_workbook( + read_with_password( + bytes, + options.password, + options~, + limits~, + cancelled~, + ), + cancelled, ) } } if is_encrypted_package(bytes) { raise EncryptedPackage } - match transcoder { + let workbook = match transcoder { Some(value) => read_zip_bytes(bytes, options~, limits~, cancelled~, transcoder=value) None => read_zip_bytes(bytes, options~, limits~, cancelled~) } + checked_read_workbook(workbook, cancelled) } ///| @@ -4312,7 +4337,7 @@ pub fn read_with_password( check_read_cancelled(cancelled) check_source_package_size(bytes, limits) let resolved_options = options_with_password(options, password) - if is_encrypted_package(bytes) { + let workbook = if is_encrypted_package(bytes) { let archive = decrypt_encrypted_archive( bytes, password, @@ -4346,6 +4371,7 @@ pub fn read_with_password( read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) } } + checked_read_workbook(workbook, cancelled) } ///| diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index 4cfed9b5..b6675ec4 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -465,6 +465,30 @@ test "read cancellation callback stops semantic preflight" { inspect(result is Err(@xlsx.ReadCancelled), content="true") } +///| +test "read completion observes cancellation at its final checkpoint" { + let bytes = bounded_workbook_fixture() + let baseline_checks = [0] + ignore( + @xlsx.read(bytes, cancelled=() => { + baseline_checks[0] += 1 + false + }), + ) + let checks = [0] + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(bytes, cancelled=() => { + checks[0] += 1 + checks[0] >= baseline_checks[0] + }), + ) catch { + error => Err(error) + } + assert_true(baseline_checks[0] > 1) + assert_eq(checks[0], baseline_checks[0]) + assert_true(result is Err(@xlsx.ReadCancelled)) +} + ///| test "logical XML limit follows a worksheet relationship with a non-XML suffix" { let workbook = @xlsx.Workbook::new() diff --git a/zip/deflate.mbt b/zip/deflate.mbt index 9c87dd21..1c9f9e63 100644 --- a/zip/deflate.mbt +++ b/zip/deflate.mbt @@ -4,11 +4,31 @@ priv struct BitReader { mut byte_pos : Int mut bit_buf : UInt mut bit_count : Int + cancelled : () -> Bool + mut next_cancel_byte : Int } ///| -fn BitReader::new(bytes : BytesView) -> BitReader { - { bytes, byte_pos: 0, bit_buf: 0, bit_count: 0 } +fn BitReader::new( + bytes : BytesView, + cancelled? : () -> Bool = () => false, +) -> BitReader { + { + bytes, + byte_pos: 0, + bit_buf: 0, + bit_count: 0, + cancelled, + next_cancel_byte: 4096, + } +} + +///| +fn BitReader::checkpoint_input(self : BitReader) -> Unit raise ZipError { + if self.byte_pos >= self.next_cancel_byte { + check_zip_cancelled(self.cancelled) + self.next_cancel_byte = self.byte_pos + 4096 + } } ///| @@ -24,6 +44,7 @@ fn BitReader::ensure_bits(self : BitReader, count : Int) -> Unit raise ZipError self.bit_buf = self.bit_buf | (byte << self.bit_count) self.bit_count = self.bit_count + 8 self.byte_pos = self.byte_pos + 1 + self.checkpoint_input() } } @@ -49,12 +70,13 @@ fn BitReader::drop_bits(self : BitReader, count : Int) -> Unit raise ZipError { /// consuming any. Near the end of input fewer bits may remain; missing high /// bits are zero-padded, and `self.bit_count` reflects how many bits are /// actually available. `count` must be in `0..=15`. -fn BitReader::peek_bits(self : BitReader, count : Int) -> UInt { +fn BitReader::peek_bits(self : BitReader, count : Int) -> UInt raise ZipError { while self.bit_count < count && self.byte_pos < self.bytes.length() { let byte = self.bytes[self.byte_pos].to_uint() self.bit_buf = self.bit_buf | (byte << self.bit_count) self.bit_count = self.bit_count + 8 self.byte_pos = self.byte_pos + 1 + self.checkpoint_input() } self.bit_buf & bit_mask(count) } @@ -603,7 +625,7 @@ fn deflate_decode_to_output( cancelled : () -> Bool, ) -> Unit raise ZipError { check_zip_cancelled(cancelled) - let reader = BitReader::new(bytes) + let reader = BitReader::new(bytes, cancelled~) for state = false { if state { break @@ -641,6 +663,7 @@ fn deflate_decode_to_output( if reader.consumed_bytes() != bytes.length() { raise UnsupportedFeature(msg="trailing bytes after deflate stream") } + check_zip_cancelled(cancelled) } ///| @@ -658,7 +681,9 @@ fn deflate_decode( let exact_size = sizing.length() let out = FixedByteOutput::allocated(exact_size) deflate_decode_to_output(bytes, out, cancelled) - out.finish() + let decoded = out.finish() + check_zip_cancelled(cancelled) + decoded } ///| @@ -773,6 +798,32 @@ fn wb_deflate_bit_stream( output.finish() } +///| +test "deflate cancellation follows compressed input through empty blocks" { + let block_count = 5000 + let stream = wb_deflate_bit_stream(writer => { + for index in 0.. { + checks[0] += 1 + checks[0] >= 3 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 3) + _ => fail("expected compressed-input cancellation") + } noraise { + _ => fail("expected compressed-input cancellation") + } +} + ///| test "deflate wb: direct guard error branches" { let reader = BitReader::new(b"") diff --git a/zip/reader.mbt b/zip/reader.mbt index ee35c348..86e8f38d 100644 --- a/zip/reader.mbt +++ b/zip/reader.mbt @@ -114,6 +114,27 @@ fn Reader::skip(self : Reader, len : Int) -> Unit raise ZipError { self.pos = end } +///| +/// Copies a borrowed ZIP slice into owned storage with bounded cancellation +/// latency. Large stored entries and preservation records must not hide one +/// monolithic runtime copy behind a single preflight check. +fn copy_zip_bytes_cancellable( + bytes : BytesView, + cancelled : () -> Bool, +) -> Bytes raise ZipError { + check_zip_cancelled(cancelled) + let output = FixedArray::make(bytes.length(), b'\x00') + let mut offset = 0 + while offset < bytes.length() { + let end = (offset + 64 * 1024).min(bytes.length()) + output.blit_from_bytesview(offset, bytes[offset:end]) + offset = end + check_zip_cancelled(cancelled) + } + check_zip_cancelled(cancelled) + output.unsafe_reinterpret_as_bytes() +} + ///| fn u32_to_int(value : UInt) -> Int raise ZipError { let max_u32_int : UInt = 0x7FFFFFFF @@ -370,6 +391,7 @@ fn read_zip64_eocd( bytes : BytesView, eocd_offset : Int, preserved_source_budget : PreservedSourceBudget, + cancelled : () -> Bool, ) -> (UInt64, UInt64, UInt64, Int, Zip64TrailerTemplate) raise ZipError { let locator_offset = eocd_offset - 20 if locator_offset < 0 { @@ -426,9 +448,18 @@ fn read_zip64_eocd( central_offset, zip64_offset, { - end_record: bytes[zip64_offset:locator_offset].to_owned(), - locator: bytes[locator_offset:eocd_offset].to_owned(), - classic_end_record: bytes[eocd_offset:].to_owned(), + end_record: copy_zip_bytes_cancellable( + bytes[zip64_offset:locator_offset], + cancelled, + ), + locator: copy_zip_bytes_cancellable( + bytes[locator_offset:eocd_offset], + cancelled, + ), + classic_end_record: copy_zip_bytes_cancellable( + bytes[eocd_offset:], + cancelled, + ), }, ) } @@ -735,7 +766,7 @@ fn materialize_entry_data( ) _ => () } - record.compressed.to_owned() + copy_zip_bytes_cancellable(record.compressed, cancelled) } Deflate => { // Allow legitimately large declared entries, but keep a hard ceiling so a @@ -772,6 +803,7 @@ fn materialize_entry_data( msg="decoded data length does not match central size", ) } + check_zip_cancelled(cancelled) data } @@ -812,7 +844,10 @@ fn read_impl( let central_offset_raw = reader.read_u32_raw() let comment_len = reader.read_u16() preserved_source_budget.charge(comment_len) - let archive_comment = reader.read_bytes(comment_len).to_owned() + let archive_comment = copy_zip_bytes_cancellable( + reader.read_bytes(comment_len), + cancelled, + ) if disk_no != 0 || disk_start != 0 { raise UnsupportedFeature(msg="multi-disk zip not supported") } @@ -829,7 +864,7 @@ fn read_impl( zip64_trailer, ) = if needs_zip64 { let (zip_entries, zip_size, zip_offset, zip64_offset, zip64_trailer) = read_zip64_eocd( - bytes, eocd_offset, preserved_source_budget, + bytes, eocd_offset, preserved_source_budget, cancelled, ) if disk_entries != 0xFFFF && disk_entries.to_uint64() != zip_entries { raise UnsupportedFeature(msg="classic and zip64 entry count mismatch") @@ -948,7 +983,10 @@ fn read_impl( ) central_reader.skip(comment_len) preserved_source_budget.charge(central_reader.pos - central_record_start) - let central_record = central_bytes[central_record_start:central_reader.pos].to_owned() + let central_record = copy_zip_bytes_cancellable( + central_bytes[central_record_start:central_reader.pos], + cancelled, + ) if disk != 0 { raise UnsupportedFeature(msg="multi-disk zip not supported") } @@ -1184,7 +1222,10 @@ fn read_impl( crc32: meta.crc32, source_local_offset: Some(meta.local_header_offset), source: Some({ - local_record: bytes[meta.local_header_offset:record.end].to_owned(), + local_record: copy_zip_bytes_cancellable( + bytes[meta.local_header_offset:record.end], + cancelled, + ), local_header_length: record.data_start - meta.local_header_offset, central_record: meta.central_record, local_zip64_uncompressed_position: record.local_zip64_uncompressed_position, @@ -1198,6 +1239,7 @@ fn read_impl( source_unchanged: true, }) } + check_zip_cancelled(cancelled) archive } @@ -1267,6 +1309,23 @@ fn push_bytes(buf : Array[Byte], bytes : BytesView) -> Unit { } } +///| +test "ZIP owned copies observe cancellation between chunks" { + let source = Bytes::make(256 * 1024, b'x') + let checks = [0] + try + copy_zip_bytes_cancellable(source, () => { + checks[0] += 1 + checks[0] >= 3 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 3) + _ => fail("expected ZIP copy cancellation") + } noraise { + _ => fail("expected ZIP copy cancellation") + } +} + ///| test "zip read data descriptor" { let name : Bytes = b"a.txt" From e0e3741963ef9a7b40e6b3c4029aaa9d52fa5fda Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 04:24:14 +0800 Subject: [PATCH 035/150] fix: accept case-insensitive XLSX media types --- xlsx/read_namespace_test.mbt | 24 ++++++++++++++++++++++++ xlsx/read_package_parts.mbt | 17 ++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 8ba91895..4e71e048 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -127,6 +127,30 @@ test "main-part names, relationship targets, and ids decode XML entities" { ) } +///| +test "mixed-case workbook media type reads end to end" { + let archive = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard archive.get("[Content_Types].xml") is Some(content_type_bytes) else { + fail("missing content types fixture part") + } + let content_types = @encoding/utf8.decode(content_type_bytes) catch { + _ => fail("content types fixture is not UTF-8") + } + let original = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml" + let mixed = "Application/Vnd.Openxmlformats-Officedocument.Spreadsheetml.Sheet.Main+Xml" + let updated = content_types.replace(old=original, new=mixed) + if updated == content_types { + fail("workbook media type marker was not found") + } + archive.add("[Content_Types].xml", @encoding/utf8.encode(updated)) + let parsed = @xlsx.read(@zip.write(archive)) + debug_inspect(parsed.get_sheet_list(), content="[\"Qualified\"]") + debug_inspect( + parsed.get_cell_value("Qualified", "A1"), + content="Some(\"qualified-value\")", + ) +} + ///| test "SpreadsheetML-looking extension payload cannot create sheets or cells" { for strict in [false, true] { diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index 610f457f..ac7c412e 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -36,14 +36,29 @@ fn workbook_part_from_root_relationships( ///| fn workbook_content_type_is_supported(value : StringView) -> Bool { + let normalized = value.to_owned().to_lower() for candidate in workbook_content_type_candidates() { - if value == candidate { + if normalized == candidate.to_lower() { return true } } false } +///| +test "read package parts: workbook media types are case-insensitive" { + assert_true( + workbook_content_type_is_supported( + "Application/Vnd.Openxmlformats-Officedocument.Spreadsheetml.Sheet.Main+Xml", + ), + ) + assert_true( + workbook_content_type_is_supported( + "APPLICATION/VND.MS-EXCEL.SHEET.MACROENABLED.MAIN+XML", + ), + ) +} + ///| fn resolve_workbook_xml_part_path( archive : @zip.Archive, From fb382ebc79315f30394688fa9a1b1d1922916e6c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 04:32:36 +0800 Subject: [PATCH 036/150] fix: preserve XLSX query literal whitespace --- docs/agent-json-schemas.md | 14 ++- docs/office-xlsx-read.md | 11 ++- office/capabilities.mbt | 2 +- office/capabilities_test.mbt | 18 ++-- office/cmd/office/cram/cli.t | 16 +++- office/cmd/office/xlsx_query.mbt | 124 +++++++++++++++++++++++-- office/cmd/office/xlsx_read_wbtest.mbt | 63 +++++++++++++ 7 files changed, 218 insertions(+), 30 deletions(-) diff --git a/docs/agent-json-schemas.md b/docs/agent-json-schemas.md index 3266b3ca..16d68bf8 100644 --- a/docs/agent-json-schemas.md +++ b/docs/agent-json-schemas.md @@ -170,10 +170,12 @@ scanning it itself. Scans the sheet's used range by default, or the - `text=TEXT` — the string cell equals `TEXT`; `text~=TEXT` — contains it. A blank cell (an empty stored string) never matches a `text`/`string` predicate. -- A predicate argument (the `TEXT` after `=`/`~=`, the `N` after an operator) - is trimmed of surrounding whitespace and cannot contain `]` (which closes - the predicate). `text=`/`text~=`/`formula~=` require a non-empty value, and a - `value` bound must be a finite number (`NaN`/`Infinity` are rejected). +- A `TEXT` predicate argument is preserved exactly, including surrounding + whitespace. Balanced brackets are allowed unquoted. A value beginning with + `"` must be one complete JSON string, which represents whitespace-only text, + `]`, quotes, backslashes, or control characters without selector ambiguity. + `text=`/`text~=`/`formula~=` require a non-empty value. Numeric `value` + bounds are trimmed and must be finite (`NaN`/`Infinity` are rejected). - `query` is read-only. A malformed selector, an oversized scan, or a missing sheet fails with a non-zero exit and a one-line `error:` message; nothing is written. @@ -577,7 +579,9 @@ Predicates are literal and ANDed: `type=formula|number|string|bool|error`, `formula`, `formula~=TEXT`, `text=TEXT`, `text~=TEXT`, and numeric `value>NUMBER` comparisons (`>=`, `<`, `<=`, `=`, and `!=` are also supported). Substring predicates are guaranteed-linear and command-work -bounded. No regular expression or arbitrary expression is evaluated. +bounded. Text literals preserve whitespace exactly; a JSON-string literal can +represent closing brackets and escaped characters. No regular expression or +arbitrary expression is evaluated. ## Standalone `docx` CLI schemas diff --git a/docs/office-xlsx-read.md b/docs/office-xlsx-read.md index f5ce976f..3c4f3e9c 100644 --- a/docs/office-xlsx-read.md +++ b/docs/office-xlsx-read.md @@ -138,10 +138,17 @@ cell[type=formula][formula~=SUM] cell[formula~=SUM(Table1[[#Headers],[Amount]])] cell[type=number][value>=0] cell[text~=revenue] +cell[text= leading and trailing ] +cell[text=" "] +cell[text="bracket] and quote\""] ``` -Balanced brackets inside literal predicate values are part of the surrounding -predicate, which permits Excel structured references without quoting. +Unquoted `TEXT` is preserved exactly, including leading and trailing +whitespace. Balanced brackets inside an unquoted value are part of the +surrounding predicate, which permits Excel structured references without +quoting. A value that starts with `"` is one complete JSON string: use that +form for whitespace-only text, `]`, quotes, backslashes, control characters, +or other values that need escaping. Empty literals remain invalid. Regular expressions, arbitrary expressions, locale-sensitive matching, and the DOCX-only `--kind`, `--text`, `--id`, `--property`, and `--ignore-case` diff --git a/office/capabilities.mbt b/office/capabilities.mbt index 956736b1..3357f8d0 100644 --- a/office/capabilities.mbt +++ b/office/capabilities.mbt @@ -510,7 +510,7 @@ pub fn capability_commands() -> Array[CapabilityCommand] { inputs: [ capability_field("file", "path", true, "existing XLSX or DOCX package"), capability_field( - "selector", "xlsx-cell-selector", false, "XLSX only: cell followed by up to 16 bounded type, value, formula, or text predicates; default cell", + "selector", "xlsx-cell-selector", false, "XLSX only: cell followed by up to 16 bounded type, value, formula, or exact-whitespace text predicates with optional JSON-string escaping; default cell", ), capability_field( "under", "office.selector/1", false, "optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope", diff --git a/office/capabilities_test.mbt b/office/capabilities_test.mbt index 0ff836c0..7415e4bf 100644 --- a/office/capabilities_test.mbt +++ b/office/capabilities_test.mbt @@ -283,13 +283,13 @@ test "capability registry format filters remain truthful" { inspect( records, content=( - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"outline","summary":"Summarize bounded XLSX or DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office outline FILE","result_schema":"office.docx.outline/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"get","summary":"Resolve one canonical XLSX or DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office get FILE /docx/...","result_schema":"office.docx.element/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} - #|{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"format","name":"docx","aliases":["word"],"description":"WordprocessingML documents","selector":{"schema":"office.selector/1","root":"/docx","status":"read-resolved","examples":["/docx/body/p[1]/r[2]","/docx/comments/comment[id=\"7\"]"],"description":"bounded canonical resolution for outline, get, text, and declared query predicates"}} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"outline","summary":"Summarize bounded XLSX or DOCX structure using canonical selectors","usage":"office outline FILE [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office outline FILE","result_schema":"office.docx.outline/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.outline/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: number of elements in the bounded projection"},{"name":"counts","type":"object","required":false,"description":"DOCX only: deterministic structural counts across every story"},{"name":"stories","type":"array","required":false,"description":"DOCX only: story roots"},{"name":"headings","type":"array","required":false,"description":"DOCX only: bounded heading previews"},{"name":"styles_in_use","type":"array","required":false,"description":"DOCX only: deduplicated referenced styles"},{"name":"images","type":"array","required":false,"description":"DOCX only: image metadata"},{"name":"sections","type":"array","required":false,"description":"DOCX only: section boundaries and header/footer references"},{"name":"diagnostics","type":"array","required":false,"description":"DOCX only: reader and source diagnostics"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"get","summary":"Resolve one canonical XLSX or DOCX selector","usage":"office get FILE SELECTOR [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office get FILE /docx/...","result_schema":"office.docx.element/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"selector","type":"office.selector/1","required":true,"description":"canonical selector matching the validated package format"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.element/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"path","type":"office.selector/1","required":true,"description":"resolved canonical path"},{"name":"kind","type":"string","required":true,"description":"resolved workbook, sheet, coordinate, story, annotation, or element kind"},{"name":"role","type":"enum(story-root|annotation-collection|annotation-item|element)","required":false,"description":"DOCX only: projection role"},{"name":"stability","type":"enum(stable|snapshot-relative)","required":true,"description":"selector stability classification"},{"name":"source","type":"object","required":false,"description":"DOCX only: physical story source metadata"},{"name":"parent","type":"office.selector/1","required":false,"description":"canonical parent path; absent for projection roots"},{"name":"id","type":"string","required":false,"description":"annotation id when the resolved item carries one"},{"name":"children","type":"array","required":false,"description":"DOCX only: addressable direct children"},{"name":"properties","type":"object","required":false,"description":"DOCX only: declared formatting and element summary"},{"name":"metadata","type":"object","required":false,"description":"DOCX only: role-specific metadata"},{"name":"text","type":"string","required":false,"description":"DOCX only: bounded raw text projection"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"text","summary":"Extract bounded path-tagged XLSX cell or DOCX paragraph text","usage":"office text FILE [--under SELECTOR] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office text FILE [--under /docx/...]","result_schema":"office.docx.text/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based matching paragraph or cell offset; default 0"},{"name":"limit","type":"integer(0..10000)","required":false,"description":"maximum returned paragraphs or cells; default 2000"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.text/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"entries","type":"array(object{path,text,stability})","required":true,"description":"paragraphs in document order or cells in sheet/row-major order"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan matching paragraph or cell count"},{"name":"returned","type":"integer","required":true,"description":"number of returned entries"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"query","summary":"Run bounded deterministic predicates over XLSX cells or DOCX elements","usage":"office query FILE [CELL_SELECTOR] [--under SELECTOR] [--kind KIND] [--text TEXT] [--id ID] [--property NAME=VALUE]... [--ignore-case] [--offset N] [--limit N] [--max-elements N] [--max-output-chars N] [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"output_modes":["human","json"],"variants":[{"name":"docx","usage":"office query FILE [DOCX predicates]","result_schema":"office.docx.query/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"under","type":"office.selector/1","required":false,"description":"optional canonical DOCX subtree or XLSX workbook/sheet/range/cell scope"},{"name":"kind","type":"enum(body|header|footer|footnotes|endnotes|comments|note|comment|p|r|tbl|tr|tc|hyperlink|image)","required":false,"description":"DOCX only: exact kind; documented aliases normalize before matching"},{"name":"text","type":"literal-string(1..1048576 chars)","required":false,"description":"DOCX only: literal substring predicate; regular expressions are not accepted"},{"name":"id","type":"string(1..1048576 chars)","required":false,"description":"DOCX only: exact annotation id predicate"},{"name":"property","type":"array(NAME=VALUE)","required":false,"description":"DOCX only: up to 16 exact declared-property predicates"},{"name":"ignore-case","type":"boolean","required":false,"description":"DOCX only: locale-independent Unicode simple-case --text matching"},{"name":"offset","type":"integer(0..200000)","required":false,"description":"zero-based match offset; default 0"},{"name":"limit","type":"integer(0..1000)","required":false,"description":"maximum returned matches; default 100"},{"name":"max-elements","type":"integer(1..200000)","required":false,"description":"DOCX projection-node or XLSX cell scan ceiling; default 50000; XLSX effective hard ceiling 100000"},{"name":"max-output-chars","type":"integer(1..4194304)","required":false,"description":"successful stdout ceiling including trailing LF; failure envelopes are separate; default 1048576"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"schema","type":"literal(office.docx.query/1)","required":true,"description":"format-specific versioned result schema"},{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"resolved document format"},{"name":"filters","type":"object","required":false,"description":"DOCX only: normalized predicates applied by this query"},{"name":"matches","type":"array","required":true,"description":"deterministic document-order or sheet/row-major match records"},{"name":"matched_total","type":"integer","required":true,"description":"complete bounded-scan match count"},{"name":"returned","type":"integer","required":true,"description":"number of returned matches"},{"name":"truncated","type":"boolean","required":true,"description":"whether pagination omitted later matches"},{"name":"offset","type":"integer","required":true,"description":"applied zero-based match offset"},{"name":"limit","type":"integer","required":true,"description":"applied page-size ceiling"},{"name":"scanned_elements","type":"integer","required":false,"description":"DOCX only: projected element count"},{"name":"under","type":"office.selector/1","required":false,"description":"resolved canonical scope when one was requested"}],"constraints":["format=docx"],"actions":[],"output_modes":["human","json"]}]} + #|{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"raw","summary":"Inventory, read, and atomically edit validated OOXML parts","usage":"office raw ...","formats":["docx"],"aliases":[],"inputs":[{"name":"operation","type":"enum(list|read|replace|edit)","required":true,"description":"bounded raw OOXML operation"}],"outputs":[],"output_modes":["human","json","base64","file"],"variants":[{"name":"list","usage":"office raw list FILE [--json]","result_schema":"office.raw.inventory/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part_count","type":"integer","required":true,"description":"number of inventoried package parts"},{"name":"parts","type":"array(object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)})","required":true,"description":"bounded canonical part inventory records"}],"constraints":[],"actions":[],"output_modes":["human","json"]},{"name":"read","usage":"office raw read FILE PART [--json] [--base64 | --output FILE]","result_schema":"office.raw.part/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"/name when unambiguous, alias:/name, or part:/name"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"},{"name":"base64","type":"boolean","required":false,"description":"emit exact payload as base64"},{"name":"output","type":"path","required":false,"description":"create a file with the exact payload"}],"outputs":[{"name":"format","type":"literal(docx)","required":true,"description":"structurally verified package format"},{"name":"part","type":"object{name:path,content_type:string,kind:enum(xml|binary),size:integer,aliases:array(string)}","required":true,"description":"resolved package-part metadata"},{"name":"encoding","type":"enum(xml|base64|binary)","required":true,"description":"selected payload representation"},{"name":"content","type":"string","required":false,"description":"decoded XML text or base64 payload"},{"name":"output","type":"path","required":false,"description":"created payload destination"}],"constraints":["mutually-exclusive(base64,output)","binary-requires(base64|output)","output-create-mode(create-new)"],"actions":[],"output_modes":["human","json","base64","file"]},{"name":"replace","usage":"office raw replace FILE PART (--xml XML | --xml-file FILE) [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"xml","type":"utf8-xml","required":false,"description":"complete replacement XML document"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 replacement XML file"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["exactly-one(xml,xml-file)","overwrite-requires(out)","out-extension-must-match-input-format","transactional-publication"],"actions":[],"output_modes":["human","json"]},{"name":"edit","usage":"office raw edit FILE PART --path PATH --action ACTION [action arguments] [--namespace PREFIX=URI]... [--all] [--out FILE] [--dry-run] [--overwrite] [--json]","result_schema":"office.raw.result/1","inputs":[{"name":"file","type":"path","required":true,"description":"existing XLSX or DOCX package"},{"name":"part","type":"part-selector","required":true,"description":"existing XML part selector"},{"name":"path","type":"office.raw.path/1","required":true,"description":"bounded namespace-aware element selector"},{"name":"action","type":"enum(append|prepend|insert-before|insert-after|replace|remove|set-attribute)","required":true,"description":"bounded edit action"},{"name":"xml","type":"utf8-xml-element","required":false,"description":"one self-contained XML element"},{"name":"xml-file","type":"path","required":false,"description":"bounded UTF-8 XML element file"},{"name":"attribute","type":"qname","required":false,"description":"set-attribute target name"},{"name":"value","type":"string","required":false,"description":"set-attribute exact decoded value"},{"name":"namespace","type":"array(PREFIX=URI)","required":false,"description":"repeatable selector namespace overrides"},{"name":"all","type":"boolean","required":false,"description":"allow multiple bounded matches"},{"name":"out","type":"path","required":false,"description":"separate destination with the same .docx or .xlsx extension as the input"},{"name":"dry-run","type":"boolean","required":false,"description":"validate without publishing"},{"name":"overwrite","type":"boolean","required":false,"description":"replace an existing separate destination"},{"name":"json","type":"boolean","required":false,"description":"emit office.output/1 JSON"}],"outputs":[{"name":"change","type":"office.raw.change/1","required":true,"description":"validated raw mutation summary"},{"name":"transaction","type":"office.transaction/1","required":true,"description":"transaction validation and publication report"}],"constraints":["mutually-exclusive(xml,xml-file)","element-actions-require(exactly-one(xml,xml-file))","set-attribute-requires(attribute,value)","remove-forbids(xml,xml-file,attribute,value)","overwrite-requires(out)","flag-looking-values-require-attached-syntax","out-extension-must-match-input-format","transactional-publication"],"actions":[{"name":"append","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"prepend","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"insert-before","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"insert-after","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"replace","requires":["exactly-one(xml,xml-file)"],"forbids":["attribute","value"],"restrictions":[]},{"name":"remove","requires":[],"forbids":["xml","xml-file","attribute","value"],"restrictions":["path-must-not-select-document-element"]},{"name":"set-attribute","requires":["attribute","value"],"forbids":["xml","xml-file"],"restrictions":[]}],"output_modes":["human","json"]}]} ), ) } @@ -299,7 +299,7 @@ test "capability fingerprint and inventory ordering are deterministic" { inspect( @office.capability_fingerprint(), content=( - #|crc32:3f8b2b34 + #|crc32:9e349fa4 ), ) let first = @office.capabilities_data().stringify() @@ -315,7 +315,7 @@ test "capability operation queries do not leak unrelated records" { inspect( data.stringify(), content=( - #|{"schema":"office.capabilities/2","fingerprint":"crc32:3f8b2b34","records":[{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} + #|{"schema":"office.capabilities/2","fingerprint":"crc32:9e349fa4","records":[{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"identify","summary":"Identify a structurally valid XLSX or DOCX package","usage":"office identify [--json]","formats":["docx"],"aliases":[],"inputs":[{"name":"file","type":"path","required":true,"description":"path to an XLSX or DOCX package"},{"name":"json","type":"boolean","required":false,"description":"emit one office.output/1 JSON document"}],"outputs":[{"name":"file","type":"path","required":true,"description":"the input path exactly as supplied"},{"name":"format","type":"literal(docx)","required":true,"description":"the structurally verified package format"}],"output_modes":["human","json"],"variants":[]}],"format":"docx","operation":"identify"} ), ) } diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index 30eaccef..fdbff05a 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -17,7 +17,7 @@ and JSONL inventories without deferred PowerPoint or MCP entries. $ office.exe help | sed -n '1,8p' Office capability registry Schema: office.capabilities/2 - Fingerprint: crc32:3f8b2b34 + Fingerprint: crc32:9e349fa4 Formats: docx (aliases: word) — WordprocessingML documents xlsx (aliases: excel) — SpreadsheetML workbooks @@ -40,10 +40,10 @@ and JSONL inventories without deferred PowerPoint or MCP entries. {"formats":["xlsx"],"variants":[{"name":"xlsx","result_schema":"office.xlsx.query/1","constraints":["format=xlsx"]}]} $ office.exe help all --json | jq -c '{schema,success,capability_schema:.data.schema,fingerprint:.data.fingerprint,names:[.data.records[].name]}' - {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:3f8b2b34","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} + {"schema":"office.output/1","success":true,"capability_schema":"office.capabilities/2","fingerprint":"crc32:9e349fa4","names":["docx","xlsx","help","identify","outline","get","text","query","raw"]} $ office.exe help all --jsonl | jq -s -c 'map({schema,fingerprint,kind,name})' - [{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:3f8b2b34","kind":"command","name":"raw"}] + [{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"format","name":"docx"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"format","name":"xlsx"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"help"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"identify"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"outline"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"get"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"text"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"query"},{"schema":"office.capability/2","fingerprint":"crc32:9e349fa4","kind":"command","name":"raw"}] The raw command publishes explicit subcommand schemas, including every edit input and its conditional constraints. @@ -152,6 +152,16 @@ round-trips through the public selector grammar. $ office.exe query Book1-valid.xlsx 'cell[type=formula][formula~=IF]' --under '/xlsx/sheet[name="Sheet2"]' --json | jq -c '{schema:.data.schema,selector:.data.selector,under:.data.under,paths:[.data.matches[].path],matched_total:.data.matched_total,returned:.data.returned,truncated:.data.truncated,scanned:.data.scanned_cells}' {"schema":"office.xlsx.query/1","selector":"cell[type=formula][formula~=IF]","under":"/xlsx/sheet[name=\"Sheet2\"]","paths":["/xlsx/sheet[name=\"Sheet2\"]/cell[F11]","/xlsx/sheet[name=\"Sheet2\"]/cell[G11]","/xlsx/sheet[name=\"Sheet2\"]/cell[H11]","/xlsx/sheet[name=\"Sheet2\"]/cell[I11]"],"matched_total":4,"returned":4,"truncated":false,"scanned":99} +Exact text predicates preserve whitespace through the real command-line path; +JSON quoting also keeps selector delimiters unambiguous. + + $ xlsx.exe create whitespace.xlsx --sheet Data >/dev/null + $ xlsx.exe set whitespace.xlsx Data A1 ' leading and trailing ' >/dev/null + $ office.exe query whitespace.xlsx 'cell[text= leading and trailing ]' --json | jq -c '{selector:.data.selector,values:[.data.matches[].raw.value],matched_total:.data.matched_total}' + {"selector":"cell[text= leading and trailing ]","values":[" leading and trailing "],"matched_total":1} + $ office.exe query whitespace.xlsx 'cell[text=" leading and trailing "]' --json | jq -c '{selector:.data.selector,values:[.data.matches[].raw.value],matched_total:.data.matched_total}' + {"selector":"cell[text=\" leading and trailing \"]","values":[" leading and trailing "],"matched_total":1} + Cross-format selectors and DOCX-only XLSX query flags fail with XLSX-specific, machine-correctable codes. diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt index a4515d25..e11e2c74 100644 --- a/office/cmd/office/xlsx_query.mbt +++ b/office/cmd/office/xlsx_query.mbt @@ -66,7 +66,7 @@ fn xlsx_query_failure(message : String, selector : String) -> CliFailure { } ///| -fn require_xlsx_query_value( +fn require_xlsx_query_number_value( label : String, value : String, selector : String, @@ -84,6 +84,93 @@ fn require_xlsx_query_value( trimmed } +///| +fn xlsx_query_quoted_literal_end(value : String) -> Int? { + let mut escaped = false + let mut index = 1 + while index < value.length() { + let character = value[index] + if escaped { + escaped = false + } else if character == ('\\' : UInt16) { + escaped = true + } else if character == ('"' : UInt16) { + return Some(index) + } + index += 1 + } + None +} + +///| +fn xlsx_query_literal_has_valid_unicode(value : String) -> Bool { + let mut offset = 0 + while offset < value.length() { + let unit = value[offset] + if unit.is_leading_surrogate() { + if offset + 1 >= value.length() || + !value[offset + 1].is_trailing_surrogate() { + return false + } + offset += 2 + } else if unit.is_trailing_surrogate() { + return false + } else { + offset += 1 + } + } + true +} + +///| +fn require_xlsx_query_literal( + label : String, + value : String, + selector : String, +) -> String raise CliFailure { + let decoded = if value.has_prefix("\"") { + guard xlsx_query_quoted_literal_end(value) is Some(close) && + close == value.length() - 1 else { + raise xlsx_query_failure( + "\{label} quoted value must be one complete JSON string", + selector, + ) + } + let parsed = @json.parse(value) catch { + _ => + raise xlsx_query_failure( + "\{label} quoted value contains an invalid JSON string escape", + selector, + ) + } + guard parsed is String(text) else { + raise xlsx_query_failure( + "\{label} quoted value must be a JSON string", + selector, + ) + } + text + } else { + value + } + if decoded == "" { + raise xlsx_query_failure("\{label} needs a non-empty value", selector) + } + if !xlsx_query_literal_has_valid_unicode(decoded) { + raise xlsx_query_failure( + "\{label} value contains an isolated UTF-16 surrogate", + selector, + ) + } + if !has_at_most_chars(decoded, xlsx_cli_max_query_value_chars) { + raise xlsx_query_failure( + "\{label} value exceeds \{xlsx_cli_max_query_value_chars} characters", + selector, + ) + } + decoded +} + ///| fn parse_xlsx_number_predicate( rest : String, @@ -106,7 +193,7 @@ fn parse_xlsx_number_predicate( "a value predicate requires one of >, >=, <, <=, =, or !=", selector, ) } - let bounded = require_xlsx_query_value( + let bounded = require_xlsx_query_number_value( "value comparison", number_text, selector, ) let number = @string.parse_double(bounded) catch { @@ -128,7 +215,7 @@ fn parse_xlsx_predicate( body : String, selector : String, ) -> XlsxCellPredicate raise CliFailure { - let predicate = body.trim().to_owned() + let predicate = body if predicate.has_prefix("type=") { match predicate[5:].to_owned() { "formula" => HasType(Formula) @@ -145,17 +232,21 @@ fn parse_xlsx_predicate( parse_xlsx_number_predicate(predicate[5:].to_owned(), selector) } else if predicate.has_prefix("formula~=") { FormulaContains( - require_xlsx_query_value("formula~=", predicate[9:].to_owned(), selector), + require_xlsx_query_literal( + "formula~=", + predicate[9:].to_owned(), + selector, + ), ) } else if predicate == "formula" { HasFormula } else if predicate.has_prefix("text~=") { TextContains( - require_xlsx_query_value("text~=", predicate[6:].to_owned(), selector), + require_xlsx_query_literal("text~=", predicate[6:].to_owned(), selector), ) } else if predicate.has_prefix("text=") { TextEquals( - require_xlsx_query_value("text=", predicate[5:].to_owned(), selector), + require_xlsx_query_literal("text=", predicate[5:].to_owned(), selector), ) } else { raise xlsx_query_failure("unknown XLSX cell predicate", selector) @@ -164,18 +255,31 @@ fn parse_xlsx_predicate( ///| /// Finds the terminator for one outer predicate group while permitting -/// balanced brackets inside its literal value. This is required for normal -/// Excel structured references such as `Table1[Amount]` and -/// `Table1[[#Headers],[Amount]]`. +/// balanced brackets in unquoted values and arbitrary brackets in JSON-string +/// values. The former supports normal Excel structured references such as +/// `Table1[[#Headers],[Amount]]`; the latter makes every exact string +/// representable without confusing it with selector structure. fn xlsx_query_predicate_close( rest : String, selector : String, ) -> Int raise CliFailure { let mut depth = 0 let mut index = 0 + let mut in_string = false + let mut escaped = false while index < rest.length() { let character = rest[index] - if character == ('[' : UInt16) { + if in_string { + if escaped { + escaped = false + } else if character == ('\\' : UInt16) { + escaped = true + } else if character == ('"' : UInt16) { + in_string = false + } + } else if character == ('"' : UInt16) { + in_string = true + } else if character == ('[' : UInt16) { depth += 1 } else if character == (']' : UInt16) { depth -= 1 diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 5fa785bc..9d83c1f3 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -508,6 +508,69 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { ) } +///| +test "XLSX query literals preserve whitespace and support JSON quoting" { + let unquoted = parse_xlsx_query_selector("cell[text= leading and trailing ]") + guard unquoted[0] is TextEquals(unquoted_text) else { + fail("expected exact-text predicate") + } + assert_eq(unquoted_text, " leading and trailing ") + + let whitespace = parse_xlsx_query_selector("cell[text=\" \"]") + guard whitespace[0] is TextEquals(whitespace_text) else { + fail("expected quoted exact-text predicate") + } + assert_eq(whitespace_text, " ") + + let escaped = parse_xlsx_query_selector( + "cell[text=\"bracket]quote\\\"slash\\\\\"]", + ) + guard escaped[0] is TextEquals(escaped_text) else { + fail("expected escaped exact-text predicate") + } + assert_eq(escaped_text, "bracket]quote\"slash\\") + + for + malformed in [ + "cell[text=\"unterminated]", "cell[text=\"value\"suffix]", "cell[text=\"bad\\q\"]", + "cell[text=\"\"]", "cell[text=\"\\ud800\"]", + ] { + let error = xlsx_cli_error(() => { + ignore(parse_xlsx_query_selector(malformed)) + }) + assert_eq(error.code, "office.xlsx.invalid_query") + } + + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_str("Data", "A1", " leading and trailing ") + let projection = make_xlsx_projection("whitespace.xlsx", workbook, 10) + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]") + is Cell(sheet, rect, ..) else { + fail("expected whitespace fixture cell") + } + let snapshot = xlsx_cell_snapshot( + projection, + sheet, + rect.row_lo, + rect.col_lo, + XlsxScanBudget::new(1), + ) + for + selector in [ + "cell[text= leading and trailing ]", "cell[text=\" leading and trailing \"]", + ] { + let work = OfficeQueryWorkBudget::new(1024, format="xlsx") + assert_true( + xlsx_cell_matches( + snapshot, + prepare_xlsx_query_predicates(parse_xlsx_query_selector(selector), work), + work, + ), + ) + } +} + ///| test "XLSX snapshots preserve and resolve shared-formula followers" { let workbook = @xlsx.Workbook::new() From 284bf7f6e5754ab5351723357f75b8d48338063f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 08:25:25 +0800 Subject: [PATCH 037/150] fix: bind XLSX payloads to their cells --- xlsx/read_namespace_test.mbt | 69 +++++++++++++++++++++++++++++++++--- xlsx/read_worksheet_xml.mbt | 29 ++++++++++----- xlsx/read_xml_namespaces.mbt | 48 +++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 13 deletions(-) diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 4e71e048..ede8f93f 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -142,7 +142,9 @@ test "mixed-case workbook media type reads end to end" { if updated == content_types { fail("workbook media type marker was not found") } - archive.add("[Content_Types].xml", @encoding/utf8.encode(updated)) + assert_true( + archive.replace("[Content_Types].xml", @encoding/utf8.encode(updated)), + ) let parsed = @xlsx.read(@zip.write(archive)) debug_inspect(parsed.get_sheet_list(), content="[\"Qualified\"]") debug_inspect( @@ -169,7 +171,12 @@ test "SpreadsheetML-looking extension payload cannot create sheets or cells" { if workbook_with_decoy == workbook_xml { fail("workbook injection marker was not found") } - archive.add("xl/workbook.xml", @encoding/utf8.encode(workbook_with_decoy)) + assert_true( + archive.replace( + "xl/workbook.xml", + @encoding/utf8.encode(workbook_with_decoy), + ), + ) guard archive.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { fail("missing worksheet fixture part") } @@ -183,9 +190,11 @@ test "SpreadsheetML-looking extension payload cannot create sheets or cells" { if worksheet_with_decoy == worksheet_xml { fail("worksheet injection marker was not found") } - archive.add( - "xl/worksheets/sheet.xml", - @encoding/utf8.encode(worksheet_with_decoy), + assert_true( + archive.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode(worksheet_with_decoy), + ), ) let parsed = @xlsx.read(@zip.write(archive)) debug_inspect(parsed.get_sheet_list(), content="[\"Qualified\"]") @@ -196,3 +205,53 @@ test "SpreadsheetML-looking extension payload cannot create sheets or cells" { ) } } + +///| +test "cell payload cannot escape its element or cross a namespace boundary" { + let archive = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard archive.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { + fail("missing worksheet fixture part") + } + let worksheet_xml = @encoding/utf8.decode(worksheet_bytes) catch { + _ => fail("worksheet fixture is not UTF-8") + } + let two_cells = worksheet_xml.replace( + old="0", + new="1+1second", + ) + if two_cells == worksheet_xml { + fail("worksheet cell marker was not found") + } + assert_true( + archive.replace("xl/worksheets/sheet.xml", @encoding/utf8.encode(two_cells)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + debug_inspect(parsed.get_cell_value("Qualified", "A1"), content="Some(\"\")") + debug_inspect(parsed.get_cell_formula("Qualified", "A1"), content="None") + debug_inspect( + parsed.get_cell_value("Qualified", "B1"), + content="Some(\"second\")", + ) + debug_inspect( + parsed.get_cell_formula("Qualified", "B1"), + content="Some(\"1+1\")", + ) + + let foreign_value = two_cells.replace( + old="", + new="injected", + ) + assert_true( + archive.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode(foreign_value), + ), + ) + try @xlsx.read(@zip.write(archive)) catch { + @xlsx.InvalidXml(msg~) => + inspect(msg, content="worksheet cell payload element is misplaced") + _ => fail("unexpected foreign cell payload error") + } noraise { + _ => fail("expected foreign cell payload rejection") + } +} diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index bc6b9ca7..b4cace73 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -58,14 +58,27 @@ fn parse_worksheet( raise InvalidStyleId(index=style_id) } let cell_type = attr_value(tag, "t") - let rest = text[end + 1:] - let formula_start = match rest.find("") { + let mut tag_last = tag.length() - 1 + while tag_last >= 0 && is_xml_attr_space_unit(tag[tag_last]) { + tag_last = tag_last - 1 + } + let self_closing = tag_last >= 0 && tag[tag_last] == '/' + let after_open = text[end + 1:] + let cell_body = if self_closing { + after_open[:0] + } else { + match after_open.find("") { + Some(pos) => after_open[:pos] + None => raise InvalidXml(msg="cell not closed") + } + } + let formula_start = match cell_body.find("") { Some(pos) => Some(pos) None => - match rest.find(" Some(pos) None => - match rest.find(" Some(pos) None => None } @@ -74,7 +87,7 @@ fn parse_worksheet( let (formula, formula_type, formula_ref, formula_shared_index) = match formula_start { Some(pos) => { - let f_rest = rest[pos:] + let f_rest = cell_body[pos:] let open_end = match xml_open_tag_end_from(f_rest, 0) { Some(value) => value None => raise InvalidXml(msg="formula tag not closed") @@ -134,9 +147,9 @@ fn parse_worksheet( (Some(Shared), None, Some(_)) => Some("") _ => formula } - let raw_value_opt = match find_xml_open_tag_start(rest, "v") { + let raw_value_opt = match find_xml_open_tag_start(cell_body, "v") { Some(pos) => { - let v_rest = rest[pos:] + let v_rest = cell_body[pos:] let open_end = match xml_open_tag_end_from(v_rest, 0) { Some(end) => end None => raise InvalidXml(msg="cell value tag not closed") @@ -157,7 +170,7 @@ fn parse_worksheet( } let raw_value = raw_value_opt.unwrap_or("") let inline_entry = match cell_type { - Some("inlineStr") => Some(parse_inline_string(rest)) + Some("inlineStr") => Some(parse_inline_string(cell_body)) _ => None } let force_string = raw_value_opt is None && diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 2fe19619..e936a3fb 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -24,6 +24,9 @@ fn validate_xlsx_worksheet_core(xml : StringView) -> Unit raise XlsxError { let scanner = @ooxml.XmlStartTagScanner::new(xml) let mut worksheet_namespace : String? = None let mut sheet_data_seen = false + let mut cell_formula_seen = false + let mut cell_value_seen = false + let mut cell_inline_string_seen = false while workbook_scanner_next(scanner) { let local_name = scanner.local_name() let namespace_uri = scanner.namespace_uri() @@ -64,6 +67,51 @@ fn validate_xlsx_worksheet_core(xml : StringView) -> Unit raise XlsxError { scanner.parent_namespace_uri() != worksheet_namespace { raise InvalidXml(msg="worksheet cell element is misplaced") } + cell_formula_seen = false + cell_value_seen = false + cell_inline_string_seen = false + } else if local_name == "f" || local_name == "v" || local_name == "is" { + if namespace_uri != dialect || + scanner.depth() != 5 || + scanner.parent_local_name() != Some("c") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet cell payload element is misplaced") + } + if local_name == "f" { + if cell_formula_seen { + raise InvalidXml(msg="worksheet cell has multiple formula elements") + } + cell_formula_seen = true + } else if local_name == "v" { + if cell_value_seen { + raise InvalidXml(msg="worksheet cell has multiple value elements") + } + cell_value_seen = true + } else { + if cell_inline_string_seen { + raise InvalidXml( + msg="worksheet cell has multiple inline string elements", + ) + } + cell_inline_string_seen = true + } + } else if local_name == "r" { + if namespace_uri != dialect || + scanner.depth() != 6 || + scanner.parent_local_name() != Some("is") || + scanner.parent_namespace_uri() != worksheet_namespace { + raise InvalidXml(msg="worksheet inline string run is misplaced") + } + } else if local_name == "t" { + let valid_parent = scanner.parent_local_name() == Some("is") || + scanner.parent_local_name() == Some("r") || + scanner.parent_local_name() == Some("rPh") + if namespace_uri != dialect || + !valid_parent || + scanner.parent_namespace_uri() != worksheet_namespace || + (scanner.depth() != 6 && scanner.depth() != 7) { + raise InvalidXml(msg="worksheet inline string text is misplaced") + } } else if local_name == "cols" { if namespace_uri != dialect || scanner.depth() != 2 || From e229e5e72ffe604cf3a0e34d2615cc5d07653c70 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 08:28:29 +0800 Subject: [PATCH 038/150] fix: cancel XLSX CRC verification promptly --- xlsx/read.mbt | 6 +++++- xlsx/read_limits_test.mbt | 24 ++++++++++++++++++++++ zip/crc32.mbt | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 5466384c..666a5418 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2992,7 +2992,11 @@ fn enforce_archive_limits( actual=size, ) } - if @zip.crc32(entry.data()) != entry.crc32() { + let actual_crc = @zip.crc32_cancellable(entry.data(), cancelled~) catch { + ReadCancelled => raise ReadCancelled + _ => raise InvalidPackage(msg="ZIP entry CRC verification failed") + } + if actual_crc != entry.crc32() { raise InvalidPackage(msg="ZIP entry CRC mismatch") } check_read_cancelled(cancelled) diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index b6675ec4..93ec8774 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -465,6 +465,30 @@ test "read cancellation callback stops semantic preflight" { inspect(result is Err(@xlsx.ReadCancelled), content="true") } +///| +test "bounded archive CRC verification polls cancellation inside an entry" { + let source = @zip.read(bounded_workbook_fixture()) + let padded = @zip.Archive::new() + // Keep this first so the sixth callback is the 4 KiB checkpoint inside its + // CRC loop, after the archive and entry boundary checks. + padded.add("padding.bin", Bytes::make(16 * 1024, b'x')) + for entry in source.entries() { + padded.add(entry.name(), entry.data()) + } + let bounded = bounded_fixture_archive(@zip.write(padded)) + let checks = [0] + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read_bounded_archive(bounded, cancelled=() => { + checks[0] += 1 + checks[0] >= 6 + }), + ) catch { + error => Err(error) + } + assert_eq(checks[0], 6) + assert_true(result is Err(@xlsx.ReadCancelled)) +} + ///| test "read completion observes cancellation at its final checkpoint" { let bytes = bounded_workbook_fixture() diff --git a/zip/crc32.mbt b/zip/crc32.mbt index 864383a6..cde1ae90 100644 --- a/zip/crc32.mbt +++ b/zip/crc32.mbt @@ -30,3 +30,45 @@ pub fn crc32(bytes : BytesView) -> UInt { } crc ^ 0xFFFFFFFF } + +///| +/// Computes ZIP CRC-32 while polling `cancelled` at bounded byte intervals. +/// This is intended for archive verification on untrusted, size-bounded input; +/// callers that do not need cooperative cancellation can use `crc32`. +pub fn crc32_cancellable( + bytes : BytesView, + cancelled? : () -> Bool = () => false, +) -> UInt raise ZipError { + check_zip_cancelled(cancelled) + let mut crc : UInt = 0xFFFFFFFF + let mut index = 0 + while index < bytes.length() { + if (index & 4095) == 0 { + check_zip_cancelled(cancelled) + } + let byte = bytes[index] + crc = crc32_table[((crc ^ byte.to_uint()) & 0xFF).reinterpret_as_int()] ^ + (crc >> 8) + index += 1 + } + check_zip_cancelled(cancelled) + crc ^ 0xFFFFFFFF +} + +///| +test "cancellable CRC matches CRC-32 and polls inside long input" { + let bytes = Bytes::make(16 * 1024, b'x') + assert_eq(crc32_cancellable(bytes), crc32(bytes)) + let checks = [0] + try + crc32_cancellable(bytes, cancelled=() => { + checks[0] += 1 + checks[0] >= 4 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("expected CRC cancellation") + } noraise { + _ => fail("expected CRC cancellation") + } +} From 0c16c5cd9892a758cc550608fc9ca796c66cf0b0 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 08:34:33 +0800 Subject: [PATCH 039/150] fix: cancel XLSX XML passes promptly --- ooxml/read_parse.mbt | 4 + ooxml/read_parse_test.mbt | 6 + ooxml/start_tag_scanner.mbt | 220 +++++++++++++++++++++++++++++++---- xlsx/ooxml_rels.mbt | 4 + xlsx/read.mbt | 5 + xlsx/read_package_parts.mbt | 1 + xlsx/read_workbook_xml.mbt | 5 +- xlsx/read_xml_namespaces.mbt | 40 ++++++- xlsx/xml.mbt | 1 + 9 files changed, 257 insertions(+), 29 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 59fbd8a7..ee1988e5 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -1,6 +1,7 @@ ///| pub suberror ParseXmlError { InvalidXml(msg~ : String) + ReadCancelled } derive(Debug) ///| @@ -427,6 +428,7 @@ test "ooxml read_parse wb: every relationship record is validated" { #| try parse_internal_relationship_targets(xml, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship type missing") + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected relationship type missing") } @@ -443,6 +445,7 @@ test "ooxml read_parse wb: content-type override required attrs" { try parse_content_types(missing_part) catch { InvalidXml(msg~) => inspect(msg, content="content type override part name missing") + _ => fail("unexpected content type parse cancellation") } noraise { _ => fail("expected content type override part name missing") } @@ -456,6 +459,7 @@ test "ooxml read_parse wb: content-type override required attrs" { try parse_content_types(missing_content_type) catch { InvalidXml(msg~) => inspect(msg, content="content type override content type missing") + _ => fail("unexpected content type parse cancellation") } noraise { _ => fail("expected content type override content type missing") } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 11714ff7..e606cf89 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -27,6 +27,7 @@ test "ooxml read_parse: internal relationships report missing id/target and malf #| try @ooxml.parse_internal_relationship_targets(missing_id, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship id missing") + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected relationship id missing") } @@ -39,6 +40,7 @@ test "ooxml read_parse: internal relationships report missing id/target and malf #| try @ooxml.parse_internal_relationship_targets(missing_target, "t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship target missing") + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected relationship target missing") } @@ -51,6 +53,7 @@ test "ooxml read_parse: internal relationships report missing id/target and malf #| try @ooxml.parse_internal_relationship_targets(malformed, "t1") catch { InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected relationship tag not closed") } @@ -117,6 +120,7 @@ test "ooxml read_parse: invalid target modes fail closed" { } catch { InvalidXml(msg~) => inspect(msg, content="relationship target mode invalid") + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected invalid relationship target mode") } @@ -136,6 +140,7 @@ test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay ] { try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { InvalidXml(_) => () + _ => fail("unexpected relationship parse cancellation") } noraise { _ => fail("expected invalid relationship document") } @@ -199,6 +204,7 @@ test "ooxml read_parse: parse_content_type_overrides reports malformed override ]) catch { InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") + _ => fail("unexpected content type parse cancellation") } noraise { _ => fail("expected content type override tag not closed") } diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 2310ce9f..3108c372 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -43,6 +43,7 @@ priv struct XmlAttributeSpan { /// remain borrowed from the caller's XML string. pub struct XmlStartTagScanner { priv xml : StringView + priv cancelled : () -> Bool priv mut index : Int priv mut depth : Int priv namespace_bindings : Array[XmlNamespaceBinding] @@ -78,6 +79,13 @@ pub struct XmlStartTagScanner { priv mut capture_depth : Int } +///| +fn check_xml_cancelled(cancelled : () -> Bool) -> Unit raise ParseXmlError { + if cancelled() { + raise ReadCancelled + } +} + ///| fn is_xml_name_start(unit : UInt16) -> Bool { (unit >= ('A' : UInt16) && unit <= ('Z' : UInt16)) || @@ -190,13 +198,20 @@ fn find_xml_sequence( xml : StringView, start : Int, sequence : StringView, -) -> Int? { + cancelled? : () -> Bool = () => false, +) -> Int? raise ParseXmlError { + check_xml_cancelled(cancelled) if sequence.length() == 0 { return Some(start) } let last = xml.length() - sequence.length() let mut index = start + let mut next_checkpoint = start while index <= last { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } if xml_span_equals(xml, index, index + sequence.length(), sequence) { return Some(index) } @@ -210,10 +225,17 @@ fn find_xml_tag_end( xml : StringView, search_start : Int, markup_start : Int, + cancelled? : () -> Bool = () => false, ) -> Int raise ParseXmlError { + check_xml_cancelled(cancelled) let mut index = search_start + let mut next_checkpoint = search_start let mut quote : UInt16? = None while index < xml.length() { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } if index - markup_start > max_xml_start_tag_chars { raise InvalidXml(msg="XML start tag is too long") } @@ -246,9 +268,16 @@ fn next_xml_attribute( xml : StringView, start : Int, tag_end : Int, + cancelled? : () -> Bool = () => false, ) -> XmlAttributeSpan? raise ParseXmlError { + check_xml_cancelled(cancelled) let mut index = start + let mut next_checkpoint = start + 4096 while index < tag_end && is_attr_space(xml[index]) { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } index = index + 1 } if index >= tag_end { @@ -265,11 +294,19 @@ fn next_xml_attribute( !is_attr_space(xml[index]) && xml[index] != ('=' : UInt16) && xml[index] != ('/' : UInt16) { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } index = index + 1 } let name_end = index ignore(validate_xml_qname(xml, name_start, name_end)) while index < tag_end && is_attr_space(xml[index]) { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } index = index + 1 } if index >= tag_end || xml[index] != ('=' : UInt16) { @@ -277,6 +314,10 @@ fn next_xml_attribute( } index = index + 1 while index < tag_end && is_attr_space(xml[index]) { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } index = index + 1 } if index >= tag_end || @@ -287,6 +328,10 @@ fn next_xml_attribute( index = index + 1 let value_start = index while index < tag_end && xml[index] != quote { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } if xml[index] == ('<' : UInt16) { raise InvalidXml(msg="XML attribute contains '<'") } @@ -356,9 +401,16 @@ fn xml_entity_value(entity : StringView) -> Char? { /// Decodes one XML attribute value in a single pass. Unknown, malformed, or /// XML-forbidden entity references are rejected instead of being decoded a /// second time by a downstream path resolver. -pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { +pub fn decode_xml_attribute( + value : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise ParseXmlError { + check_xml_cancelled(cancelled) let mut needs_decode = false - for unit in value { + for index, unit in value { + if (index & 4095) == 0 { + check_xml_cancelled(cancelled) + } if unit == '&' || unit == '\t' || unit == '\n' || unit == '\r' { needs_decode = true break @@ -373,11 +425,17 @@ pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { let output = StringBuilder::new() let mut index = 0 let mut literal_start = 0 + let mut next_checkpoint = 0 while index < value.length() { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } let unit = value[index] if unit == ('&' : UInt16) { output.write_view(value[literal_start:index]) - let entity_end = match find_xml_sequence(value, index + 1, ";") { + let entity_end = match + find_xml_sequence(value, index + 1, ";", cancelled~) { Some(found) => found None => raise InvalidXml(msg="XML entity is not closed") } @@ -419,12 +477,22 @@ pub fn decode_xml_attribute(value : StringView) -> String raise ParseXmlError { /// Checks an attribute value without retaining a decoded copy. This is used /// for attributes a caller does not select: malformed entities must never be /// hidden merely because an attribute is semantically irrelevant. -fn validate_xml_attribute_value(value : StringView) -> Unit raise ParseXmlError { +fn validate_xml_attribute_value( + value : StringView, + cancelled : () -> Bool, +) -> Unit raise ParseXmlError { + check_xml_cancelled(cancelled) let mut index = 0 + let mut next_checkpoint = 0 while index < value.length() { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } let unit = value[index] if unit == ('&' : UInt16) { - let entity_end = match find_xml_sequence(value, index + 1, ";") { + let entity_end = match + find_xml_sequence(value, index + 1, ";", cancelled~) { Some(found) => found None => raise InvalidXml(msg="XML entity is not closed") } @@ -582,7 +650,8 @@ fn XmlStartTagScanner::add_namespace_declarations( let declared : Set[String] = Set([]) let mut next = attributes_start let mut count = 0 - while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { + while next_xml_attribute(self.xml, next, tag_end, cancelled=self.cancelled) + is Some(attribute) { count = count + 1 if count > max_xml_start_tag_attributes { raise InvalidXml(msg="XML start tag has too many attributes") @@ -597,6 +666,7 @@ fn XmlStartTagScanner::add_namespace_declarations( declared.add(prefix) let uri = decode_xml_attribute( self.xml[attribute.value_start:attribute.value_end], + cancelled=self.cancelled, ) if prefix == "xmlns" || uri == xmlns_namespace_uri { raise InvalidXml(msg="XML namespace declaration is reserved") @@ -637,7 +707,8 @@ fn XmlStartTagScanner::validate_attributes( let canonical_names : Set[String] = Set([]) let mut next = attributes_start let mut count = 0 - while next_xml_attribute(self.xml, next, tag_end) is Some(attribute) { + while next_xml_attribute(self.xml, next, tag_end, cancelled=self.cancelled) + is Some(attribute) { count = count + 1 if count > max_xml_start_tag_attributes { raise InvalidXml(msg="XML start tag has too many attributes") @@ -645,6 +716,7 @@ fn XmlStartTagScanner::validate_attributes( next = attribute.next validate_xml_attribute_value( self.xml[attribute.value_start:attribute.value_end], + self.cancelled, ) if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { continue @@ -724,12 +796,29 @@ fn XmlStartTagScanner::record_rewrite( } self.charge_rewrite_output(start - self.rewrite_cursor) self.charge_rewrite_output(replacement.length()) - self.rewrite_output.write_view(self.xml[self.rewrite_cursor:start]) + self.write_rewrite_source(self.rewrite_cursor, start) self.rewrite_output.write_string(replacement) + check_xml_cancelled(self.cancelled) self.rewrite_cursor = end self.rewrite_count = self.rewrite_count + 1 } +///| +fn XmlStartTagScanner::write_rewrite_source( + self : XmlStartTagScanner, + start : Int, + end : Int, +) -> Unit raise ParseXmlError { + let mut index = start + while index < end { + check_xml_cancelled(self.cancelled) + let next = (index + 4096).min(end) + self.rewrite_output.write_view(self.xml[index:next]) + index = next + } + check_xml_cancelled(self.cancelled) +} + ///| fn XmlStartTagScanner::charge_rewrite_output( self : XmlStartTagScanner, @@ -759,7 +848,10 @@ fn XmlStartTagScanner::escaped_character_data( let value = self.xml[value_start:end] let output = StringBuilder::new(size_hint=value.length().min(available)) let mut used = 0 - for unit in value { + for index, unit in value { + if (index & 4095) == 0 { + check_xml_cancelled(self.cancelled) + } let needed = if unit == '&' { 5 } else if unit == '<' { 4 } else { 1 } if needed > available - used { raise InvalidXml(msg="XML canonical output limit exceeded") @@ -773,6 +865,7 @@ fn XmlStartTagScanner::escaped_character_data( output.write_char(unit) } } + check_xml_cancelled(self.cancelled) output.to_string() } @@ -798,9 +891,11 @@ fn xml_start_tag_scanner_with_rewrites( rewrite_output_limit : Int, capture_element_namespaces? : Array[String] = [], capture_element_local_name? : String = "", + cancelled? : () -> Bool = () => false, ) -> XmlStartTagScanner { { xml, + cancelled, index: 0, depth: 0, namespace_bindings: [], @@ -840,8 +935,11 @@ fn xml_start_tag_scanner_with_rewrites( ///| /// Creates a namespace-aware cursor over `xml`. The source is borrowed; no /// copy of the full XML part is made. -pub fn XmlStartTagScanner::new(xml : StringView) -> XmlStartTagScanner { - xml_start_tag_scanner_with_rewrites(xml, [], Map([]), false, 0) +pub fn XmlStartTagScanner::new( + xml : StringView, + cancelled? : () -> Bool = () => false, +) -> XmlStartTagScanner { + xml_start_tag_scanner_with_rewrites(xml, [], Map([]), false, 0, cancelled~) } ///| @@ -849,6 +947,7 @@ fn XmlStartTagScanner::consume_end_tag( self : XmlStartTagScanner, start : Int, ) -> Unit raise ParseXmlError { + check_xml_cancelled(self.cancelled) if self.depth <= 0 || self.open_name_starts.length() == 0 { raise InvalidXml(msg="XML end tag has no matching start tag") } @@ -860,8 +959,18 @@ fn XmlStartTagScanner::consume_end_tag( name_end = name_end + 1 } let colon = validate_xml_qname(self.xml, name_start, name_end) - let tag_end = find_xml_tag_end(self.xml, name_end, start) + let tag_end = find_xml_tag_end( + self.xml, + name_end, + start, + cancelled=self.cancelled, + ) + let mut next_checkpoint = name_end for index in name_end..= next_checkpoint { + check_xml_cancelled(self.cancelled) + next_checkpoint = index + 4096 + } if !is_attr_space(self.xml[index]) { raise InvalidXml(msg="XML end tag is invalid") } @@ -913,6 +1022,7 @@ fn XmlStartTagScanner::consume_end_tag( pub fn XmlStartTagScanner::next( self : XmlStartTagScanner, ) -> Bool raise ParseXmlError { + check_xml_cancelled(self.cancelled) match self.pending_namespace_pop { Some(depth) => { self.pop_namespace_depth(depth) @@ -920,9 +1030,14 @@ pub fn XmlStartTagScanner::next( } None => () } + let mut next_checkpoint = self.index while self.index < self.xml.length() { while self.index < self.xml.length() && self.xml[self.index] != ('<' : UInt16) { + if self.index >= next_checkpoint { + check_xml_cancelled(self.cancelled) + next_checkpoint = self.index + 4096 + } if self.depth == 0 && !is_attr_space(self.xml[self.index]) { raise InvalidXml(msg="XML text is outside the document element") } @@ -936,7 +1051,8 @@ pub fn XmlStartTagScanner::next( raise InvalidXml(msg="XML markup is not closed") } if self.xml[start + 1] == ('?' : UInt16) { - let end = match find_xml_sequence(self.xml, start + 2, "?>") { + let end = match + find_xml_sequence(self.xml, start + 2, "?>", cancelled=self.cancelled) { Some(value) => value None => raise InvalidXml(msg="XML processing instruction is not closed") } @@ -949,7 +1065,13 @@ pub fn XmlStartTagScanner::next( if self.xml[start + 1] == ('!' : UInt16) { if start + 4 <= self.xml.length() && xml_span_equals(self.xml, start, start + 4, "") { + let end = match + find_xml_sequence( + self.xml, + start + 4, + "-->", + cancelled=self.cancelled, + ) { Some(value) => value None => raise InvalidXml(msg="XML comment is not closed") } @@ -964,7 +1086,13 @@ pub fn XmlStartTagScanner::next( if self.depth == 0 { raise InvalidXml(msg="XML CDATA is outside the document element") } - let end = match find_xml_sequence(self.xml, start + 9, "]]>") { + let end = match + find_xml_sequence( + self.xml, + start + 9, + "]]>", + cancelled=self.cancelled, + ) { Some(value) => value None => raise InvalidXml(msg="XML CDATA section is not closed") } @@ -999,7 +1127,12 @@ pub fn XmlStartTagScanner::next( name_end = name_end + 1 } let colon = validate_xml_qname(self.xml, name_start, name_end) - let tag_end = find_xml_tag_end(self.xml, name_end, start) + let tag_end = find_xml_tag_end( + self.xml, + name_end, + start, + cancelled=self.cancelled, + ) let mut before_end = tag_end while before_end > name_end && is_attr_space(self.xml[before_end - 1]) { before_end = before_end - 1 @@ -1142,7 +1275,9 @@ pub fn remove_xml_expanded_name_subtrees( xml : StringView, element_namespaces : ArrayView[String], local_name : String, + cancelled? : () -> Bool = () => false, ) -> String? raise ParseXmlError { + check_xml_cancelled(cancelled) if local_name == "" { raise InvalidXml(msg="XML removal local name is empty") } @@ -1156,6 +1291,7 @@ pub fn remove_xml_expanded_name_subtrees( xml.length(), capture_element_namespaces=namespaces, capture_element_local_name=local_name, + cancelled~, ) while scanner.next() { @@ -1164,7 +1300,7 @@ pub fn remove_xml_expanded_name_subtrees( return None } scanner.charge_rewrite_output(xml.length() - scanner.rewrite_cursor) - scanner.rewrite_output.write_view(xml[scanner.rewrite_cursor:]) + scanner.write_rewrite_source(scanner.rewrite_cursor, xml.length()) Some(scanner.rewrite_output.to_string()) } @@ -1181,7 +1317,9 @@ pub fn canonicalize_xml_expanded_names( element_namespaces : ArrayView[String], attribute_prefixes : ArrayView[(String, String)], max_output_chars? : Int = max_xml_start_tag_chars, + cancelled? : () -> Bool = () => false, ) -> String raise ParseXmlError { + check_xml_cancelled(cancelled) if max_output_chars < 0 { raise InvalidXml(msg="XML canonical output limit is invalid") } @@ -1193,7 +1331,12 @@ pub fn canonicalize_xml_expanded_names( attributes[namespace_uri] = prefix } let scanner = xml_start_tag_scanner_with_rewrites( - xml, elements, attributes, true, max_output_chars, + xml, + elements, + attributes, + true, + max_output_chars, + cancelled~, ) while scanner.next() { @@ -1202,10 +1345,12 @@ pub fn canonicalize_xml_expanded_names( if xml.length() > max_output_chars { raise InvalidXml(msg="XML canonical output limit exceeded") } - return xml.to_owned() + scanner.charge_rewrite_output(xml.length()) + scanner.write_rewrite_source(0, xml.length()) + return scanner.rewrite_output.to_string() } scanner.charge_rewrite_output(xml.length() - scanner.rewrite_cursor) - scanner.rewrite_output.write_view(xml[scanner.rewrite_cursor:]) + scanner.write_rewrite_source(scanner.rewrite_cursor, xml.length()) scanner.rewrite_output.to_string() } @@ -1218,7 +1363,12 @@ pub fn XmlStartTagScanner::attribute_view( local_name : StringView, ) -> StringView? raise ParseXmlError { let mut next = self.current_name_end - while next_xml_attribute(self.xml, next, self.current_tag_end) + while next_xml_attribute( + self.xml, + next, + self.current_tag_end, + cancelled=self.cancelled, + ) is Some(attribute) { next = attribute.next if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { @@ -1256,7 +1406,7 @@ pub fn XmlStartTagScanner::attribute( local_name : StringView, ) -> String? raise ParseXmlError { match self.attribute_view(namespace_uri, local_name) { - Some(value) => Some(decode_xml_attribute(value)) + Some(value) => Some(decode_xml_attribute(value, cancelled=self.cancelled)) None => None } } @@ -1328,6 +1478,7 @@ test "start tag scanner rejects undeclared and duplicate expanded attributes" { let scanner = XmlStartTagScanner::new(xml) try scanner.next() catch { InvalidXml(_) => () + _ => fail("unexpected scanner cancellation") } noraise { _ => fail("expected invalid namespace-qualified XML") } @@ -1355,6 +1506,7 @@ test "XML attributes decode once and reject invalid entities" { for value in ["&unknown;", "�", "�", "&"] { try decode_xml_attribute(value) catch { InvalidXml(_) => () + _ => fail("unexpected attribute cancellation") } noraise { _ => fail("expected invalid XML entity") } @@ -1375,6 +1527,7 @@ test "start tag scanner validates unused attributes and document structure" { } } catch { InvalidXml(_) => () + _ => fail("unexpected scanner cancellation") } noraise { _ => fail("expected malformed XML document") } @@ -1387,6 +1540,7 @@ test "start tag scanner rejects a misplaced self-closing slash" { try scanner.next() catch { InvalidXml(msg~) => inspect(msg, content="XML self-closing slash is misplaced") + _ => fail("unexpected scanner cancellation") } noraise { _ => fail("expected misplaced self-closing slash") } @@ -1454,6 +1608,7 @@ test "expanded-name canonicalization rejects lexical attribute collisions" { catch { InvalidXml(msg~) => inspect(msg, content="XML canonical attribute name collision") + _ => fail("unexpected canonicalization cancellation") } noraise { _ => fail("expected canonical attribute collision") } @@ -1471,6 +1626,7 @@ test "expanded-name canonicalization bounds escaped CDATA output" { catch { InvalidXml(msg~) => inspect(msg, content="XML canonical output limit exceeded") + _ => fail("unexpected canonicalization cancellation") } noraise { _ => fail("expected canonical output limit") } @@ -1481,7 +1637,25 @@ test "start tag scanner bounds qualified names" { let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") try scanner.next() catch { InvalidXml(msg~) => inspect(msg, content="XML qualified name is too long") + _ => fail("unexpected scanner cancellation") } noraise { _ => fail("expected qualified-name limit") } } + +///| +test "expanded-name canonicalization polls cancellation inside long XML" { + let checks = [0] + let xml = "" + "x".repeat(32 * 1024) + "" + try + canonicalize_xml_expanded_names(xml, [], [], cancelled=() => { + checks[0] += 1 + checks[0] >= 12 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 12) + _ => fail("expected canonicalization cancellation") + } noraise { + _ => fail("expected canonicalization cancellation") + } +} diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index a9a999c2..8c12b265 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -11,6 +11,7 @@ fn parse_internal_relationship_targets_exact( ) -> Map[String, String] raise XlsxError { @ooxml.parse_internal_relationship_targets(xml, rel_type) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } @@ -21,6 +22,7 @@ fn parse_internal_relationship_targets_exact_types( ) -> Map[String, String] raise XlsxError { @ooxml.parse_internal_relationship_targets_by_types(xml, rel_types) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } @@ -31,6 +33,7 @@ fn parse_external_relationship_targets_exact( ) -> Map[String, String] raise XlsxError { @ooxml.parse_external_relationship_targets(xml, rel_type) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } @@ -41,6 +44,7 @@ fn parse_external_relationship_targets_exact_types( ) -> Map[String, String] raise XlsxError { @ooxml.parse_external_relationship_targets_by_types(xml, rel_types) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 666a5418..a2794fdf 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3221,6 +3221,7 @@ fn read_zip_archive_core_unchecked( let canonical = canonicalize_xlsx_xml( decoded, max_output_chars=limits.max_xml_part_bytes, + cancelled~, ) read_budget.checkpoint() read_budget.charge_work(canonical.length()) @@ -3239,6 +3240,7 @@ fn read_zip_archive_core_unchecked( let workbook_xml = canonicalize_xlsx_xml( workbook_source_xml, max_output_chars=limits.max_xml_part_bytes, + cancelled~, ) read_budget.checkpoint() read_budget.charge_work(workbook_xml.length()) @@ -3253,6 +3255,7 @@ fn read_zip_archive_core_unchecked( let sheets = parse_workbook_sheets( workbook_source_xml, limits.max_workbook_sheets, + cancelled~, ) let sheet_names : Array[String] = [] for entry in sheets { @@ -3512,6 +3515,7 @@ fn read_zip_archive_core_unchecked( let sheet_xml = canonicalize_xlsx_xml( sheet_source_xml, max_output_chars=limits.max_xml_part_bytes, + cancelled~, ) read_budget.checkpoint() read_budget.charge_work(sheet_xml.length()) @@ -3519,6 +3523,7 @@ fn read_zip_archive_core_unchecked( sheet_source_xml, sheet_xml, limits.max_xml_part_bytes, + cancelled~, ) read_budget.checkpoint() if sheet_core_xml != sheet_xml { diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index ac7c412e..eb567b22 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -79,6 +79,7 @@ fn resolve_workbook_xml_part_path( content_types_xml, workbook_part, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } match content_type { Some(value) if workbook_content_type_is_supported(value) => workbook_part diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index a6ca3fb5..e4386e64 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -23,6 +23,7 @@ fn workbook_scanner_next( ) -> Bool raise XlsxError { scanner.next() catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } @@ -34,6 +35,7 @@ fn workbook_scanner_attribute( ) -> String? raise XlsxError { scanner.attribute(namespace_uri, local_name) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } @@ -41,9 +43,10 @@ fn workbook_scanner_attribute( fn parse_workbook_sheets( xml : StringView, max_sheets : Int, + cancelled? : () -> Bool = () => false, ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] - let scanner = @ooxml.XmlStartTagScanner::new(xml) + let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) let mut workbook_namespace : String? = None while workbook_scanner_next(scanner) { let namespace_uri = scanner.namespace_uri() diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index e936a3fb..8643e52b 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -5,6 +5,7 @@ fn canonicalize_xlsx_xml( xml : StringView, max_output_chars? : Int = default_max_xml_part_bytes, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { @ooxml.canonicalize_xml_expanded_names( xml, @@ -14,14 +15,19 @@ fn canonicalize_xlsx_xml( (strict_relationship_attribute_namespace, "r"), ], max_output_chars~, + cancelled~, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } ///| -fn validate_xlsx_worksheet_core(xml : StringView) -> Unit raise XlsxError { - let scanner = @ooxml.XmlStartTagScanner::new(xml) +fn validate_xlsx_worksheet_core( + xml : StringView, + cancelled? : () -> Bool = () => false, +) -> Unit raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) let mut worksheet_namespace : String? = None let mut sheet_data_seen = false let mut cell_formula_seen = false @@ -138,21 +144,24 @@ fn canonicalize_xlsx_worksheet_core( source : StringView, full_canonical : String, max_output_chars : Int, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let without_extensions = @ooxml.remove_xml_expanded_name_subtrees( source, [transitional_spreadsheet_namespace, strict_spreadsheet_namespace], "extLst", + cancelled~, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } match without_extensions { Some(core_source) => { - validate_xlsx_worksheet_core(core_source) - canonicalize_xlsx_xml(core_source, max_output_chars~) + validate_xlsx_worksheet_core(core_source, cancelled~) + canonicalize_xlsx_xml(core_source, max_output_chars~, cancelled~) } None => { - validate_xlsx_worksheet_core(source) + validate_xlsx_worksheet_core(source, cancelled~) full_canonical } } @@ -169,3 +178,24 @@ test "XLSX XML canonicalization handles Strict elements and relationship attribu ), ) } + +///| +test "XLSX XML canonicalization propagates cancellation from a long pass" { + let checks = [0] + let source = "" + + "x".repeat(32 * 1024) + + "" + try + canonicalize_xlsx_xml(source, cancelled=() => { + checks[0] += 1 + checks[0] >= 12 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 12) + _ => fail("expected XLSX canonicalization cancellation") + } noraise { + _ => fail("expected XLSX canonicalization cancellation") + } +} diff --git a/xlsx/xml.mbt b/xlsx/xml.mbt index 3efde5cc..5ba766b0 100644 --- a/xlsx/xml.mbt +++ b/xlsx/xml.mbt @@ -128,6 +128,7 @@ fn is_attr_space(ch : Char) -> Bool { fn attr_value(tag : StringView, name : StringView) -> String? raise XlsxError { @ooxml.attr_value(tag, name) catch { InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled } } From 32628a7b9899686ab3eb50b55536baa9d0746ffb Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 08:37:50 +0800 Subject: [PATCH 040/150] fix: unify OPC part-name identity --- ooxml/read_parse.mbt | 32 ++++++++++++++++++++++++-------- ooxml/read_parse_test.mbt | 19 +++++++++++++++++-- xlsx/read.mbt | 16 ++++++++-------- xlsx/read_namespace_test.mbt | 27 +++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 18 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index ee1988e5..66ff695c 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -284,14 +284,27 @@ pub fn parse_external_relationship_targets( ///| priv struct PackageContentTypes { overrides : Map[String, String] + override_part_names : Map[String, String] defaults : Map[String, String] } +///| +/// Returns the ASCII-case-insensitive identity key used for OPC part names. +/// The leading slash, when present, remains part of the key. +pub fn package_part_name_key(name : StringView) -> String { + let output = StringBuilder::new(size_hint=name.length()) + for character in name { + output.write_char(character.to_ascii_lowercase()) + } + output.to_string() +} + ///| fn parse_content_types( xml : StringView, ) -> PackageContentTypes raise ParseXmlError { let overrides : Map[String, String] = Map([]) + let override_part_names : Map[String, String] = Map([]) let defaults : Map[String, String] = Map([]) let scanner = XmlStartTagScanner::new(xml) if !scanner.next() || @@ -322,10 +335,12 @@ fn parse_content_types( if part_name == "" || content_type == "" { raise InvalidXml(msg="content type override value is empty") } - if overrides.contains(part_name) { + let part_name_key = package_part_name_key(part_name) + if overrides.contains(part_name_key) { raise InvalidXml(msg="duplicate content type override") } - overrides[part_name] = content_type + overrides[part_name_key] = content_type + override_part_names[part_name_key] = part_name } "Default" => { let extension = match scanner.attribute("", "Extension") { @@ -348,7 +363,7 @@ fn parse_content_types( _ => raise InvalidXml(msg="content types element is invalid") } } - { overrides, defaults } + { overrides, override_part_names, defaults } } ///| @@ -358,9 +373,9 @@ pub fn find_override_part_by_content_types( ) -> String? raise ParseXmlError { let content_types_value = parse_content_types(xml) for wanted in content_types { - for part_name, content_type in content_types_value.overrides { + for part_name_key, content_type in content_types_value.overrides { if content_type == wanted { - return Some(part_name) + return content_types_value.override_part_names.get(part_name_key) } } } @@ -368,8 +383,9 @@ pub fn find_override_part_by_content_types( } ///| -/// Returns the declared content type for `part_name`. An exact override wins; -/// otherwise the case-insensitive extension default is used. +/// Returns the declared content type for `part_name`. An ASCII-case-insensitive +/// OPC part-name override wins; otherwise the case-insensitive extension +/// default is used. pub fn find_part_content_type( xml : StringView, part_name : StringView, @@ -380,7 +396,7 @@ pub fn find_part_content_type( } else { "/" + part_name.to_owned() } - match content_types_value.overrides.get(normalized) { + match content_types_value.overrides.get(package_part_name_key(normalized)) { Some(value) => Some(value) None => { let filename = match normalized.rev_find("/") { diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index e606cf89..ea78f69a 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -225,14 +225,14 @@ test "ooxml read_parse: prefixed content types decode names and media types" { } ///| -test "ooxml read_parse: content type lookup honors exact override then default" { +test "ooxml read_parse: content type lookup honors part identity then default" { let xml = #| #| #| #| assert_eq( - @ooxml.find_part_content_type(xml, "custom/book.xml"), + @ooxml.find_part_content_type(xml, "CUSTOM/BOOK.XML"), Some("application/workbook+xml"), ) assert_eq( @@ -240,3 +240,18 @@ test "ooxml read_parse: content type lookup honors exact override then default" Some("application/default+xml"), ) } + +///| +test "ooxml read_parse: content type overrides reject case-variant duplicates" { + let xml = + #| + #| + #| + #| + try @ooxml.find_part_content_type(xml, "custom/book.xml") catch { + InvalidXml(msg~) => inspect(msg, content="duplicate content type override") + _ => fail("unexpected content type parse cancellation") + } noraise { + _ => fail("expected case-variant duplicate override rejection") + } +} diff --git a/xlsx/read.mbt b/xlsx/read.mbt index a2794fdf..f298d5fd 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3084,7 +3084,7 @@ fn first_existing_rel_target_path( ) -> String? raise XlsxError { for _, target in targets { let path = actual_relationship_target_path(source_part, target, part_names) - if part_names.contains(path.to_lower()) { + if part_names.contains(@ooxml.package_part_name_key(path)) { return Some(path) } } @@ -3099,7 +3099,7 @@ fn first_existing_workbook_rel_target_path( ) -> String? raise XlsxError { for _, target in targets { let path = resolve_part_rel_target(workbook_part, target) - match part_names.get(path.to_lower()) { + match part_names.get(@ooxml.package_part_name_key(path)) { Some(actual) => return Some(actual) None => () } @@ -3114,7 +3114,7 @@ fn archive_part_name_index( let names : Map[String, String] = Map([]) for entry in archive.entries() { let name = entry.name() - let key = name.to_lower() + let key = @ooxml.package_part_name_key(name) match names.get(key) { Some(existing) if existing != name => raise InvalidPackage(msg="case-insensitive ZIP part name collision") @@ -3130,7 +3130,7 @@ fn actual_archive_part_path( part_names : Map[String, String], requested : String, ) -> String { - part_names.get(requested.to_lower()).unwrap_or(requested) + part_names.get(@ooxml.package_part_name_key(requested)).unwrap_or(requested) } ///| @@ -3148,7 +3148,7 @@ fn actual_relationship_target_path( part_names : Map[String, String], ) -> String raise XlsxError { let source_relative = resolve_part_rel_target(source_part, target) - match part_names.get(source_relative.to_lower()) { + match part_names.get(@ooxml.package_part_name_key(source_relative)) { Some(actual) => actual None => { // Some producers emit package-root paths without the leading slash. @@ -3158,7 +3158,7 @@ fn actual_relationship_target_path( let package_root = try normalize_rel_part_path(target) catch { _ => None } noraise { - path => part_names.get(path.to_lower()) + path => part_names.get(@ooxml.package_part_name_key(path)) } package_root.unwrap_or(source_relative) } @@ -3188,7 +3188,7 @@ fn workbook_related_part_path( resolve_part_rel_target(workbook_part, target), ), ) - None => part_names.get(fallback.to_owned().to_lower()) + None => part_names.get(@ooxml.package_part_name_key(fallback)) } } } @@ -3489,7 +3489,7 @@ fn read_zip_archive_core_unchecked( part_names, resolve_part_rel_target(workbook_xml_part_path, target), ) - let part_key = path.to_lower() + let part_key = @ooxml.package_part_name_key(path) if seen_sheet_parts.contains(part_key) { raise InvalidXml(msg="sheet part referenced more than once") } diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index ede8f93f..dc7f49e1 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -153,6 +153,33 @@ test "mixed-case workbook media type reads end to end" { ) } +///| +test "mixed-case content-type PartName matches the workbook part" { + let archive = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard archive.get("[Content_Types].xml") is Some(content_type_bytes) else { + fail("missing content types fixture part") + } + let content_types = @encoding/utf8.decode(content_type_bytes) catch { + _ => fail("content types fixture is not UTF-8") + } + let updated = content_types.replace( + old="PartName=\"/xl/workbook.xml\"", + new="PartName=\"/XL/WORKBOOK.XML\"", + ) + if updated == content_types { + fail("workbook PartName marker was not found") + } + assert_true( + archive.replace("[Content_Types].xml", @encoding/utf8.encode(updated)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + debug_inspect(parsed.get_sheet_list(), content="[\"Qualified\"]") + debug_inspect( + parsed.get_cell_value("Qualified", "A1"), + content="Some(\"qualified-value\")", + ) +} + ///| test "SpreadsheetML-looking extension payload cannot create sheets or cells" { for strict in [false, true] { From 66fbe40b4165d193605a4e091c391c899c93aa04 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 08:38:21 +0800 Subject: [PATCH 041/150] chore: refresh package interfaces --- ooxml/pkg.generated.mbti | 11 +++++++---- zip/pkg.generated.mbti | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 71837fcb..a0308782 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -8,9 +8,9 @@ import { // Values pub fn attr_value(StringView, StringView) -> String? raise ParseXmlError -pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)], max_output_chars? : Int) -> String raise ParseXmlError +pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)], max_output_chars? : Int, cancelled? : () -> Bool) -> String raise ParseXmlError -pub fn decode_xml_attribute(StringView) -> String raise ParseXmlError +pub fn decode_xml_attribute(StringView, cancelled? : () -> Bool) -> String raise ParseXmlError pub fn escape_xml_attr(StringView) -> String @@ -20,6 +20,8 @@ pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> Str pub fn find_part_content_type(StringView, StringView) -> String? raise ParseXmlError +pub fn package_part_name_key(StringView) -> String + pub fn parse_external_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError pub fn parse_external_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError @@ -28,13 +30,14 @@ pub fn parse_internal_relationship_targets(StringView, StringView) -> Map[String pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError -pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String) -> String? raise ParseXmlError +pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError pub fn xml_needs_escape(StringView, attr~ : Bool) -> Bool // Errors pub suberror ParseXmlError { InvalidXml(msg~ : String) + ReadCancelled } derive(@debug.Debug) #deprecated pub fn ParseXmlError::to_repr(Self) -> @debug.Repr @@ -60,7 +63,7 @@ pub fn XmlStartTagScanner::attribute_view(Self, StringView, StringView) -> Strin pub fn XmlStartTagScanner::depth(Self) -> Int pub fn XmlStartTagScanner::local_name(Self) -> StringView pub fn XmlStartTagScanner::namespace_uri(Self) -> StringView -pub fn XmlStartTagScanner::new(StringView) -> Self +pub fn XmlStartTagScanner::new(StringView, cancelled? : () -> Bool) -> Self pub fn XmlStartTagScanner::next(Self) -> Bool raise ParseXmlError pub fn XmlStartTagScanner::parent_local_name(Self) -> StringView? pub fn XmlStartTagScanner::parent_namespace_uri(Self) -> String? diff --git a/zip/pkg.generated.mbti b/zip/pkg.generated.mbti index d91d11bc..3e062f59 100644 --- a/zip/pkg.generated.mbti +++ b/zip/pkg.generated.mbti @@ -8,6 +8,8 @@ import { // Values pub fn crc32(BytesView) -> UInt +pub fn crc32_cancellable(BytesView, cancelled? : () -> Bool) -> UInt raise ZipError + pub fn gunzip(BytesView) -> Bytes raise ZipError pub fn gzip(BytesView) -> Bytes raise ZipError From 9ff84724a3168cfd5377e55a1bdcf7cefb466ad4 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 09:36:29 +0800 Subject: [PATCH 042/150] fix: resolve OPC physical part names safely --- ooxml/moon.pkg | 1 + ooxml/opc_part_name.mbt | 237 ++++++++++++++++++++++++++++++++++ ooxml/pkg.generated.mbti | 4 + xlsx/read.mbt | 117 ++++++++++++++--- xlsx/read_namespace_test.mbt | 49 ++++++- xlsx/read_sheet_rel_parts.mbt | 2 +- 6 files changed, 391 insertions(+), 19 deletions(-) create mode 100644 ooxml/opc_part_name.mbt diff --git a/ooxml/moon.pkg b/ooxml/moon.pkg index b619fd35..45cfbe27 100644 --- a/ooxml/moon.pkg +++ b/ooxml/moon.pkg @@ -1,4 +1,5 @@ import { "moonbitlang/core/debug", "moonbitlang/core/string", + "moonbitlang/core/encoding/utf8", } diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt new file mode 100644 index 00000000..f480dfd9 --- /dev/null +++ b/ooxml/opc_part_name.mbt @@ -0,0 +1,237 @@ +///| +fn opc_ascii_hex_digit(character : Char) -> Bool { + (character >= '0' && character <= '9') || + (character >= 'A' && character <= 'F') || + (character >= 'a' && character <= 'f') +} + +///| +fn opc_ascii_hex_value(character : Char) -> Int { + if character >= '0' && character <= '9' { + character.to_int() - '0'.to_int() + } else if character >= 'A' && character <= 'F' { + character.to_int() - 'A'.to_int() + 10 + } else { + character.to_int() - 'a'.to_int() + 10 + } +} + +///| +fn opc_percent_byte(characters : ArrayView[Char], index : Int) -> Int? { + if index + 2 >= characters.length() || + characters[index] != '%' || + !opc_ascii_hex_digit(characters[index + 1]) || + !opc_ascii_hex_digit(characters[index + 2]) { + None + } else { + Some( + opc_ascii_hex_value(characters[index + 1]) * 16 + + opc_ascii_hex_value(characters[index + 2]), + ) + } +} + +///| +fn opc_uri_unreserved(character : Char) -> Bool { + character.is_ascii_alphabetic() || + character.is_ascii_digit() || + character == '-' || + character == '.' || + character == '_' || + character == '~' +} + +///| +fn opc_uri_unreserved_byte(value : Int) -> Bool { + (value >= 'A'.to_int() && value <= 'Z'.to_int()) || + (value >= 'a'.to_int() && value <= 'z'.to_int()) || + (value >= '0'.to_int() && value <= '9'.to_int()) || + value == '-'.to_int() || + value == '.'.to_int() || + value == '_'.to_int() || + value == '~'.to_int() +} + +///| +fn opc_uri_sub_delimiter(character : Char) -> Bool { + character == '!' || + character == '$' || + character == '&' || + character.to_int() == 0x27 || + character == '(' || + character == ')' || + character == '*' || + character == '+' || + character == ',' || + character == ';' || + character == '=' +} + +///| +fn opc_iri_ucschar(character : Char) -> Bool { + let value = character.to_int() + (value >= 0x00a0 && value <= 0xd7ff) || + (value >= 0xf900 && value <= 0xfdcf) || + (value >= 0xfdf0 && value <= 0xffef) || + (value >= 0x10000 && value <= 0x1fffd) || + (value >= 0x20000 && value <= 0x2fffd) || + (value >= 0x30000 && value <= 0x3fffd) || + (value >= 0x40000 && value <= 0x4fffd) || + (value >= 0x50000 && value <= 0x5fffd) || + (value >= 0x60000 && value <= 0x6fffd) || + (value >= 0x70000 && value <= 0x7fffd) || + (value >= 0x80000 && value <= 0x8fffd) || + (value >= 0x90000 && value <= 0x9fffd) || + (value >= 0xa0000 && value <= 0xafffd) || + (value >= 0xb0000 && value <= 0xbfffd) || + (value >= 0xc0000 && value <= 0xcfffd) || + (value >= 0xd0000 && value <= 0xdfffd) || + (value >= 0xe1000 && value <= 0xefffd) +} + +///| +fn opc_utf8_percent_length(first : Int) -> Int? { + if first >= 0xc2 && first <= 0xdf { + Some(2) + } else if first >= 0xe0 && first <= 0xef { + Some(3) + } else if first >= 0xf0 && first <= 0xf4 { + Some(4) + } else { + None + } +} + +///| +fn opc_decode_percent_scalar( + characters : ArrayView[Char], + index : Int, +) -> (String, Char, Int)? { + guard opc_percent_byte(characters, index) is Some(first) && first >= 0x80 else { + return None + } + guard opc_utf8_percent_length(first) is Some(length) else { return None } + let bytes : Array[Byte] = [] + for ordinal in 0.. return None + } + let decoded_characters = decoded.to_array() + guard decoded_characters.length() == 1 else { return None } + Some((decoded, decoded_characters[0], index + length * 3)) +} + +///| +/// Reports whether one logical OPC PartName segment is canonical under the +/// RFC 3987 mapping required by ECMA-376 Part 2. +pub fn is_valid_opc_part_segment(segment : StringView) -> Bool { + if segment == "" || segment.has_suffix(".") { + return false + } + let characters = segment.to_array() + let mut index = 0 + while index < characters.length() { + let character = characters[index] + if character == '%' { + guard opc_percent_byte(characters, index) is Some(decoded) else { + return false + } + if decoded == 0x2f || decoded == 0x5c || opc_uri_unreserved_byte(decoded) { + return false + } + if decoded >= 0x80 { + match opc_decode_percent_scalar(characters, index) { + Some((_, scalar, next)) => { + if opc_iri_ucschar(scalar) { + return false + } + index = next + } + None => index = index + 3 + } + } else { + index = index + 3 + } + } else { + if !opc_uri_unreserved(character) && + !opc_iri_ucschar(character) && + !opc_uri_sub_delimiter(character) && + character != ':' && + character != '@' { + return false + } + index = index + 1 + } + } + true +} + +///| +/// Maps an ASCII physical ZIP item name to its logical OPC PartName. Literal +/// non-ASCII IRI scalars are restored from their canonical UTF-8 `%HH` +/// projection; opaque legal percent escapes remain unchanged. +pub fn logical_opc_part_name_from_zip_item_name(name : StringView) -> String? { + if name == "" || name.has_prefix("/") || name.has_suffix("/") { + return None + } + let characters = name.to_array() + let output = StringBuilder::new(size_hint=name.length()) + let mut index = 0 + while index < characters.length() { + let character = characters[index] + if !character.is_ascii() { + return None + } + if character == '%' { + guard opc_percent_byte(characters, index) is Some(decoded) else { + return None + } + if decoded >= 0x80 { + match opc_decode_percent_scalar(characters, index) { + Some((scalar_text, scalar, next)) if opc_iri_ucschar(scalar) => { + output.write_string(scalar_text) + index = next + } + _ => { + output.write_char(characters[index]) + output.write_char(characters[index + 1]) + output.write_char(characters[index + 2]) + index = index + 3 + } + } + } else { + output.write_char(characters[index]) + output.write_char(characters[index + 1]) + output.write_char(characters[index + 2]) + index = index + 3 + } + } else { + output.write_char(character) + index = index + 1 + } + } + let logical = output.to_string() + for segment in logical.split("/") { + if !is_valid_opc_part_segment(segment) { + return None + } + } + Some(logical) +} + +///| +test "OPC physical names restore Unicode logical identity" { + debug_inspect( + logical_opc_part_name_from_zip_item_name("custom/%E6%96%87.xml"), + content="Some(\"custom/文.xml\")", + ) + debug_inspect( + logical_opc_part_name_from_zip_item_name("custom/文.xml"), + content="None", + ) +} diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index a0308782..bf67be43 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -20,6 +20,10 @@ pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> Str pub fn find_part_content_type(StringView, StringView) -> String? raise ParseXmlError +pub fn is_valid_opc_part_segment(StringView) -> Bool + +pub fn logical_opc_part_name_from_zip_item_name(StringView) -> String? + pub fn package_part_name_key(StringView) -> String pub fn parse_external_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError diff --git a/xlsx/read.mbt b/xlsx/read.mbt index f298d5fd..d995f2e7 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3098,7 +3098,10 @@ fn first_existing_workbook_rel_target_path( part_names : Map[String, String], ) -> String? raise XlsxError { for _, target in targets { - let path = resolve_part_rel_target(workbook_part, target) + let path = resolve_part_rel_target( + logical_archive_part_path(workbook_part), + target, + ) match part_names.get(@ooxml.package_part_name_key(path)) { Some(actual) => return Some(actual) None => () @@ -3113,18 +3116,73 @@ fn archive_part_name_index( ) -> Map[String, String] raise XlsxError { let names : Map[String, String] = Map([]) for entry in archive.entries() { - let name = entry.name() - let key = @ooxml.package_part_name_key(name) + let physical_name = entry.name() + if physical_name.has_suffix("/") { + continue + } + let logical_name = if physical_name == content_types_part_path { + physical_name + } else if @ooxml.package_part_name_key(physical_name) == + @ooxml.package_part_name_key(content_types_part_path) { + raise InvalidPackage( + msg="reserved content-types part name has invalid spelling", + ) + } else { + match @ooxml.logical_opc_part_name_from_zip_item_name(physical_name) { + Some(value) => value + None => raise InvalidPackage(msg="invalid OPC ZIP item name") + } + } + let key = @ooxml.package_part_name_key(logical_name) match names.get(key) { - Some(existing) if existing != name => - raise InvalidPackage(msg="case-insensitive ZIP part name collision") - Some(_) => () - None => names[key] = name + Some(_) => + raise InvalidPackage(msg="duplicate or equivalent OPC part name") + None => () } + names[key] = physical_name } names } +///| +test "read archive index maps physical Unicode OPC names to logical identity" { + let archive = @zip.Archive::new() + archive.add(content_types_part_path, b"manifest") + archive.add("custom/%E6%96%87.xml", b"workbook") + let names = archive_part_name_index(archive) + debug_inspect( + names.get(@ooxml.package_part_name_key("custom/文.xml")), + content="Some(\"custom/%E6%96%87.xml\")", + ) +} + +///| +test "read archive index rejects ambiguous and misspelled reserved entries" { + let duplicate = @zip.Archive::new() + duplicate.add(content_types_part_path, b"first") + duplicate.add(content_types_part_path, b"second") + try archive_part_name_index(duplicate) catch { + InvalidPackage(msg~) => + inspect(msg, content="duplicate or equivalent OPC part name") + _ => fail("unexpected duplicate OPC part error") + } noraise { + _ => fail("expected duplicate OPC part rejection") + } + + let misspelled = @zip.Archive::new() + misspelled.add("[content_types].xml", b"manifest") + try archive_part_name_index(misspelled) catch { + InvalidPackage(msg~) => + inspect( + msg, + content="reserved content-types part name has invalid spelling", + ) + _ => fail("unexpected reserved OPC part error") + } noraise { + _ => fail("expected reserved OPC part spelling rejection") + } +} + ///| fn actual_archive_part_path( part_names : Map[String, String], @@ -3133,12 +3191,30 @@ fn actual_archive_part_path( part_names.get(@ooxml.package_part_name_key(requested)).unwrap_or(requested) } +///| +/// Returns the logical OPC name for either a logical PartName or its physical +/// ASCII ZIP projection. Relationship resolution must never use the physical +/// `%HH` spelling because relationships and content types live in the logical +/// PartName space. +fn logical_archive_part_path(path : StringView) -> String { + if path == content_types_part_path { + content_types_part_path + } else { + @ooxml.logical_opc_part_name_from_zip_item_name(path).unwrap_or( + path.to_owned(), + ) + } +} + ///| fn actual_relationship_part_path( source_part : StringView, part_names : Map[String, String], ) -> String raise XlsxError { - actual_archive_part_path(part_names, rels_path_for_part(source_part)) + actual_archive_part_path( + part_names, + rels_path_for_part(logical_archive_part_path(source_part)), + ) } ///| @@ -3147,7 +3223,10 @@ fn actual_relationship_target_path( target : StringView, part_names : Map[String, String], ) -> String raise XlsxError { - let source_relative = resolve_part_rel_target(source_part, target) + let source_relative = resolve_part_rel_target( + logical_archive_part_path(source_part), + target, + ) match part_names.get(@ooxml.package_part_name_key(source_relative)) { Some(actual) => actual None => { @@ -3185,7 +3264,10 @@ fn workbook_related_part_path( Some( actual_archive_part_path( part_names, - resolve_part_rel_target(workbook_part, target), + resolve_part_rel_target( + logical_archive_part_path(workbook_part), + target, + ), ), ) None => part_names.get(@ooxml.package_part_name_key(fallback)) @@ -3244,9 +3326,8 @@ fn read_zip_archive_core_unchecked( ) read_budget.checkpoint() read_budget.charge_work(workbook_xml.length()) - let workbook_rels_path = actual_archive_part_path( - part_names, - rels_path_for_part(workbook_xml_part_path), + let workbook_rels_path = actual_relationship_part_path( + workbook_xml_part_path, part_names, ) let workbook_rels = match archive.get(workbook_rels_path) { Some(value) => decode(value) @@ -3443,7 +3524,10 @@ fn read_zip_archive_core_unchecked( Some( actual_archive_part_path( part_names, - resolve_part_rel_target(workbook_xml_part_path, target), + resolve_part_rel_target( + logical_archive_part_path(workbook_xml_part_path), + target, + ), ), ) None => None @@ -3487,7 +3571,10 @@ fn read_zip_archive_core_unchecked( } let path = actual_archive_part_path( part_names, - resolve_part_rel_target(workbook_xml_part_path, target), + resolve_part_rel_target( + logical_archive_part_path(workbook_xml_part_path), + target, + ), ) let part_key = @ooxml.package_part_name_key(path) if seen_sheet_parts.contains(part_key) { diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index dc7f49e1..4767ed99 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -34,17 +34,17 @@ fn namespace_qualified_xlsx_fixture( "xl/_rels/workbook.xml.rels" } let worksheet_path = if entity_paths { - "data/a b.xml" + "data/a%20b.xml" } else { "xl/worksheets/sheet.xml" } let worksheet_part_name = if entity_paths { - "/data/a b.xml" + "/data/a%20b.xml" } else { "/xl/worksheets/sheet.xml" } let worksheet_target = if entity_paths { - "../data/a b.xml" + "../data/a%20b.xml" } else { "worksheets/sheet.xml" } @@ -91,6 +91,49 @@ fn namespace_qualified_xlsx_fixture( @zip.write(archive) } +///| +fn unicode_part_name_xlsx_fixture() -> Bytes raise { + let spreadsheet_namespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + let relationship_namespace = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + let relationship_prefix = relationship_namespace + "/" + let content_types = + #| + #| + #| + #| + #| + #| + let root_relationships = + #| + #| + #| + let workbook = "" + let workbook_relationships = "" + let worksheet = "0" + let shared_strings = "unicode-part" + let archive = @zip.Archive::new() + archive.add("[Content_Types].xml", @encoding/utf8.encode(content_types)) + archive.add("_rels/.rels", @encoding/utf8.encode(root_relationships)) + archive.add("custom/%E6%96%87.xml", @encoding/utf8.encode(workbook)) + archive.add( + "custom/_rels/%E6%96%87.xml.rels", + @encoding/utf8.encode(workbook_relationships), + ) + archive.add("xl/worksheets/sheet.xml", @encoding/utf8.encode(worksheet)) + archive.add("xl/sharedStrings.xml", @encoding/utf8.encode(shared_strings)) + @zip.write(archive) +} + +///| +test "physical ZIP names follow Unicode logical OPC part names end to end" { + let workbook = @xlsx.read(unicode_part_name_xlsx_fixture()) + debug_inspect(workbook.get_sheet_list(), content="[\"Unicode\"]") + debug_inspect( + workbook.get_cell_value("Unicode", "A1"), + content="Some(\"unicode-part\")", + ) +} + ///| test "namespace-qualified Transitional and Strict workbook parts read end to end" { for strict in [false, true] { diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index b3177605..415874ef 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -352,7 +352,7 @@ fn parse_slicer_cache_definitions( for _rel_id, target in targets { let cache_path = actual_archive_part_path( part_names, - resolve_part_rel_target(workbook_part, target), + resolve_part_rel_target(logical_archive_part_path(workbook_part), target), ) let cache_bytes = match archive.get(cache_path) { Some(value) => value From 8fa13304ce0c6d0919875c1b149c82359eba386d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 09:56:44 +0800 Subject: [PATCH 043/150] fix: harden XLSX XML core parsing --- ooxml/pkg.generated.mbti | 2 + ooxml/start_tag_scanner.mbt | 139 +++++++++++++++++++++++++++++++++ xlsx/ooxml_utils.mbt | 83 +++++++++++++++----- xlsx/read.mbt | 54 +++++++++++-- xlsx/read_namespace_test.mbt | 143 +++++++++++++++++++++++++++++++++- xlsx/read_shared_strings.mbt | 106 +++++++++++++++++-------- xlsx/read_sheet_rel_parts.mbt | 2 +- xlsx/read_worksheet_xml.mbt | 30 +++---- xlsx/read_xml_namespaces.mbt | 59 ++++++++++++-- 9 files changed, 526 insertions(+), 92 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index bf67be43..239ece9f 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -36,6 +36,8 @@ pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError +pub fn remove_xml_foreign_namespace_subtrees(StringView, ArrayView[String], cancelled? : () -> Bool) -> String? raise ParseXmlError + pub fn xml_needs_escape(StringView, attr~ : Bool) -> Bool // Errors diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 3108c372..f38cb8e7 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -75,6 +75,7 @@ pub struct XmlStartTagScanner { priv mut current_namespace_uri : String priv capture_element_namespaces : Array[String] priv capture_element_local_name : String + priv retain_element_namespaces : Array[String] priv mut capture_start : Int? priv mut capture_depth : Int } @@ -891,6 +892,7 @@ fn xml_start_tag_scanner_with_rewrites( rewrite_output_limit : Int, capture_element_namespaces? : Array[String] = [], capture_element_local_name? : String = "", + retain_element_namespaces? : Array[String] = [], cancelled? : () -> Bool = () => false, ) -> XmlStartTagScanner { { @@ -927,6 +929,7 @@ fn xml_start_tag_scanner_with_rewrites( current_namespace_uri: "", capture_element_namespaces, capture_element_local_name, + retain_element_namespaces, capture_start: None, capture_depth: 0, } @@ -1041,6 +1044,49 @@ pub fn XmlStartTagScanner::next( if self.depth == 0 && !is_attr_space(self.xml[self.index]) { raise InvalidXml(msg="XML text is outside the document element") } + let unit = self.xml[self.index] + if unit == ('&' : UInt16) { + let entity_end = match + find_xml_sequence( + self.xml, + self.index + 1, + ";", + cancelled=self.cancelled, + ) { + Some(found) => found + None => raise InvalidXml(msg="XML entity is not closed") + } + if xml_entity_value(self.xml[self.index + 1:entity_end]) is None { + raise InvalidXml(msg="XML entity is invalid") + } + self.index = entity_end + 1 + continue + } + if unit == (']' : UInt16) && + self.index + 2 < self.xml.length() && + self.xml[self.index + 1] == (']' : UInt16) && + self.xml[self.index + 2] == ('>' : UInt16) { + raise InvalidXml(msg="XML character data contains ']]>'") + } + if unit < 0x20 && + unit != ('\t' : UInt16) && + unit != ('\n' : UInt16) && + unit != ('\r' : UInt16) { + raise InvalidXml(msg="XML character data contains an invalid character") + } + if unit == 0xfffe || unit == 0xffff || unit.is_trailing_surrogate() { + raise InvalidXml(msg="XML character data contains an invalid character") + } + if unit.is_leading_surrogate() { + if self.index + 1 >= self.xml.length() || + !self.xml[self.index + 1].is_trailing_surrogate() { + raise InvalidXml( + msg="XML character data contains an invalid character", + ) + } + self.index = self.index + 2 + continue + } self.index = self.index + 1 } if self.index >= self.xml.length() { @@ -1180,6 +1226,21 @@ pub fn XmlStartTagScanner::next( } } } + if self.capture_start is None && + self.retain_element_namespaces.length() > 0 && + scope_depth > 1 { + let mut retained = false + for wanted in self.retain_element_namespaces { + if xml_string_equals_view(wanted, self.current_namespace_uri) { + retained = true + break + } + } + if !retained { + self.capture_start = Some(start) + self.capture_depth = scope_depth + } + } self.index = tag_end + 1 if self_closing { match self.capture_start { @@ -1304,6 +1365,55 @@ pub fn remove_xml_expanded_name_subtrees( Some(scanner.rewrite_output.to_string()) } +///| +/// Removes outermost descendant subtrees whose expanded element namespace is +/// not in `element_namespaces`. The document element itself must be in the +/// retained set. This creates a safe namespace-oblivious core view while the +/// caller may retain the original XML separately for extension round-tripping. +pub fn remove_xml_foreign_namespace_subtrees( + xml : StringView, + element_namespaces : ArrayView[String], + cancelled? : () -> Bool = () => false, +) -> String? raise ParseXmlError { + check_xml_cancelled(cancelled) + if element_namespaces.length() == 0 { + raise InvalidXml(msg="XML retained namespace set is empty") + } + let namespaces : Array[String] = [] + namespaces.append(element_namespaces) + let scanner = xml_start_tag_scanner_with_rewrites( + xml, + [], + Map([]), + false, + xml.length(), + retain_element_namespaces=namespaces, + cancelled~, + ) + if !scanner.next() { + raise InvalidXml(msg="XML document element is missing") + } + let mut root_retained = false + for wanted in namespaces { + if xml_string_equals_view(wanted, scanner.namespace_uri()) { + root_retained = true + break + } + } + if !root_retained { + raise InvalidXml(msg="XML document element namespace is invalid") + } + while scanner.next() { + + } + if scanner.rewrite_count == 0 { + return None + } + scanner.charge_rewrite_output(xml.length() - scanner.rewrite_cursor) + scanner.write_rewrite_source(scanner.rewrite_cursor, xml.length()) + Some(scanner.rewrite_output.to_string()) +} + ///| /// Validates `xml` and rewrites selected expanded names into lexical forms /// expected by a namespace-oblivious parser. Elements in `element_namespaces` @@ -1469,6 +1579,18 @@ test "expanded-name subtree removal preserves text and other namespaces" { ) } +///| +test "foreign namespace subtree removal preserves only the core dialect" { + let xml = + #|corenestedforeign + debug_inspect( + remove_xml_foreign_namespace_subtrees(xml, ["urn:sheet"]), + content=( + #|Some("core") + ), + ) +} + ///| test "start tag scanner rejects undeclared and duplicate expanded attributes" { for @@ -1485,6 +1607,23 @@ test "start tag scanner rejects undeclared and duplicate expanded attributes" { } } +///| +test "start tag scanner rejects malformed character data" { + for xml in ["&bogus;", "bad]]>"] { + let scanner = XmlStartTagScanner::new(xml) + try { + while scanner.next() { + + } + } catch { + InvalidXml(_) => () + _ => fail("unexpected character-data cancellation") + } noraise { + _ => fail("expected malformed character data rejection") + } + } +} + ///| test "start tag scanner restores namespace scopes after self-closing tags" { let scanner = XmlStartTagScanner::new( diff --git a/xlsx/ooxml_utils.mbt b/xlsx/ooxml_utils.mbt index d4ef38a1..6f1ca4f7 100644 --- a/xlsx/ooxml_utils.mbt +++ b/xlsx/ooxml_utils.mbt @@ -30,21 +30,68 @@ fn is_xml_open_tag_boundary(unit : UInt16) -> Bool { } ///| -fn find_xml_open_tag_start(xml : StringView, tag_name : StringView) -> Int? { - let text = xml.to_owned() - let needle = "<\{tag_name.to_owned()}" - let mut offset = 0 - while offset < text.length() { - let relative = match text[offset:].find(needle) { - Some(value) => value - None => return None +fn find_xml_open_tag_start_from( + xml : StringView, + tag_name : StringView, + start_at : Int, +) -> Int? { + let mut start = start_at.max(0) + let name_length = tag_name.length() + while start + 1 + name_length <= xml.length() { + if xml[start] == ('<' : UInt16) { + let mut matches = true + for offset in 0.. Int? { + find_xml_open_tag_start_from(xml, tag_name, 0) +} + +///| +/// Finds an XML end tag and accepts the XML-permitted whitespace between its +/// QName and `>`, for example `` and ``. +fn find_xml_close_tag_from( + xml : StringView, + tag_name : StringView, + start_at : Int, +) -> (Int, Int)? { + let mut start = start_at.max(0) + let name_length = tag_name.length() + while start + 2 + name_length <= xml.length() { + if xml[start] == ('<' : UInt16) && xml[start + 1] == ('/' : UInt16) { + let mut matches = true + for offset in 0..' : UInt16) { + return Some((start, end)) + } + } } - offset = start + 1 + start = start + 1 } None } @@ -68,7 +115,6 @@ fn tag_attributes_in(xml : StringView, tag_name : StringView) -> String? { /// Extract the body text from the first `...` pair. fn extract_tag_body_from(xml : StringView, tag_name : StringView) -> String? { let xml_str = xml.to_owned() - let close_tag = "" let start = match find_xml_open_tag_start(xml_str, tag_name) { Some(pos) => pos None => return None @@ -78,8 +124,8 @@ fn extract_tag_body_from(xml : StringView, tag_name : StringView) -> String? { None => return None } let body_start = open_end + 1 - let end = match xml_str.find(close_tag) { - Some(pos) => pos + let end = match find_xml_close_tag_from(xml_str, tag_name, body_start) { + Some((pos, _)) => pos None => return None } let body = xml_str[body_start:end] @@ -97,7 +143,6 @@ fn extract_tag_body( tag_name : StringView, ) -> String? raise XlsxError { let xml_str = xml.to_owned() - let close_tag = "" let start = match find_xml_open_tag_start(xml_str, tag_name) { Some(pos) => pos None => return None @@ -110,8 +155,8 @@ fn extract_tag_body( return Some("") } let body_start = open_end + 1 - let end = match xml_str.find(close_tag) { - Some(pos) => pos + let end = match find_xml_close_tag_from(xml_str, tag_name, body_start) { + Some((pos, _)) => pos None => raise InvalidXml(msg="tag close missing") } let body = xml_str[body_start:end] diff --git a/xlsx/read.mbt b/xlsx/read.mbt index d995f2e7..ee8ba540 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2779,8 +2779,8 @@ fn parse_inline_string(rest : StringView) -> InlineStringEntry raise XlsxError { None => raise InvalidXml(msg="inline string not closed") } let body_start = start + open_end + 1 - let end = match text.find("") { - Some(pos) => pos + let end = match find_xml_close_tag_from(text, "is", body_start) { + Some((pos, _)) => pos None => raise InvalidXml(msg="inline string end missing") } let body = text[body_start:end] @@ -3326,6 +3326,25 @@ fn read_zip_archive_core_unchecked( ) read_budget.checkpoint() read_budget.charge_work(workbook_xml.length()) + let workbook_core_xml = match + xlsx_core_source_without_foreign( + workbook_source_xml, + "workbook", + cancelled~, + ) { + Some(filtered) => { + read_budget.charge_work(filtered.length()) + let canonical = canonicalize_xlsx_xml( + filtered, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(canonical.length()) + canonical + } + None => workbook_xml + } let workbook_rels_path = actual_relationship_part_path( workbook_xml_part_path, part_names, ) @@ -3342,8 +3361,8 @@ fn read_zip_archive_core_unchecked( for entry in sheets { sheet_names.push(entry.name) } - let defined_names = parse_defined_names(workbook_xml, sheet_names) - let active_index = match parse_active_sheet_index(workbook_xml) { + let defined_names = parse_defined_names(workbook_core_xml, sheet_names) + let active_index = match parse_active_sheet_index(workbook_core_xml) { Some(value) => value None => 0 } @@ -3359,7 +3378,26 @@ fn read_zip_archive_core_unchecked( let shared_strings = match shared_strings_path { Some(path) => match archive.get(path) { - Some(value) => parse_shared_strings(decode(value)) + Some(value) => { + let source = decode_source(value) + read_budget.charge_work(source.length()) + let core_source = match + xlsx_core_source_without_foreign(source, "sst", cancelled~) { + Some(filtered) => filtered + None => source + } + if core_source != source { + read_budget.charge_work(core_source.length()) + } + let canonical = canonicalize_xlsx_xml( + core_source, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(canonical.length()) + parse_shared_strings(canonical) + } None => raise MissingPart(path~) } None => [] @@ -3440,9 +3478,9 @@ fn read_zip_archive_core_unchecked( Some(value) => parse_custom_properties(decode(value)) None => [] } - let workbook_props = parse_workbook_props(workbook_xml) - let calc_props = parse_calc_props(workbook_xml) - let workbook_protection = parse_workbook_protection(workbook_xml) + let workbook_props = parse_workbook_props(workbook_core_xml) + let calc_props = parse_calc_props(workbook_core_xml) + let workbook_protection = parse_workbook_protection(workbook_core_xml) let style_count = styles.length() let cell_images = match archive.get("xl/cellimages.xml") { Some(value) => { diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 4767ed99..d265ce1f 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -317,11 +317,146 @@ test "cell payload cannot escape its element or cross a namespace boundary" { @encoding/utf8.encode(foreign_value), ), ) - try @xlsx.read(@zip.write(archive)) catch { + let foreign_parsed = @xlsx.read(@zip.write(archive)) + debug_inspect( + foreign_parsed.get_cell_value("Qualified", "A1"), + content="Some(\"\")", + ) + debug_inspect( + foreign_parsed.get_cell_value("Qualified", "B1"), + content="Some(\"second\")", + ) +} + +///| +test "foreign core-looking elements cannot supply XLSX data" { + let archive = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard archive.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { + fail("missing worksheet fixture part") + } + let worksheet_xml = @encoding/utf8.decode(worksheet_bytes) catch { + _ => fail("worksheet fixture is not UTF-8") + } + let with_foreign_merge = worksheet_xml.replace( + old="", + new="", + ) + assert_true( + archive.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode(with_foreign_merge), + ), + ) + let parsed = @xlsx.read(@zip.write(archive)) + assert_eq(parsed.get_merge_cells("Qualified"), ["A1:B1"]) + + let foreign_strings = @zip.read( + namespace_qualified_xlsx_fixture(false, false), + ) + assert_true( + foreign_strings.replace( + "xl/sharedStrings.xml", + @encoding/utf8.encode( + "spoof", + ), + ), + ) + try @xlsx.read(@zip.write(foreign_strings)) catch { @xlsx.InvalidXml(msg~) => - inspect(msg, content="worksheet cell payload element is misplaced") - _ => fail("unexpected foreign cell payload error") + inspect(msg, content="sst document element is invalid") + _ => fail("unexpected foreign shared-string error") + } noraise { + _ => fail("expected foreign shared-string root rejection") + } +} + +///| +test "XML whitespace spellings preserve cells values and formulas" { + let archive = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard archive.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { + fail("missing worksheet fixture part") + } + let worksheet_xml = @encoding/utf8.decode(worksheet_bytes) catch { + _ => fail("worksheet fixture is not UTF-8") + } + let spaced = worksheet_xml.replace( + old="0", + new="1+1two", + ) + assert_true( + archive.replace("xl/worksheets/sheet.xml", @encoding/utf8.encode(spaced)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + debug_inspect( + parsed.get_cell_value("Qualified", "A1"), + content="Some(\"two\")", + ) + debug_inspect( + parsed.get_cell_formula("Qualified", "A1"), + content="Some(\"1+1\")", + ) +} + +///| +test "malformed entities fail and phonetic annotations stay out of cell text" { + let malformed = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard malformed.get("xl/worksheets/sheet.xml") is Some(worksheet_bytes) else { + fail("missing worksheet fixture part") + } + let worksheet_xml = @encoding/utf8.decode(worksheet_bytes) catch { + _ => fail("worksheet fixture is not UTF-8") + } + assert_true( + malformed.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode( + worksheet_xml.replace(old="0", new="&bogus;"), + ), + ), + ) + try @xlsx.read(@zip.write(malformed)) catch { + @xlsx.InvalidXml(_) => () + _ => fail("unexpected malformed entity error") } noraise { - _ => fail("expected foreign cell payload rejection") + _ => fail("expected malformed entity rejection") + } + + let phonetic = @zip.read(namespace_qualified_xlsx_fixture(false, false)) + guard phonetic.get("xl/sharedStrings.xml") is Some(shared_bytes) else { + fail("missing shared-string fixture part") + } + let shared_xml = @encoding/utf8.decode(shared_bytes) catch { + _ => fail("shared-string fixture is not UTF-8") } + assert_true( + phonetic.replace( + "xl/sharedStrings.xml", + @encoding/utf8.encode( + shared_xml.replace( + old="qualified-value", + new="漢字かんじ", + ), + ), + ), + ) + assert_true( + phonetic.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode( + worksheet_xml.replace( + old="", + new="東京とうきょう", + ), + ), + ), + ) + let parsed = @xlsx.read(@zip.write(phonetic)) + debug_inspect( + parsed.get_cell_value("Qualified", "A1"), + content="Some(\"漢字\")", + ) + debug_inspect( + parsed.get_cell_value("Qualified", "B1"), + content="Some(\"東京\")", + ) } diff --git a/xlsx/read_shared_strings.mbt b/xlsx/read_shared_strings.mbt index cadcf57c..11d55ea2 100644 --- a/xlsx/read_shared_strings.mbt +++ b/xlsx/read_shared_strings.mbt @@ -11,24 +11,65 @@ priv struct InlineStringEntry { } ///| -fn parse_text_nodes(xml : StringView) -> String { +fn remove_lexical_element_subtrees( + xml : StringView, + tag_name : StringView, +) -> String raise XlsxError { + let first = match find_xml_open_tag_start(xml, tag_name) { + Some(value) => value + None => return xml.to_owned() + } + let output = StringBuilder::new(size_hint=xml.length()) + let mut cursor = 0 + let mut start = first + for ;; { + output.write_view(xml[cursor:start]) + let open_end = match xml_open_tag_end_from(xml, start + 1) { + Some(value) => value + None => raise InvalidXml(msg="\{tag_name.to_owned()} tag not closed") + } + if xml[start + 1:open_end].trim().has_suffix("/") { + cursor = open_end + 1 + } else { + let (close_start, close_end) = match + find_xml_close_tag_from(xml, tag_name, open_end + 1) { + Some(value) => value + None => raise InvalidXml(msg="\{tag_name.to_owned()} close missing") + } + ignore(close_start) + cursor = close_end + 1 + } + match find_xml_open_tag_start_from(xml, tag_name, cursor) { + Some(next) => start = next + None => break + } + } + output.write_view(xml[cursor:]) + output.to_string() +} + +///| +fn parse_text_nodes(xml : StringView) -> String raise XlsxError { + // Phonetic guides (`rPh`) are annotations, not displayed cell text. + let semantic = remove_lexical_element_subtrees(xml, "rPh") let sb = StringBuilder::new() - let mut first = true - for chunk in xml.split(" value + None => raise InvalidXml(msg="shared string text tag not closed") } - let text = chunk.to_owned() - let start = match text.find(">") { - Some(pos) => pos + 1 - None => continue + if semantic[start + 1:open_end].trim().has_suffix("/") { + cursor = open_end + 1 + continue } - let rest = text[start:] - match rest.find("") { - Some(end) => sb.write_view(unescape_xml_text(rest[:end])) - None => () + let (close_start, close_end) = match + find_xml_close_tag_from(semantic, "t", open_end + 1) { + Some(value) => value + None => raise InvalidXml(msg="shared string text close missing") } + sb.write_view(unescape_xml_text(semantic[open_end + 1:close_start])) + cursor = close_end + 1 } sb.to_string() } @@ -307,8 +348,9 @@ fn parse_rich_text_runs(xml : StringView) -> Array[RichTextRun] raise XlsxError } let body_start = after + open_end_rel + 1 let body_rest = xml_str[body_start:] - let close_rel = match body_rest.find("") { - Some(pos) => pos + let (close_rel, close_end_rel) = match + find_xml_close_tag_from(body_rest, "r", 0) { + Some((pos, end)) => (pos, end) None => break } let body_end = body_start + close_rel @@ -316,31 +358,28 @@ fn parse_rich_text_runs(xml : StringView) -> Array[RichTextRun] raise XlsxError let run_text = parse_text_nodes(body) let font = parse_rich_text_font(body) runs.push({ text: run_text, font }) - index = body_end + "".length() + index = body_start + close_end_rel + 1 } runs } ///| -fn parse_shared_strings(xml : StringView) -> Array[SharedStringEntry] { +fn parse_shared_strings( + xml : StringView, +) -> Array[SharedStringEntry] raise XlsxError { let values : Array[SharedStringEntry] = [] - let mut first = true - for chunk in xml.split(" value + None => raise InvalidXml(msg="shared string item tag not closed") } - let text = chunk.to_owned() - let start = match text.find(">") { - Some(pos) => pos + 1 - None => continue - } - let rest = text[start:] - let end = match rest.find("") { - Some(pos) => pos - None => continue + let (close_start, close_end) = match + find_xml_close_tag_from(xml, "si", open_end + 1) { + Some(value) => value + None => raise InvalidXml(msg="shared string item close missing") } - let body = rest[:end] + let body = xml[open_end + 1:close_start] let runs = if body.contains(" [] } if parsed.length() > 0 { @@ -352,6 +391,7 @@ fn parse_shared_strings(xml : StringView) -> Array[SharedStringEntry] { None } values.push({ text: parse_text_nodes(body), runs }) + cursor = close_end + 1 } values } diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index 415874ef..bc66c775 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -69,7 +69,7 @@ fn parse_comments_xml(xml : StringView) -> Array[Comment] raise XlsxError { } else { [] } - let text_value = parse_text_nodes(text_body) + let text_value = parse_text_nodes(text_body) catch { _ => "" } comments.push({ cell, author, diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index b4cace73..6797441f 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -32,11 +32,14 @@ fn parse_worksheet( let cells : Array[Cell] = [] let sheet_data = worksheet_sheet_data_body(xml) let mut first = true - for chunk in sheet_data.split(" pos @@ -67,23 +70,12 @@ fn parse_worksheet( let cell_body = if self_closing { after_open[:0] } else { - match after_open.find("") { - Some(pos) => after_open[:pos] + match find_xml_close_tag_from(after_open, "c", 0) { + Some((pos, _)) => after_open[:pos] None => raise InvalidXml(msg="cell not closed") } } - let formula_start = match cell_body.find("") { - Some(pos) => Some(pos) - None => - match cell_body.find(" Some(pos) - None => - match cell_body.find(" Some(pos) - None => None - } - } - } + let formula_start = find_xml_open_tag_start(cell_body, "f") let (formula, formula_type, formula_ref, formula_shared_index) = match formula_start { Some(pos) => { @@ -130,8 +122,8 @@ fn parse_worksheet( (Some(""), kind, ref_value, si_value) } else { let after_open = f_rest[open_end + 1:] - let close_end = match after_open.find("") { - Some(value) => value + let close_end = match find_xml_close_tag_from(after_open, "f", 0) { + Some((value, _)) => value None => raise InvalidXml(msg="formula not closed") } let content = unescape_xml_text(after_open[:close_end]) @@ -159,8 +151,8 @@ fn parse_worksheet( Some("") } else { let value_rest = v_rest[open_end + 1:] - let v_end = match value_rest.find("") { - Some(end) => end + let v_end = match find_xml_close_tag_from(value_rest, "v", 0) { + Some((end, _)) => end None => raise InvalidXml(msg="cell value not closed") } Some(value_rest[:v_end].to_owned()) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 8643e52b..229a1fd8 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -22,6 +22,34 @@ fn canonicalize_xlsx_xml( } } +///| +/// Validates the SpreadsheetML document element and removes foreign-namespace +/// descendant subtrees from the namespace-oblivious core view. The original +/// canonical view remains available to extension-specific readers. +fn xlsx_core_source_without_foreign( + source : StringView, + root_local_name : StringView, + cancelled? : () -> Bool = () => false, +) -> String? raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(source, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != root_local_name || + ( + scanner.namespace_uri() != transitional_spreadsheet_namespace && + scanner.namespace_uri() != strict_spreadsheet_namespace + ) { + raise InvalidXml( + msg="\{root_local_name.to_owned()} document element is invalid", + ) + } + let dialect = scanner.namespace_uri().to_owned() + @ooxml.remove_xml_foreign_namespace_subtrees(source, [dialect], cancelled~) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } +} + ///| fn validate_xlsx_worksheet_core( xml : StringView, @@ -156,14 +184,29 @@ fn canonicalize_xlsx_worksheet_core( ReadCancelled => raise ReadCancelled } match without_extensions { - Some(core_source) => { - validate_xlsx_worksheet_core(core_source, cancelled~) - canonicalize_xlsx_xml(core_source, max_output_chars~, cancelled~) - } - None => { - validate_xlsx_worksheet_core(source, cancelled~) - full_canonical - } + Some(core_source) => + match + xlsx_core_source_without_foreign(core_source, "worksheet", cancelled~) { + Some(filtered) => { + validate_xlsx_worksheet_core(filtered, cancelled~) + canonicalize_xlsx_xml(filtered, max_output_chars~, cancelled~) + } + None => { + validate_xlsx_worksheet_core(core_source, cancelled~) + canonicalize_xlsx_xml(core_source, max_output_chars~, cancelled~) + } + } + None => + match xlsx_core_source_without_foreign(source, "worksheet", cancelled~) { + Some(filtered) => { + validate_xlsx_worksheet_core(filtered, cancelled~) + canonicalize_xlsx_xml(filtered, max_output_chars~, cancelled~) + } + None => { + validate_xlsx_worksheet_core(source, cancelled~) + full_canonical + } + } } } From cf40d6773e347abc68a9c9f93e007ad856dcf755 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 10:20:56 +0800 Subject: [PATCH 044/150] fix: bound Office format preflight --- docx2html/xml/pkg.generated.mbti | 4 +- docx2html/xml/xml.mbt | 36 ++++ office/cmd/office/main.mbt | 31 ++- office/cmd/office/read_package.mbt | 37 +++- office/format.mbt | 324 ++++++++++++++++++++++++----- office/format_test.mbt | 78 +++++++ office/moon.pkg | 1 + office/pkg.generated.mbti | 6 +- 8 files changed, 447 insertions(+), 70 deletions(-) diff --git a/docx2html/xml/pkg.generated.mbti b/docx2html/xml/pkg.generated.mbti index 971a35b3..e43a9925 100644 --- a/docx2html/xml/pkg.generated.mbti +++ b/docx2html/xml/pkg.generated.mbti @@ -28,6 +28,8 @@ pub fn read_xml_string_strict(String, namespace_map? : Map[String, String]) -> X pub fn read_xml_string_strict_encoded(String, XmlSourceEncoding, namespace_map? : Map[String, String]) -> XmlElement raise @core.DocxError +pub fn read_xml_string_strict_encoded_limited(String, XmlSourceEncoding, XmlReadBudget, namespace_map? : Map[String, String]) -> XmlElement raise @core.DocxError + pub fn read_xml_string_strict_limited(String, XmlReadBudget, namespace_map? : Map[String, String]) -> XmlElement raise @core.DocxError pub fn read_xml_string_strict_mce_encoded_limited(String, XmlSourceEncoding, XmlReadBudget, Array[String]) -> XmlElement raise @core.DocxError @@ -42,7 +44,7 @@ pub fn write_xml_string(XmlElement, namespaces? : Map[String, String]) -> String pub fn xml_element(String, attributes? : Map[String, String], children? : Array[XmlNode]) -> XmlElement -pub fn xml_read_budget(max_source_units~ : Int, max_tokens~ : Int, max_materialized_chars~ : Int, max_token_chars~ : Int) -> XmlReadBudget +pub fn xml_read_budget(max_source_units~ : Int, max_tokens~ : Int, max_materialized_chars~ : Int, max_token_chars~ : Int, cancelled? : () -> Bool) -> XmlReadBudget pub fn xml_text(String) -> XmlNode diff --git a/docx2html/xml/xml.mbt b/docx2html/xml/xml.mbt index 855e79bf..173e051f 100644 --- a/docx2html/xml/xml.mbt +++ b/docx2html/xml/xml.mbt @@ -195,6 +195,26 @@ pub fn read_xml_string_strict_encoded( ) } +///| +/// Strictly parses an already-decoded XML string under a caller-owned +/// cumulative budget while enforcing the declared byte encoding. +pub fn read_xml_string_strict_encoded_limited( + source : String, + source_encoding : XmlSourceEncoding, + budget : XmlReadBudget, + namespace_map? : Map[String, String] = Map([]), +) -> XmlElement raise DocxError { + read_xml_string_with_mode( + source, + namespace_map, + true, + Some(budget), + false, + Some(source_encoding), + false, + ) +} + ///| /// A cumulative allocation budget shared by one or more XML parses. /// @@ -207,6 +227,7 @@ pub struct XmlReadBudget { priv mut remaining_tokens : Int priv mut remaining_materialized_chars : Int priv max_token_chars : Int + priv cancelled : () -> Bool } ///| @@ -217,6 +238,7 @@ pub fn xml_read_budget( max_tokens~ : Int, max_materialized_chars~ : Int, max_token_chars~ : Int, + cancelled? : () -> Bool = () => false, ) -> XmlReadBudget { { remaining_source_units: if max_source_units > 0 { @@ -239,6 +261,16 @@ pub fn xml_read_budget( } else { 0 }, + cancelled, + } +} + +///| +fn XmlReadBudget::checkpoint(self : XmlReadBudget) -> Unit raise DocxError { + if (self.cancelled)() { + // Higher layers that supply a cancellation callback translate this + // internal parser abort back to their own typed cancellation error. + raise InvalidXml(message="XML read cancelled") } } @@ -247,6 +279,7 @@ fn XmlReadBudget::charge_source( self : XmlReadBudget, units : Int, ) -> Unit raise DocxError { + self.checkpoint() if units < 0 || units > self.remaining_source_units { raise @core.docx_xml_resource_limit_error(DocxXmlSourceUnits) } @@ -258,6 +291,7 @@ fn XmlReadBudget::charge_token( self : XmlReadBudget, chars : Int, ) -> Unit raise DocxError { + self.checkpoint() if chars < 0 || chars > self.max_token_chars { raise @core.docx_xml_resource_limit_error(DocxXmlTokenLength) } @@ -280,6 +314,7 @@ pub fn XmlReadBudget::charge_derived_chars( self : XmlReadBudget, chars : Int, ) -> Unit raise DocxError { + self.checkpoint() if chars < 0 || chars > self.remaining_materialized_chars { raise @core.docx_xml_resource_limit_error(DocxXmlMaterializedCharacters) } @@ -291,6 +326,7 @@ fn XmlReadBudget::charge_namespace_bindings( self : XmlReadBudget, operations : Int, ) -> Unit raise DocxError { + self.checkpoint() // Namespace declarations allocate one scope-delta record and later restore // one map binding. Charge both operations before recording the declaration. if operations < 0 || operations > self.remaining_tokens { diff --git a/office/cmd/office/main.mbt b/office/cmd/office/main.mbt index afa50f17..55adb647 100644 --- a/office/cmd/office/main.mbt +++ b/office/cmd/office/main.mbt @@ -68,28 +68,27 @@ fn identify_error( "actual": Json::string(actual.name()), }), ) + ResourceLimit(kind~, limit~, actual~) => + @lib.protocol_error( + "office.resource_limit", + "Office \{bounded_text(kind, 80)} exceeds the configured limit", + details=Json::object({ + "file": Json::string(bounded_file), + "resource": Json::string(bounded_text(kind, 80)), + "limit": Json::number(limit.to_double()), + "actual": Json::number(actual.to_double()), + }), + ) + Cancelled => + @lib.protocol_error("office.cancelled", "Office read was cancelled") } } ///| async fn run_identify(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") - let data = @afs.read_file(file).binary() catch { - error if @async.is_being_cancelled() => raise error - _ => - raise CliFailure( - @lib.protocol_error( - "office.file_read_failed", - "could not read input file: " + bounded_text(file, 160), - details=Json::object({ "file": Json::string(bounded_text(file, 160)) }), - ), - ) - } - check_office_read_cancelled(() => @async.is_being_cancelled()) - let format = @lib.detect_format(file, data) catch { - error => raise CliFailure(identify_error(error, file)) - } - check_office_read_cancelled(() => @async.is_being_cancelled()) + let source = read_office_package(file, cancelled=office_async_cancelled) + let format = source.format if matches.flags.get_or_default("json", false) { println( @lib.output_success( diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index 8a7228e6..a8bad7cf 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -16,6 +16,9 @@ let office_read_max_preserved_source_bytes : Int = 64 * 1024 * 1024 + 65_535 ///| let office_read_max_xml_part_bytes : Int = 16 * 1024 * 1024 +///| +let office_read_max_xml_total_units : Int = 64 * 1024 * 1024 + ///| let office_read_max_workbook_sheets : Int = 256 @@ -32,6 +35,13 @@ struct OfficeReadPackage { archive : @zip.Archive } +///| +/// Centralizes cancellation observation for asynchronous Office commands. +/// CPU-heavy command paths add explicit async yield points around bounded work. +fn office_async_cancelled() -> Bool { + @async.is_being_cancelled() +} + ///| fn check_office_read_cancelled(cancelled : () -> Bool) -> Unit raise CliFailure { if cancelled() { @@ -48,7 +58,32 @@ fn detect_office_archive_format( cancelled : () -> Bool, ) -> @lib.DocumentFormat raise CliFailure { check_office_read_cancelled(cancelled) - let format = @lib.detect_archive_format(file, archive) catch { + let format = @lib.detect_archive_format( + file, + archive, + max_xml_part_bytes=office_read_max_xml_part_bytes, + max_xml_total_units=office_read_max_xml_total_units, + cancelled~, + ) catch { + ResourceLimit(kind~, limit~, actual~) => + raise CliFailure( + @lib.protocol_error( + "office.\{office_read_format_hint(file).name()}.resource_limit", + "Office \{bounded_text(kind, 80)} exceeds the configured limit", + details=Json::object({ + "file": Json::string(bounded_text(file, 160)), + "resource": Json::string(bounded_text(kind, 80)), + "limit": Json::number(limit.to_double()), + "actual": Json::number(actual.to_double()), + }), + ), + ) + Cancelled => { + check_office_read_cancelled(cancelled) + raise CliFailure( + @lib.protocol_error("office.cancelled", "Office read was cancelled"), + ) + } error => raise CliFailure(identify_error(error, file)) } check_office_read_cancelled(cancelled) diff --git a/office/format.mbt b/office/format.mbt index edbd2c2f..782d70a5 100644 --- a/office/format.mbt +++ b/office/format.mbt @@ -25,6 +25,8 @@ pub(all) suberror OfficeError { UnsupportedFileExtension(String) InvalidPackage(String) FormatMismatch(expected~ : DocumentFormat, actual~ : DocumentFormat) + ResourceLimit(kind~ : String, limit~ : Int, actual~ : Int) + Cancelled } derive(Debug, Eq) ///| @@ -40,9 +42,32 @@ pub impl Show for OfficeError with fn output(self, logger) { logger.write_string( "file extension says \{expected.name()}, but package content is \{actual.name()}", ) + ResourceLimit(kind~, limit~, actual~) => + logger.write_string( + "Office \{kind} exceeds limit \{limit} (actual \{actual})", + ) + Cancelled => logger.write_string("Office format detection was cancelled") } } +///| +const DEFAULT_DETECT_MAX_PACKAGE_BYTES : Int = 64 * 1024 * 1024 + +///| +const DEFAULT_DETECT_MAX_ARCHIVE_ENTRIES : Int = 4096 + +///| +const DEFAULT_DETECT_MAX_ENTRY_BYTES : Int = 32 * 1024 * 1024 + +///| +const DEFAULT_DETECT_MAX_TOTAL_BYTES : Int = 128 * 1024 * 1024 + +///| +const DEFAULT_DETECT_MAX_XML_PART_BYTES : Int = 16 * 1024 * 1024 + +///| +const DEFAULT_DETECT_MAX_XML_TOTAL_UNITS : Int = 64 * 1024 * 1024 + ///| const XLSX_MAIN_CONTENT_TYPE : String = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml" @@ -271,7 +296,77 @@ priv struct ArchivePartIndex { priv struct DecodedXmlPart { text : String source_encoding : @xml.XmlSourceEncoding - source_units : Int +} + +///| +/// Shared fail-closed state for one format-detection pass. XML trees are +/// cached so repeated identity and structural checks do not parse or +/// materialize the same package metadata more than once. +priv struct OfficeFormatContext { + max_xml_part_bytes : Int + xml_work_limit : Int + xml_budget : @xml.XmlReadBudget + cancelled : () -> Bool + xml_parts : StableStringMap[@xml.XmlElement] + relationship_parts : StableStringMap[@xml.XmlElement] +} + +///| +fn bounded_scale(value : Int, factor : Int) -> Int { + if value <= 0 || factor <= 0 { + 0 + } else { + let scaled = value.to_int64() * factor.to_int64() + if scaled > 2147483647L { + 2147483647 + } else { + scaled.to_int() + } + } +} + +///| +fn OfficeFormatContext::new( + max_xml_part_bytes : Int, + max_xml_total_units : Int, + cancelled? : () -> Bool = () => false, +) -> OfficeFormatContext { + let xml_work_limit = bounded_scale(max_xml_total_units, 4) + { + max_xml_part_bytes: if max_xml_part_bytes > 0 { + max_xml_part_bytes + } else { + 0 + }, + xml_work_limit, + xml_budget: @xml.xml_read_budget( + max_source_units=if max_xml_total_units > 0 { + max_xml_total_units + } else { + 0 + }, + max_tokens=xml_work_limit, + max_materialized_chars=xml_work_limit, + max_token_chars=if max_xml_part_bytes > 0 { + max_xml_part_bytes + } else { + 0 + }, + cancelled~, + ), + cancelled, + xml_parts: SortedMap([]), + relationship_parts: SortedMap([]), + } +} + +///| +fn OfficeFormatContext::checkpoint( + self : OfficeFormatContext, +) -> Unit raise OfficeError { + if (self.cancelled)() { + raise Cancelled + } } ///| @@ -284,20 +379,43 @@ fn archive_entry_logical_part_name(name : StringView) -> String? { } ///| -fn archive_part_index(archive : @zip.Archive) -> ArchivePartIndex { +fn archive_part_index( + archive : @zip.Archive, + context : OfficeFormatContext, +) -> ArchivePartIndex raise OfficeError { let exact : StableStringMap[BytesView] = SortedMap([]) let canonical : StableStringMap[BytesView] = SortedMap([]) + let physical : StableStringMap[Bool] = SortedMap([]) for entry in archive.entries() { + context.checkpoint() let physical_name = entry.name() + if physical.contains(physical_name) { + raise InvalidPackage("duplicate entry name: \{physical_name}") + } + physical[physical_name] = true + if physical_name.to_lower() == "[content_types].xml" && + physical_name != "[Content_Types].xml" { + raise InvalidPackage( + "reserved content-types manifest has invalid physical spelling: \{physical_name}", + ) + } let logical_name = archive_entry_logical_part_name(physical_name) guard logical_name is Some(name) else { continue } - if !exact.contains(name) { - exact[name] = entry.data() + if opc_part_key(name) == opc_part_key("[Content_Types].xml") && + physical_name != "[Content_Types].xml" { + raise InvalidPackage( + "reserved content-types manifest has invalid physical spelling: \{physical_name}", + ) + } + if exact.contains(name) { + raise InvalidPackage("duplicate logical part name: \{name}") } + exact[name] = entry.data() let key = opc_part_key(name) - if !canonical.contains(key) { - canonical[key] = entry.data() + if canonical.contains(key) { + raise InvalidPackage("duplicate entry name: \{key}") } + canonical[key] = entry.data() } { exact, canonical } } @@ -314,11 +432,20 @@ fn ArchivePartIndex::find(self : ArchivePartIndex, name : String) -> BytesView? fn decode_xml_part( parts : ArchivePartIndex, name : String, + context : OfficeFormatContext, ) -> DecodedXmlPart raise OfficeError { + context.checkpoint() let bytes = match parts.find(name) { Some(bytes) => bytes None => raise InvalidPackage("missing required part: \{name}") } + if bytes.length() > context.max_xml_part_bytes { + raise ResourceLimit( + kind="xml_part_bytes", + limit=context.max_xml_part_bytes, + actual=bytes.length(), + ) + } let (text, source_encoding) : (String, @xml.XmlSourceEncoding) = if bytes.length() >= 2 && bytes[0] == 0xff && @@ -363,56 +490,76 @@ fn decode_xml_part( Utf8, ) } - { text, source_encoding, source_units: bytes.length() } + context.checkpoint() + { text, source_encoding } } ///| fn read_xml_part( parts : ArchivePartIndex, name : String, + context : OfficeFormatContext, ) -> @xml.XmlElement raise OfficeError { - let decoded = decode_xml_part(parts, name) - @xml.read_xml_string_strict_encoded(decoded.text, decoded.source_encoding) catch { - _ => raise InvalidPackage("\{name} is not well-formed XML") - } -} - -///| -fn input_linear_xml_budget(decoded : DecodedXmlPart) -> @xml.XmlReadBudget { - let source_units = if decoded.source_units > decoded.text.length() { - decoded.source_units - } else { - decoded.text.length() + match context.xml_parts.get(opc_part_key(name)) { + Some(value) => return value + None => () } - let scaled = source_units.to_int64() * 32L - let allowance = if scaled <= 0L { - 1 - } else if scaled > 2147483647L { - 2147483647 - } else { - scaled.to_int() + let decoded = decode_xml_part(parts, name, context) + let root = @xml.read_xml_string_strict_encoded_limited( + decoded.text, + decoded.source_encoding, + context.xml_budget, + ) catch { + _ if (context.cancelled)() => raise Cancelled + @docx_core.ResourceLimit(_) => + raise ResourceLimit( + kind="xml_preflight_work_units", + limit=context.xml_work_limit, + actual=if context.xml_work_limit < 2147483647 { + context.xml_work_limit + 1 + } else { + context.xml_work_limit + }, + ) + _ => raise InvalidPackage("\{name} is not well-formed XML") } - @xml.xml_read_budget( - max_source_units=source_units, - max_tokens=allowance, - max_materialized_chars=allowance, - max_token_chars=if source_units > 0 { source_units } else { 1 }, - ) + context.checkpoint() + context.xml_parts[opc_part_key(name)] = root + root } ///| fn read_relationships_part( parts : ArchivePartIndex, name : String, + context : OfficeFormatContext, ) -> @xml.XmlElement raise OfficeError { - let decoded = decode_xml_part(parts, name) - @opc.read_opc_relationships_string_encoded_limited( + match context.relationship_parts.get(opc_part_key(name)) { + Some(value) => return value + None => () + } + let decoded = decode_xml_part(parts, name, context) + let root = @opc.read_opc_relationships_string_encoded_limited( decoded.text, decoded.source_encoding, - input_linear_xml_budget(decoded), + context.xml_budget, ) catch { + _ if (context.cancelled)() => raise Cancelled + @docx_core.ResourceLimit(_) => + raise ResourceLimit( + kind="xml_preflight_work_units", + limit=context.xml_work_limit, + actual=if context.xml_work_limit < 2147483647 { + context.xml_work_limit + 1 + } else { + context.xml_work_limit + }, + ) _ => raise InvalidPackage("\{name} is not well-formed XML") } + context.checkpoint() + context.relationship_parts[opc_part_key(name)] = root + root } ///| @@ -1115,6 +1262,7 @@ fn validate_package_structure( part_index : ArchivePartIndex, main_part : String, require_main_relationships : Bool, + context : OfficeFormatContext, ) -> Array[String] raise OfficeError { let findings : Array[String] = [] let parts : StableStringMap[Int] = SortedMap([]) @@ -1123,6 +1271,7 @@ fn validate_package_structure( let document_parts : StableStringMap[Bool] = SortedMap([]) let part_name_registry = @opc.PartNameRegistry::new() for entry in archive.entries() { + context.checkpoint() let physical_name = entry.name() let is_directory = physical_name.has_suffix("/") let item_path = if is_directory && physical_name.length() > 0 { @@ -1210,7 +1359,7 @@ fn validate_package_structure( let defaults : StableStringMap[String] = SortedMap([]) let overrides : StableStringMap[String] = SortedMap([]) - let content_types = read_xml_part(part_index, "[Content_Types].xml") + let content_types = read_xml_part(part_index, "[Content_Types].xml", context) match unexpected_xml_attribute(content_types, []) { Some(attribute) => findings.push( @@ -1219,6 +1368,7 @@ fn validate_package_structure( None => () } for child in content_types.children { + context.checkpoint() match child { XmlText(text) => if !text.trim().is_empty() { @@ -1228,6 +1378,7 @@ fn validate_package_structure( } } for child in content_types.children { + context.checkpoint() guard child is XmlElement(element) else { continue } if element.name == CONTENT_TYPES_DEFAULT { match unexpected_xml_attribute(element, ["Extension", "ContentType"]) { @@ -1333,6 +1484,7 @@ fn validate_package_structure( } } for entry in archive.entries() { + context.checkpoint() let physical_name = entry.name() guard archive_entry_logical_part_name(physical_name) is Some(name) else { continue @@ -1362,6 +1514,7 @@ fn validate_package_structure( } for entry in archive.entries() { + context.checkpoint() guard archive_entry_logical_part_name(entry.name()) is Some(relationship_part) else { continue @@ -1399,7 +1552,9 @@ fn validate_package_structure( continue } } - let relationships = read_relationships_part(part_index, relationship_part) + let relationships = read_relationships_part( + part_index, relationship_part, context, + ) if relationships.name != RELATIONSHIPS_ROOT { findings.push("\{relationship_part} root element is not Relationships") continue @@ -1410,6 +1565,7 @@ fn validate_package_structure( ) } for child in relationships.children { + context.checkpoint() match child { XmlText(text) => if !text.trim().is_empty() { @@ -1420,6 +1576,7 @@ fn validate_package_structure( } let ids : StableStringMap[Bool] = SortedMap([]) for child in relationships.children { + context.checkpoint() guard child is XmlElement(element) else { continue } if element.name != RELATIONSHIP_ELEMENT { findings.push( @@ -1537,14 +1694,16 @@ fn validate_main_identity( part_index : ArchivePartIndex, format : DocumentFormat, declared_main_part : String, + context : OfficeFormatContext, ) -> Unit raise OfficeError { let label = if format is Xlsx { "XLSX" } else { "DOCX" } - let root_rels = read_relationships_part(part_index, "_rels/.rels") + let root_rels = read_relationships_part(part_index, "_rels/.rels", context) if root_rels.name != RELATIONSHIPS_ROOT { raise InvalidPackage("_rels/.rels root element is not Relationships") } let main_targets : Array[(String, String)] = [] for child in root_rels.children { + context.checkpoint() guard child is XmlElement(element) && element.name == RELATIONSHIP_ELEMENT else { continue } @@ -1596,7 +1755,7 @@ fn validate_main_identity( "\{label} officeDocument relationship targets '\{main_target}', but content types declare '\{declared_main_part}'", ) } - let main = read_xml_part(part_index, main_target) + let main = read_xml_part(part_index, main_target, context) let expected_main_relationship = match (format, main.name) { (Xlsx, TRANSITIONAL_WORKBOOK_ROOT) | (Docx, TRANSITIONAL_DOCUMENT_ROOT) => Some(TRANSITIONAL_OFFICE_DOCUMENT_REL) @@ -1620,13 +1779,15 @@ fn validate_main_identity( ///| fn default_main_target_from_root( part_index : ArchivePartIndex, + context : OfficeFormatContext, ) -> String? raise OfficeError { - let root_rels = read_relationships_part(part_index, "_rels/.rels") + let root_rels = read_relationships_part(part_index, "_rels/.rels", context) if root_rels.name != RELATIONSHIPS_ROOT { return None } let targets : Array[String] = [] for child in root_rels.children { + context.checkpoint() guard child is XmlElement(element) && element.name == RELATIONSHIP_ELEMENT else { continue } @@ -1658,9 +1819,11 @@ fn default_main_target_from_root( fn format_from_archive( archive : @zip.Archive, part_index : ArchivePartIndex, + context : OfficeFormatContext, ) -> (DocumentFormat, String) raise OfficeError { let physical_names : StableStringMap[Bool] = SortedMap([]) for entry in archive.entries() { + context.checkpoint() let name = entry.name() if physical_names.contains(name) { raise InvalidPackage("duplicate entry name: \{name}") @@ -1672,13 +1835,20 @@ fn format_from_archive( "central-directory file header exceeds the OPC \{MAX_OPC_CENTRAL_DIRECTORY_FILE_HEADER_BYTES}-byte limit: \{name}", ) } - if entry.crc32() != @zip.crc32(entry.data()) { + let actual_crc = @zip.crc32_cancellable( + entry.data(), + cancelled=context.cancelled, + ) catch { + _ if (context.cancelled)() => raise Cancelled + _ => raise InvalidPackage("entry CRC-32 verification failed: \{name}") + } + if entry.crc32() != actual_crc { raise InvalidPackage( "entry data does not match its stored CRC-32: \{name}", ) } } - let content_types = read_xml_part(part_index, "[Content_Types].xml") + let content_types = read_xml_part(part_index, "[Content_Types].xml", context) if content_types.name != CONTENT_TYPES_ROOT { raise InvalidPackage("[Content_Types].xml root element is not Types") } @@ -1687,6 +1857,7 @@ fn format_from_archive( let overridden_parts : StableStringMap[Bool] = SortedMap([]) let main_defaults : StableStringMap[DocumentFormat] = SortedMap([]) for child in content_types.children { + context.checkpoint() guard child is XmlElement(element) else { continue } if element.name == CONTENT_TYPES_DEFAULT { let format = match element.attributes.get("ContentType") { @@ -1733,7 +1904,7 @@ fn format_from_archive( } } if !main_defaults.is_empty() { - match default_main_target_from_root(part_index) { + match default_main_target_from_root(part_index, context) { Some(target) if !overridden_parts.contains(opc_part_key(target)) => match part_extension(target) { Some(extension) => @@ -1772,8 +1943,15 @@ fn structural_findings( archive : @zip.Archive, part_index : ArchivePartIndex, main_part : String, + context : OfficeFormatContext, ) -> Array[String] raise OfficeError { - validate_package_structure(archive, part_index, main_part, format is Xlsx) + validate_package_structure( + archive, + part_index, + main_part, + format is Xlsx, + context, + ) } ///| @@ -1785,13 +1963,47 @@ fn structural_findings( pub fn detect_format( path : StringView, data : BytesView, + max_package_bytes? : Int = DEFAULT_DETECT_MAX_PACKAGE_BYTES, + max_archive_entries? : Int = DEFAULT_DETECT_MAX_ARCHIVE_ENTRIES, + max_entry_uncompressed_bytes? : Int = DEFAULT_DETECT_MAX_ENTRY_BYTES, + max_total_uncompressed_bytes? : Int = DEFAULT_DETECT_MAX_TOTAL_BYTES, + max_xml_part_bytes? : Int = DEFAULT_DETECT_MAX_XML_PART_BYTES, + max_xml_total_units? : Int = DEFAULT_DETECT_MAX_XML_TOTAL_UNITS, + cancelled? : () -> Bool = () => false, ) -> DocumentFormat raise OfficeError { // Preserve the fail-fast extension contract before parsing package bytes. ignore(format_from_extension(path)) - let archive = @zip.read(data) catch { + if cancelled() { + raise Cancelled + } + let preserved_limit = if max_package_bytes > 2147483647 - 65_535 { + 2147483647 + } else if max_package_bytes < 0 { + 0 + } else { + max_package_bytes + 65_535 + } + let archive = @zip.read_limited( + data, + max_package_bytes~, + max_entries=max_archive_entries, + max_entry_uncompressed_bytes~, + max_total_uncompressed_bytes~, + max_total_preserved_source_bytes=preserved_limit, + cancelled~, + ) catch { + @zip.ResourceLimitExceeded(kind~, limit~, actual~) => + raise ResourceLimit(kind~, limit~, actual~) + _ if cancelled() => raise Cancelled _ => raise InvalidPackage("archive is not a readable ZIP") } - detect_archive_format(path, archive) + detect_archive_format( + path, + archive, + max_xml_part_bytes~, + max_xml_total_units~, + cancelled~, + ) } ///| @@ -1801,15 +2013,26 @@ pub fn detect_format( pub fn detect_archive_format( path : StringView, archive : @zip.Archive, + max_xml_part_bytes? : Int = DEFAULT_DETECT_MAX_XML_PART_BYTES, + max_xml_total_units? : Int = DEFAULT_DETECT_MAX_XML_TOTAL_UNITS, + cancelled? : () -> Bool = () => false, ) -> DocumentFormat raise OfficeError { let expected = format_from_extension(path) - let part_index = archive_part_index(archive) - let (actual, main_part) = format_from_archive(archive, part_index) + let context = OfficeFormatContext::new( + max_xml_part_bytes, + max_xml_total_units, + cancelled~, + ) + context.checkpoint() + let part_index = archive_part_index(archive, context) + let (actual, main_part) = format_from_archive(archive, part_index, context) if expected != actual { raise FormatMismatch(expected~, actual~) } - validate_main_identity(part_index, actual, main_part) - let findings = structural_findings(actual, archive, part_index, main_part) + validate_main_identity(part_index, actual, main_part, context) + let findings = structural_findings( + actual, archive, part_index, main_part, context, + ) if findings.length() > 0 { let first = findings[0] let suffix = if findings.length() == 1 { @@ -1819,5 +2042,6 @@ pub fn detect_archive_format( } raise InvalidPackage(first + suffix) } + context.checkpoint() actual } diff --git a/office/format_test.mbt b/office/format_test.mbt index cb70a93d..8d6a4bc3 100644 --- a/office/format_test.mbt +++ b/office/format_test.mbt @@ -1902,6 +1902,84 @@ test "detect_format rejects an invalid zip" { } } +///| +test "detect_archive_format enforces the XML part ceiling before DOM parsing" { + let archive = @zip.read(@excel.write(@excel.new_file())) + try + @office.detect_archive_format("book.xlsx", archive, max_xml_part_bytes=1) + catch { + ResourceLimit(kind~, limit~, actual~) => { + inspect(kind, content="xml_part_bytes") + assert_eq(limit, 1) + assert_true(actual > limit) + } + _ => fail("expected XML part resource limit") + } noraise { + _ => fail("expected XML part resource limit") + } +} + +///| +test "detect_archive_format shares one cumulative XML allocation budget" { + let archive = @zip.read(@excel.write(@excel.new_file())) + try + @office.detect_archive_format("book.xlsx", archive, max_xml_total_units=1) + catch { + ResourceLimit(kind~, limit~, actual~) => { + inspect(kind, content="xml_preflight_work_units") + assert_eq(limit, 4) + assert_true(actual > limit) + } + _ => fail("expected cumulative XML resource limit") + } noraise { + _ => fail("expected cumulative XML resource limit") + } +} + +///| +test "detect_archive_format cancels within CRC verification" { + let source = @zip.read(@excel.write(@excel.new_file())) + let archive = @zip.Archive::new() + archive.add("bulk.bin", Bytes::make(1024 * 1024, b'x')) + for entry in source.entries() { + archive.add( + entry.name(), + entry.data(), + compression=entry.compression(), + data_descriptor=entry.data_descriptor(), + ) + } + let checks = [0] + let cancel_at = archive.entries().length() + 4 + try + @office.detect_archive_format("book.xlsx", archive, cancelled=() => { + checks[0] += 1 + checks[0] >= cancel_at + }) + catch { + Cancelled => assert_true(checks[0] >= cancel_at) + _ => fail("expected format detection cancellation") + } noraise { + _ => fail("expected format detection cancellation") + } +} + +///| +test "detect_format requires exact reserved manifest spelling" { + let broken = rename_archive_part( + @excel.write(@excel.new_file()), + "[Content_Types].xml", + "[content_types].xml", + ) + try @office.detect_format("book.xlsx", broken) catch { + InvalidPackage(message) => + assert_true(message.contains("invalid physical spelling")) + _ => fail("expected reserved manifest spelling error") + } noraise { + _ => fail("expected reserved manifest spelling error") + } +} + ///| test "detect_format does not accept a content-type marker in an XML comment" { let archive = @zip.Archive::new() diff --git a/office/moon.pkg b/office/moon.pkg index 3dc0dbe3..ff7e78c5 100644 --- a/office/moon.pkg +++ b/office/moon.pkg @@ -1,6 +1,7 @@ import { "moonbitlang/core/debug", "bobzhang/mbtexcel/zip", + "bobzhang/docx2html/core" @docx_core, "bobzhang/docx2html/opc", "bobzhang/docx2html/paths" @docxpaths, "bobzhang/docx2html/xml", diff --git a/office/pkg.generated.mbti b/office/pkg.generated.mbti index 21bfa4b9..2eea13bb 100644 --- a/office/pkg.generated.mbti +++ b/office/pkg.generated.mbti @@ -57,9 +57,9 @@ pub fn capability_formats() -> Array[CapabilityFormat] pub fn capability_records(format? : DocumentFormat, operation? : String) -> Array[Json] -pub fn detect_archive_format(StringView, @zip.Archive) -> DocumentFormat raise OfficeError +pub fn detect_archive_format(StringView, @zip.Archive, max_xml_part_bytes? : Int, max_xml_total_units? : Int, cancelled? : () -> Bool) -> DocumentFormat raise OfficeError -pub fn detect_format(StringView, BytesView) -> DocumentFormat raise OfficeError +pub fn detect_format(StringView, BytesView, max_package_bytes? : Int, max_archive_entries? : Int, max_entry_uncompressed_bytes? : Int, max_total_uncompressed_bytes? : Int, max_xml_part_bytes? : Int, max_xml_total_units? : Int, cancelled? : () -> Bool) -> DocumentFormat raise OfficeError pub fn find_capability_command(StringView) -> CapabilityCommand? @@ -90,6 +90,8 @@ pub(all) suberror OfficeError { UnsupportedFileExtension(String) InvalidPackage(String) FormatMismatch(expected~ : DocumentFormat, actual~ : DocumentFormat) + ResourceLimit(kind~ : String, limit~ : Int, actual~ : Int) + Cancelled } derive(Eq, @debug.Debug) #deprecated pub fn OfficeError::equal(Self, Self) -> Bool From e016c2aaf371e95ab9411e12a87ca083ecacb69f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 10:22:48 +0800 Subject: [PATCH 045/150] fix: account for XLSX post-parse work --- ooxml/pkg.generated.mbti | 12 +-- ooxml/read_parse.mbt | 42 ++++++-- ooxml/read_parse_test.mbt | 52 ++++++++++ xlsx/cell_images.mbt | 9 +- xlsx/header_footer_image_read.mbt | 5 +- xlsx/ooxml_rels.mbt | 60 ++++++++--- xlsx/read.mbt | 163 +++++++++++++++++++++++++----- xlsx/read_drawing_xml.mbt | 25 ++++- xlsx/read_package_parts.mbt | 26 ++++- xlsx/read_shared_strings.mbt | 12 +++ xlsx/read_sheet_rel_parts.mbt | 7 +- xlsx/read_worksheet_xml.mbt | 44 ++++++++ xlsx/rich_value_images.mbt | 18 +++- 13 files changed, 411 insertions(+), 64 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 239ece9f..749b870c 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -16,9 +16,9 @@ pub fn escape_xml_attr(StringView) -> String pub fn escape_xml_text(StringView) -> String -pub fn find_override_part_by_content_types(StringView, ArrayView[String]) -> String? raise ParseXmlError +pub fn find_override_part_by_content_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> String? raise ParseXmlError -pub fn find_part_content_type(StringView, StringView) -> String? raise ParseXmlError +pub fn find_part_content_type(StringView, StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError pub fn is_valid_opc_part_segment(StringView) -> Bool @@ -26,13 +26,13 @@ pub fn logical_opc_part_name_from_zip_item_name(StringView) -> String? pub fn package_part_name_key(StringView) -> String -pub fn parse_external_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError +pub fn parse_external_relationship_targets(StringView, StringView, cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError -pub fn parse_external_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError +pub fn parse_external_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError -pub fn parse_internal_relationship_targets(StringView, StringView) -> Map[String, String] raise ParseXmlError +pub fn parse_internal_relationship_targets(StringView, StringView, cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError -pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String]) -> Map[String, String] raise ParseXmlError +pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 66ff695c..69b700c4 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -173,10 +173,11 @@ fn parse_relationship_targets_by_types_selected( xml : StringView, rel_types : ArrayView[String], selection : RelationshipTargetSelection, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise ParseXmlError { let targets : Map[String, String] = Map([]) let ids : Set[String] = Set([]) - let scanner = XmlStartTagScanner::new(xml) + let scanner = XmlStartTagScanner::new(xml, cancelled~) if !scanner.next() || scanner.depth() != 1 || scanner.namespace_uri() != package_relationships_namespace || @@ -248,8 +249,14 @@ fn parse_relationship_targets_by_types_selected( pub fn parse_internal_relationship_targets_by_types( xml : StringView, rel_types : ArrayView[String], + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise ParseXmlError { - parse_relationship_targets_by_types_selected(xml, rel_types, InternalTargets) + parse_relationship_targets_by_types_selected( + xml, + rel_types, + InternalTargets, + cancelled~, + ) } ///| @@ -257,8 +264,13 @@ pub fn parse_internal_relationship_targets_by_types( pub fn parse_internal_relationship_targets( xml : StringView, rel_type : StringView, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise ParseXmlError { - parse_internal_relationship_targets_by_types(xml, [rel_type.to_owned()]) + parse_internal_relationship_targets_by_types( + xml, + [rel_type.to_owned()], + cancelled~, + ) } ///| @@ -267,8 +279,14 @@ pub fn parse_internal_relationship_targets( pub fn parse_external_relationship_targets_by_types( xml : StringView, rel_types : ArrayView[String], + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise ParseXmlError { - parse_relationship_targets_by_types_selected(xml, rel_types, ExternalTargets) + parse_relationship_targets_by_types_selected( + xml, + rel_types, + ExternalTargets, + cancelled~, + ) } ///| @@ -277,8 +295,13 @@ pub fn parse_external_relationship_targets_by_types( pub fn parse_external_relationship_targets( xml : StringView, rel_type : StringView, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise ParseXmlError { - parse_external_relationship_targets_by_types(xml, [rel_type.to_owned()]) + parse_external_relationship_targets_by_types( + xml, + [rel_type.to_owned()], + cancelled~, + ) } ///| @@ -302,11 +325,12 @@ pub fn package_part_name_key(name : StringView) -> String { ///| fn parse_content_types( xml : StringView, + cancelled? : () -> Bool = () => false, ) -> PackageContentTypes raise ParseXmlError { let overrides : Map[String, String] = Map([]) let override_part_names : Map[String, String] = Map([]) let defaults : Map[String, String] = Map([]) - let scanner = XmlStartTagScanner::new(xml) + let scanner = XmlStartTagScanner::new(xml, cancelled~) if !scanner.next() || scanner.depth() != 1 || scanner.namespace_uri() != package_content_types_namespace || @@ -370,8 +394,9 @@ fn parse_content_types( pub fn find_override_part_by_content_types( xml : StringView, content_types : ArrayView[String], + cancelled? : () -> Bool = () => false, ) -> String? raise ParseXmlError { - let content_types_value = parse_content_types(xml) + let content_types_value = parse_content_types(xml, cancelled~) for wanted in content_types { for part_name_key, content_type in content_types_value.overrides { if content_type == wanted { @@ -389,8 +414,9 @@ pub fn find_override_part_by_content_types( pub fn find_part_content_type( xml : StringView, part_name : StringView, + cancelled? : () -> Bool = () => false, ) -> String? raise ParseXmlError { - let content_types_value = parse_content_types(xml) + let content_types_value = parse_content_types(xml, cancelled~) let normalized = if part_name.has_prefix("/") { part_name.to_owned() } else { diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index ea78f69a..2e78f140 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -170,6 +170,32 @@ test "ooxml read_parse: relationship limits accept writer boundaries" { assert_eq(targets.get("wanted"), Some(long_target)) } +///| +test "ooxml read_parse: relationship scans poll cancellation internally" { + let xml = StringBuilder::new() + xml.write_string( + "", + ) + for index in 0..<2048 { + xml.write_string( + "", + ) + } + xml.write_string("") + let checks = [0] + try + @ooxml.parse_internal_relationship_targets(xml.to_string(), "wanted", cancelled=() => { + checks[0] += 1 + checks[0] >= 4 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("expected relationship scan cancellation") + } noraise { + _ => fail("expected relationship scan cancellation") + } +} + ///| test "ooxml read_parse: parse_content_type_overrides and find workbook part" { let xml = @@ -241,6 +267,32 @@ test "ooxml read_parse: content type lookup honors part identity then default" { ) } +///| +test "ooxml read_parse: content type scans poll cancellation internally" { + let xml = StringBuilder::new() + xml.write_string( + "", + ) + for index in 0..<2048 { + xml.write_string( + "", + ) + } + xml.write_string("") + let checks = [0] + try + @ooxml.find_part_content_type(xml.to_string(), "custom/missing.xml", cancelled=() => { + checks[0] += 1 + checks[0] >= 4 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("expected content type scan cancellation") + } noraise { + _ => fail("expected content type scan cancellation") + } +} + ///| test "ooxml read_parse: content type overrides reject case-variant duplicates" { let xml = diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index 3d7bcbb1..d2ccb0fc 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -21,13 +21,20 @@ fn parse_cell_images( cell_images_xml : StringView, rels_xml : StringView, archive : @zip.Archive, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Array[CellImage] raise XlsxError { let images : Array[CellImage] = [] let targets = if rels_xml.to_owned() == "" { Map([]) } else { // cellimages relationships are always image relationships - parse_internal_relationship_targets(rels_xml, rel_image) + parse_internal_relationship_targets( + rels_xml, + rel_image, + budget?, + cancelled~, + ) } let xml_str = cell_images_xml.to_owned() // Each embedded image is one <...pic> element; cellimages.xml contains diff --git a/xlsx/header_footer_image_read.mbt b/xlsx/header_footer_image_read.mbt index 5555c296..d50d625d 100644 --- a/xlsx/header_footer_image_read.mbt +++ b/xlsx/header_footer_image_read.mbt @@ -83,10 +83,13 @@ fn parse_header_footer_images_from_vml( vml_part : StringView, part_names : Map[String, String], archive : @zip.Archive, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Array[HeaderFooterImage] raise XlsxError { let shapes = extract_vml_shapes(vml_xml) let rel_targets = match vml_rels_xml { - Some(xml) => parse_internal_relationship_targets(xml, rel_image) + Some(xml) => + parse_internal_relationship_targets(xml, rel_image, budget?, cancelled~) None => Map([]) } let images : Array[HeaderFooterImage] = [] diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 8c12b265..57b5943f 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -8,8 +8,9 @@ let strict_office_relationship_prefix = "http://purl.oclc.org/ooxml/officeDocume fn parse_internal_relationship_targets_exact( xml : StringView, rel_type : StringView, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { - @ooxml.parse_internal_relationship_targets(xml, rel_type) catch { + @ooxml.parse_internal_relationship_targets(xml, rel_type, cancelled~) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } @@ -19,8 +20,13 @@ fn parse_internal_relationship_targets_exact( fn parse_internal_relationship_targets_exact_types( xml : StringView, rel_types : ArrayView[String], + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { - @ooxml.parse_internal_relationship_targets_by_types(xml, rel_types) catch { + @ooxml.parse_internal_relationship_targets_by_types( + xml, + rel_types, + cancelled~, + ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } @@ -30,8 +36,9 @@ fn parse_internal_relationship_targets_exact_types( fn parse_external_relationship_targets_exact( xml : StringView, rel_type : StringView, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { - @ooxml.parse_external_relationship_targets(xml, rel_type) catch { + @ooxml.parse_external_relationship_targets(xml, rel_type, cancelled~) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } @@ -41,8 +48,13 @@ fn parse_external_relationship_targets_exact( fn parse_external_relationship_targets_exact_types( xml : StringView, rel_types : ArrayView[String], + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { - @ooxml.parse_external_relationship_targets_by_types(xml, rel_types) catch { + @ooxml.parse_external_relationship_targets_by_types( + xml, + rel_types, + cancelled~, + ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } @@ -55,16 +67,28 @@ fn parse_external_relationship_targets_exact_types( fn parse_internal_relationship_targets( xml : StringView, rel_type : StringView, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => check_read_cancelled(cancelled) + } let requested = rel_type.to_owned() match requested.strip_prefix(transitional_office_relationship_prefix) { Some(suffix) => { let strict_type = strict_office_relationship_prefix + suffix.to_owned() - parse_internal_relationship_targets_exact_types(xml, [ - requested, strict_type, - ]) + parse_internal_relationship_targets_exact_types( + xml, + [requested, strict_type], + cancelled~, + ) } - None => parse_internal_relationship_targets_exact(xml, requested) + None => + parse_internal_relationship_targets_exact(xml, requested, cancelled~) } } @@ -74,16 +98,28 @@ fn parse_internal_relationship_targets( fn parse_external_relationship_targets( xml : StringView, rel_type : StringView, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Map[String, String] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => check_read_cancelled(cancelled) + } let requested = rel_type.to_owned() match requested.strip_prefix(transitional_office_relationship_prefix) { Some(suffix) => { let strict_type = strict_office_relationship_prefix + suffix.to_owned() - parse_external_relationship_targets_exact_types(xml, [ - requested, strict_type, - ]) + parse_external_relationship_targets_exact_types( + xml, + [requested, strict_type], + cancelled~, + ) } - None => parse_external_relationship_targets_exact(xml, requested) + None => + parse_external_relationship_targets_exact(xml, requested, cancelled~) } } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index ee8ba540..06e604e6 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3251,9 +3251,14 @@ fn workbook_related_part_path( workbook_part : StringView, fallback : StringView, part_names : Map[String, String], + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> String? raise XlsxError { let targets = parse_internal_relationship_targets( - workbook_rels, relationship_type, + workbook_rels, + relationship_type, + budget?, + cancelled~, ) match first_existing_workbook_rel_target_path(targets, workbook_part, part_names) { @@ -3311,7 +3316,13 @@ fn read_zip_archive_core_unchecked( } let workbook_xml_part_path = actual_archive_part_path( part_names, - resolve_workbook_xml_part_path(archive, part_names, decode), + resolve_workbook_xml_part_path( + archive, + part_names, + decode, + budget=read_budget, + cancelled~, + ), ) let workbook_bytes = match archive.get(workbook_xml_part_path) { Some(value) => value @@ -3372,8 +3383,13 @@ fn read_zip_archive_core_unchecked( active_index } let shared_strings_path = workbook_related_part_path( - workbook_rels, rel_shared_strings, workbook_xml_part_path, shared_strings_part_path, + workbook_rels, + rel_shared_strings, + workbook_xml_part_path, + shared_strings_part_path, part_names, + budget=read_budget, + cancelled~, ) let shared_strings = match shared_strings_path { Some(path) => @@ -3396,14 +3412,20 @@ fn read_zip_archive_core_unchecked( ) read_budget.checkpoint() read_budget.charge_work(canonical.length()) - parse_shared_strings(canonical) + parse_shared_strings(canonical, budget=read_budget) } None => raise MissingPart(path~) } None => [] } let styles_path = workbook_related_part_path( - workbook_rels, rel_styles, workbook_xml_part_path, styles_part_path, part_names, + workbook_rels, + rel_styles, + workbook_xml_part_path, + styles_part_path, + part_names, + budget=read_budget, + cancelled~, ) let ( styles, @@ -3488,11 +3510,22 @@ fn read_zip_archive_core_unchecked( Some(rels) => decode(rels) None => "" } - parse_cell_images(decode(value), rels_xml, archive) + parse_cell_images( + decode(value), + rels_xml, + archive, + budget=read_budget, + cancelled~, + ) } None => [] } - let rich_value_images = parse_rich_value_images(archive, decode) + let rich_value_images = parse_rich_value_images( + archive, + decode, + budget=read_budget, + cancelled~, + ) let rich_value_media : Map[String, Bytes] = Map([]) match rich_value_images { Some(data) => { @@ -3546,10 +3579,19 @@ fn read_zip_archive_core_unchecked( rich_value_media, } let slicer_cache_defs = parse_slicer_cache_definitions( - workbook_rels, workbook_xml_part_path, part_names, archive, decode, + workbook_rels, + workbook_xml_part_path, + part_names, + archive, + decode, + budget=read_budget, + cancelled~, ) let vba_targets = parse_internal_relationship_targets( - workbook_rels, rel_vba_project, + workbook_rels, + rel_vba_project, + budget=read_budget, + cancelled~, ) let vba_path = match first_existing_workbook_rel_target_path( @@ -3582,10 +3624,16 @@ fn read_zip_archive_core_unchecked( None => () } let worksheet_targets = parse_internal_relationship_targets( - workbook_rels, rel_worksheet, + workbook_rels, + rel_worksheet, + budget=read_budget, + cancelled~, ) let chartsheet_targets = parse_internal_relationship_targets( - workbook_rels, rel_chartsheet, + workbook_rels, + rel_chartsheet, + budget=read_budget, + cancelled~, ) let seen_sheet_parts : Set[String] = Set([]) let resolved_sheet_parts : Map[String, String] = Map([]) @@ -3740,7 +3788,10 @@ fn read_zip_archive_core_unchecked( raise InvalidXml(msg="slicer relationship missing") } let rel_targets = parse_internal_relationship_targets( - rels_xml, rel_slicer, + rels_xml, + rel_slicer, + budget=read_budget, + cancelled~, ) let parsed : Array[ParsedSlicerPartEntry] = [] let seen_paths : Map[String, Bool] = Map([]) @@ -3771,7 +3822,10 @@ fn read_zip_archive_core_unchecked( raise InvalidXml(msg="picture relationship missing") } let image_targets = parse_internal_relationship_targets( - rels_xml, rel_image, + rels_xml, + rel_image, + budget=read_budget, + cancelled~, ) let target = match image_targets.get(rel_id) { Some(value) => value @@ -3808,7 +3862,12 @@ fn read_zip_archive_core_unchecked( [] } else { let rel_targets = if needs_hyperlink_rels { - parse_external_relationship_targets(rels_xml, rel_hyperlink) + parse_external_relationship_targets( + rels_xml, + rel_hyperlink, + budget=read_budget, + cancelled~, + ) } else { Map([]) } @@ -3850,7 +3909,10 @@ fn read_zip_archive_core_unchecked( [] } else { let rel_targets = parse_internal_relationship_targets( - rels_xml, rel_table, + rels_xml, + rel_table, + budget=read_budget, + cancelled~, ) let parsed_tables : Array[Table] = [] for rel_id in table_part_ids { @@ -3874,7 +3936,12 @@ fn read_zip_archive_core_unchecked( let rel_targets = if rels_xml == "" { Map([]) } else { - parse_internal_relationship_targets(rels_xml, rel_pivot_table) + parse_internal_relationship_targets( + rels_xml, + rel_pivot_table, + budget=read_budget, + cancelled~, + ) } let pivot_rel_ids : Array[String] = [] if pivot_part_ids.length() > 0 { @@ -3912,7 +3979,10 @@ fn read_zip_archive_core_unchecked( } let pivot_rels_xml = decode(pivot_rels_bytes) let cache_targets = parse_internal_relationship_targets( - pivot_rels_xml, rel_pivot_cache, + pivot_rels_xml, + rel_pivot_cache, + budget=read_budget, + cancelled~, ) let cache_path = match first_existing_rel_target_path( @@ -3947,7 +4017,10 @@ fn read_zip_archive_core_unchecked( Some(value) => { let cache_rels_xml = decode(value) let record_targets = parse_internal_relationship_targets( - cache_rels_xml, rel_pivot_cache_records, + cache_rels_xml, + rel_pivot_cache_records, + budget=read_budget, + cancelled~, ) let record_path = match first_existing_rel_target_path( @@ -3992,7 +4065,12 @@ fn read_zip_archive_core_unchecked( } let vml_targets = if legacy_drawing_rel_id is Some(_) || legacy_drawing_hf_rel_id is Some(_) { - parse_internal_relationship_targets(rels_xml, rel_vml) + parse_internal_relationship_targets( + rels_xml, + rel_vml, + budget=read_budget, + cancelled~, + ) } else { Map([]) } @@ -4043,7 +4121,13 @@ fn read_zip_archive_core_unchecked( None => None } parse_header_footer_images_from_vml( - xml, vml_rels_xml, path, part_names, archive, + xml, + vml_rels_xml, + path, + part_names, + archive, + budget=read_budget, + cancelled~, ) } _ => [] @@ -4052,7 +4136,10 @@ fn read_zip_archive_core_unchecked( [] } else { let comment_targets = parse_internal_relationship_targets( - rels_xml, rel_comments, + rels_xml, + rel_comments, + budget=read_budget, + cancelled~, ) let parsed_comments : Array[Comment] = [] for _rel_id, target in comment_targets { @@ -4126,7 +4213,10 @@ fn read_zip_archive_core_unchecked( raise InvalidXml(msg="drawing relationship missing") } let drawing_targets = parse_internal_relationship_targets( - rels_xml, rel_drawing, + rels_xml, + rel_drawing, + budget=read_budget, + cancelled~, ) let drawing_target = match drawing_targets.get(rel_id) { Some(value) => value @@ -4152,13 +4242,26 @@ fn read_zip_archive_core_unchecked( None => "" } let drawing_images = parse_drawing_images( - drawing_xml, drawing_rels_xml, drawing_path, part_names, drawing_metrics, + drawing_xml, + drawing_rels_xml, + drawing_path, + part_names, + drawing_metrics, archive, + budget=read_budget, + cancelled~, ) sheet.images.append(drawing_images) let drawing_charts = parse_drawing_charts( - drawing_xml, drawing_rels_xml, drawing_path, part_names, drawing_metrics, - archive, decode, + drawing_xml, + drawing_rels_xml, + drawing_path, + part_names, + drawing_metrics, + archive, + decode, + budget=read_budget, + cancelled~, ) sheet.charts.append(drawing_charts) let drawing_shapes = parse_drawing_shapes(drawing_xml) @@ -4259,7 +4362,10 @@ fn read_zip_archive_core_unchecked( } let chartsheet_rels_xml = decode(chartsheet_rels_bytes) let drawing_targets = parse_internal_relationship_targets( - chartsheet_rels_xml, rel_drawing, + chartsheet_rels_xml, + rel_drawing, + budget=read_budget, + cancelled~, ) let drawing_target = match drawing_targets.get(drawing_rel) { Some(value) => value @@ -4277,7 +4383,10 @@ fn read_zip_archive_core_unchecked( } let drawing_rels_xml = decode(drawing_rels_bytes) let chart_targets = parse_internal_relationship_targets( - drawing_rels_xml, rel_chart, + drawing_rels_xml, + rel_chart, + budget=read_budget, + cancelled~, ) let chart_path = match first_existing_rel_target_path(chart_targets, drawing_path, part_names) { diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 40973aaf..c1af42bb 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -166,17 +166,29 @@ fn parse_drawing_images( part_names : Map[String, String], metrics : DrawingMetrics, archive : @zip.Archive, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Array[Image] raise XlsxError { let images : Array[Image] = [] let image_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_internal_relationship_targets(drawing_rels_xml, rel_image) + parse_internal_relationship_targets( + drawing_rels_xml, + rel_image, + budget?, + cancelled~, + ) } let hyperlink_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_external_relationship_targets(drawing_rels_xml, rel_hyperlink) + parse_external_relationship_targets( + drawing_rels_xml, + rel_hyperlink, + budget?, + cancelled~, + ) } let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { @@ -372,12 +384,19 @@ fn parse_drawing_charts( metrics : DrawingMetrics, archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Array[Chart] raise XlsxError { let charts : Array[Chart] = [] let chart_targets = if drawing_rels_xml.to_owned() == "" { Map([]) } else { - parse_internal_relationship_targets(drawing_rels_xml, rel_chart) + parse_internal_relationship_targets( + drawing_rels_xml, + rel_chart, + budget?, + cancelled~, + ) } let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index eb567b22..126bdc0f 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -11,6 +11,8 @@ fn workbook_part_from_root_relationships( archive : @zip.Archive, part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let root_rels_path = actual_archive_part_path(part_names, root_rels_part_path) let root_rels_xml = match archive.get(root_rels_path) { @@ -20,7 +22,10 @@ fn workbook_part_from_root_relationships( let office_document_type = transitional_office_relationship_prefix + "officeDocument" let targets = parse_internal_relationship_targets( - root_rels_xml, office_document_type, + root_rels_xml, + office_document_type, + budget?, + cancelled~, ) if targets.length() > 1 { raise InvalidXml(msg="root relationships contain multiple workbook targets") @@ -64,9 +69,15 @@ fn resolve_workbook_xml_part_path( archive : @zip.Archive, part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let workbook_part = workbook_part_from_root_relationships( - archive, part_names, decode, + archive, + part_names, + decode, + budget?, + cancelled~, ) let content_types_path = actual_archive_part_path( part_names, content_types_part_path, @@ -75,8 +86,17 @@ fn resolve_workbook_xml_part_path( Some(content_types_bytes) => decode(content_types_bytes) None => raise MissingPart(path=content_types_path) } + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(content_types_xml.length()) + } + None => check_read_cancelled(cancelled) + } let content_type = @ooxml.find_part_content_type( - content_types_xml, workbook_part, + content_types_xml, + workbook_part, + cancelled~, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled diff --git a/xlsx/read_shared_strings.mbt b/xlsx/read_shared_strings.mbt index 11d55ea2..99c475b4 100644 --- a/xlsx/read_shared_strings.mbt +++ b/xlsx/read_shared_strings.mbt @@ -366,10 +366,22 @@ fn parse_rich_text_runs(xml : StringView) -> Array[RichTextRun] raise XlsxError ///| fn parse_shared_strings( xml : StringView, + budget? : ReadBudget, ) -> Array[SharedStringEntry] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => () + } let values : Array[SharedStringEntry] = [] let mut cursor = 0 while find_xml_open_tag_start_from(xml, "si", cursor) is Some(start) { + match budget { + Some(value) => value.checkpoint() + None => () + } let open_end = match xml_open_tag_end_from(xml, start + 1) { Some(value) => value None => raise InvalidXml(msg="shared string item tag not closed") diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index bc66c775..c36b5b9a 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -344,10 +344,15 @@ fn parse_slicer_cache_definitions( part_names : Map[String, String], archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Map[String, SlicerCacheDef] raise XlsxError { let defs : Map[String, SlicerCacheDef] = Map([]) let targets = parse_internal_relationship_targets( - workbook_rels_xml, rel_slicer_cache, + workbook_rels_xml, + rel_slicer_cache, + budget?, + cancelled~, ) for _rel_id, target in targets { let cache_path = actual_archive_part_path( diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index 6797441f..dd6373bb 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -6,6 +6,23 @@ fn worksheet_sheet_data_body(xml : StringView) -> String raise XlsxError { } } +///| +/// Accounts for one lexical worksheet pass before it begins. The worksheet +/// reader intentionally composes several focused parsers; charging every pass +/// keeps that convenience inside the same cumulative work policy. +fn worksheet_parse_pass( + source : StringView, + budget : ReadBudget?, +) -> Unit raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(source.length()) + } + None => () + } +} + ///| fn parse_worksheet( xml : StringView, @@ -30,9 +47,15 @@ fn parse_worksheet( String?, ) raise XlsxError { let cells : Array[Cell] = [] + worksheet_parse_pass(xml, budget) let sheet_data = worksheet_sheet_data_body(xml) + worksheet_parse_pass(sheet_data, budget) let mut first = true for chunk in sheet_data.split(" value.checkpoint() + None => () + } if first { first = false continue @@ -260,20 +283,41 @@ fn parse_worksheet( } // Shared formulas are a worksheet-wide construct: followers carry only an // index, so validate the complete group before exposing a partial model. + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(cells.length()) + } + None => () + } ignore(validated_shared_formula_masters(cells)) + worksheet_parse_pass(xml, budget) let merged_cells = parse_merge_cells(xml) + worksheet_parse_pass(xml, budget) let auto_filter = parse_auto_filter(xml) + worksheet_parse_pass(sheet_data, budget) let row_dimensions = parse_row_dimensions(sheet_data, style_count) + worksheet_parse_pass(xml, budget) let col_dimensions = parse_col_dimensions(xml, style_count) + worksheet_parse_pass(xml, budget) let page_margins = parse_page_margins(xml) + worksheet_parse_pass(xml, budget) let page_layout = parse_page_layout(xml) + worksheet_parse_pass(xml, budget) let header_footer = parse_header_footer(xml) + worksheet_parse_pass(xml, budget) let sheet_protection = parse_sheet_protection(xml) + worksheet_parse_pass(xml, budget) let row_breaks = parse_page_breaks(xml, "rowBreaks") + worksheet_parse_pass(xml, budget) let col_breaks = parse_page_breaks(xml, "colBreaks") + worksheet_parse_pass(xml, budget) let sheet_views = parse_sheet_views(xml, budget?) + worksheet_parse_pass(xml, budget) let sheet_props = parse_sheet_props(xml) + worksheet_parse_pass(xml, budget) let dimension_ref = parse_sheet_dimension(xml) + worksheet_parse_pass(xml, budget) let picture_rel_id = parse_picture_rel_id(xml) ( cells, merged_cells, auto_filter, row_dimensions, col_dimensions, page_margins, diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index c2b14215..62d38b55 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -31,6 +31,8 @@ struct RichValueImages { fn parse_rich_value_images( archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> RichValueImages? raise XlsxError { let metadata_bytes = archive.get("xl/metadata.xml") let rich_value_bytes = archive.get("xl/richData/rdrichvalue.xml") @@ -49,7 +51,13 @@ fn parse_rich_value_images( } let rel_targets = match archive.get("xl/richData/_rels/richValueRel.xml.rels") { - Some(value) => parse_internal_relationship_targets(decode(value), rel_image) + Some(value) => + parse_internal_relationship_targets( + decode(value), + rel_image, + budget?, + cancelled~, + ) None => Map([]) } let web_blip_ids = match archive.get("xl/richData/rdRichValueWebImage.xml") { @@ -58,7 +66,13 @@ fn parse_rich_value_images( } let web_rel_targets = match archive.get("xl/richData/_rels/rdRichValueWebImage.xml.rels") { - Some(value) => parse_internal_relationship_targets(decode(value), rel_image) + Some(value) => + parse_internal_relationship_targets( + decode(value), + rel_image, + budget?, + cancelled~, + ) None => Map([]) } Some({ From bbc67c82b30ccb301ede58bc7ff6fb536dc4bfc7 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 10:37:14 +0800 Subject: [PATCH 046/150] fix: yield during bounded XLSX scans --- office/cmd/office/read_package.mbt | 4 +- office/cmd/office/xlsx_read_output.mbt | 68 +++++++++++++++------ office/cmd/office/xlsx_read_wbtest.mbt | 84 +++++++++++++++++++++----- xlsx/io.mbt | 38 ++---------- 4 files changed, 124 insertions(+), 70 deletions(-) diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index a8bad7cf..f58cbece 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -36,8 +36,8 @@ struct OfficeReadPackage { } ///| -/// Centralizes cancellation observation for asynchronous Office commands. -/// CPU-heavy command paths add explicit async yield points around bounded work. +/// Reports cancellation that was delivered at an async suspension boundary. +/// The bounded synchronous package phases do not themselves run the scheduler. fn office_async_cancelled() -> Bool { @async.is_being_cancelled() } diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index e1886f30..6a89147f 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -5,6 +5,15 @@ enum XlsxCellFilter { QueryCells(Array[PreparedXlsxCellPredicate], OfficeQueryWorkBudget) } +///| +/// Cell scans are command-owned CPU work, so they yield independently of I/O. +/// The work threshold prevents a small number of large strings or predicates +/// from delaying a scheduler turn until the cell-count threshold is reached. +let xlsx_cli_scan_yield_cells : Int = 256 + +///| +let xlsx_cli_scan_yield_work_units : Int = 64 * 1024 + ///| struct XlsxCellPage { cells : Array[XlsxCellSnapshot] @@ -331,6 +340,14 @@ fn XlsxCellFilter::matches( } } +///| +fn XlsxCellFilter::work_used(self : XlsxCellFilter) -> Int { + match self { + QueryCells(_, work) => work.used + PresentCells | TextCells => 0 + } +} + ///| fn xlsx_query_filter( predicates : Array[XlsxCellPredicate], @@ -343,15 +360,16 @@ fn xlsx_query_filter( } ///| -fn collect_xlsx_cell_page( +async fn collect_xlsx_cell_page( projection : XlsxProjection, regions : Array[XlsxScanRegion], filter : XlsxCellFilter, offset : Int, limit : Int, format_work_maximum? : Int = xlsx_cli_max_format_work_units, -) -> XlsxCellPage raise CliFailure { +) -> XlsxCellPage { projection.checkpoint() + @async.pause() let scan = XlsxScanBudget::new( projection.max_scan_cells, format_work_maximum~, @@ -359,6 +377,8 @@ fn collect_xlsx_cell_page( ) let cells : Array[XlsxCellSnapshot] = [] let mut matched = 0 + let mut yielded_cells = 0 + let mut yielded_work = filter.work_used() for region in regions { projection.checkpoint() for row in region.rect.row_lo..<=region.rect.row_hi { @@ -380,10 +400,18 @@ fn collect_xlsx_cell_page( } matched += 1 } + let work = scan.strings.used + scan.formatting.used + filter.work_used() + if scan.scanned - yielded_cells >= xlsx_cli_scan_yield_cells || + work - yielded_work >= xlsx_cli_scan_yield_work_units { + @async.pause() + yielded_cells = scan.scanned + yielded_work = work + } } } } projection.checkpoint() + @async.pause() { cells, matched_total: matched, scanned_cells: scan.scanned, offset, limit } } @@ -402,10 +430,10 @@ fn xlsx_page_fields(page : XlsxCellPage) -> Map[String, Json] { } ///| -fn xlsx_get_data( +async fn xlsx_get_data( projection : XlsxProjection, resolved : XlsxResolvedSelector, -) -> Json raise CliFailure { +) -> Json { projection.checkpoint() match resolved { Workbook(path~) => { @@ -502,11 +530,11 @@ fn xlsx_get_data( } ///| -fn xlsx_get_payload( +async fn xlsx_get_payload( projection : XlsxProjection, resolved : XlsxResolvedSelector, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { projection.checkpoint() let payload = bounded_office_payload( xlsx_get_data(projection, resolved), @@ -519,12 +547,12 @@ fn xlsx_get_payload( } ///| -fn xlsx_text_data( +async fn xlsx_text_data( projection : XlsxProjection, under : XlsxResolvedSelector?, offset : Int, limit : Int, -) -> Json raise CliFailure { +) -> Json { projection.checkpoint() let page = collect_xlsx_cell_page( projection, @@ -559,13 +587,13 @@ fn xlsx_text_data( } ///| -fn xlsx_text_payload( +async fn xlsx_text_payload( projection : XlsxProjection, under : XlsxResolvedSelector?, offset : Int, limit : Int, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { projection.checkpoint() let payload = bounded_office_payload( xlsx_text_data(projection, under, offset, limit), @@ -578,11 +606,11 @@ fn xlsx_text_payload( } ///| -fn xlsx_query_data( +async fn xlsx_query_data( projection : XlsxProjection, under : XlsxResolvedSelector?, spec : XlsxQuerySpec, -) -> Json raise CliFailure { +) -> Json { projection.checkpoint() let page = collect_xlsx_cell_page( projection, @@ -606,12 +634,12 @@ fn xlsx_query_data( } ///| -fn xlsx_query_payload( +async fn xlsx_query_payload( projection : XlsxProjection, under : XlsxResolvedSelector?, spec : XlsxQuerySpec, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { projection.checkpoint() let payload = bounded_office_payload( xlsx_query_data(projection, under, spec), @@ -693,11 +721,11 @@ fn xlsx_page_human( } ///| -fn xlsx_get_human( +async fn xlsx_get_human( projection : XlsxProjection, resolved : XlsxResolvedSelector, maximum : Int, -) -> String raise CliFailure { +) -> String { projection.checkpoint() match resolved { Workbook(..) => xlsx_outline_human(projection, maximum) @@ -745,13 +773,13 @@ fn xlsx_get_human( } ///| -fn xlsx_text_human( +async fn xlsx_text_human( projection : XlsxProjection, under : XlsxResolvedSelector?, offset : Int, limit : Int, maximum : Int, -) -> String raise CliFailure { +) -> String { xlsx_page_human( projection, collect_xlsx_cell_page( @@ -767,12 +795,12 @@ fn xlsx_text_human( } ///| -fn xlsx_query_human( +async fn xlsx_query_human( projection : XlsxProjection, under : XlsxResolvedSelector?, spec : XlsxQuerySpec, maximum : Int, -) -> String raise CliFailure { +) -> String { xlsx_page_human( projection, collect_xlsx_cell_page( diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 9d83c1f3..42cdbfbc 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -19,6 +19,16 @@ fn xlsx_cli_error(run : () -> Unit raise) -> @lib.ProtocolError raise { } } +///| +async fn xlsx_cli_error_async(run : async () -> Unit) -> @lib.ProtocolError { + try run() catch { + CliFailure(error) => error + _ => fail("expected XLSX CLI failure") + } noraise { + _ => fail("expected XLSX CLI failure") + } +} + ///| fn xlsx_bounded_test_source( file : String, @@ -52,6 +62,46 @@ test "XLSX CLI preserves parser cancellation and checkpoints cell scans" { assert_eq(scan_error.code, "office.cancelled") } +///| +async test "XLSX command cell scans yield before their bounded maximum" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Data")) + let mut checks = 0 + let projection = { + ..make_xlsx_projection("cooperative.xlsx", workbook, 100_000), + cancelled: () => { + checks += 1 + false + }, + } + guard projection.sheets[0] is sheet && sheet.content is Worksheet(_) else { + fail("expected worksheet fixture") + } + let region : XlsxScanRegion = { + sheet, + rect: { col_lo: 1, row_lo: 1, col_hi: 1_000, row_hi: 100 }, + } + @async.with_task_group() <| group => { + let task = group.spawn(() => { + collect_xlsx_cell_page(projection, [region], PresentCells, 0, 0) + }) + // The child first yields before scanning, then again after its first + // quantum. Waiting for many checkpoints proves it made partial progress. + while checks <= 100 { + @async.pause() + } + assert_true(task.try_wait() is None) + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + ///| test "XLSX post-projection metadata, styles, and human output observe cancellation" { let projection = { ..xlsx_read_test_projection(), cancelled: () => true } @@ -101,7 +151,9 @@ test "Office format detection probes cancellation before and after validation" { ) }) assert_eq(error.code, "office.cancelled") - assert_eq(checks, 2) + // The detector observes cancellation internally, and the CLI boundary + // rechecks before translating it to a stable protocol error. + assert_eq(checks, 3) } ///| @@ -269,7 +321,7 @@ test "XLSX selectors resolve to stable named canonical paths" { } ///| -test "XLSX structured reads accept detector-valid Strict OOXML relationships" { +async test "XLSX structured reads accept detector-valid Strict OOXML relationships" { let source = xlsx_bounded_test_source("strict.xlsx", xlsx_strict_test_bytes()) assert_true(@lib.detect_archive_format(source.file, source.archive) is Xlsx) let projection = open_xlsx_projection(source, 10) @@ -285,7 +337,7 @@ test "XLSX structured reads accept detector-valid Strict OOXML relationships" { } ///| -test "XLSX structured reads resolve relationships from relocated workbooks" { +async test "XLSX structured reads resolve relationships from relocated workbooks" { let source = xlsx_bounded_test_source( "relocated.xlsx", xlsx_relocated_test_bytes(), @@ -320,7 +372,7 @@ test "XLSX structured reads resolve relationships from relocated workbooks" { } ///| -test "XLSX structured reads retain a default-declared relocated workbook path" { +async test "XLSX structured reads retain a default-declared relocated workbook path" { let source = xlsx_bounded_test_source( "relocated-default.xlsx", xlsx_default_declared_relocated_test_bytes(), @@ -572,7 +624,7 @@ test "XLSX query literals preserve whitespace and support JSON quoting" { } ///| -test "XLSX snapshots preserve and resolve shared-formula followers" { +async test "XLSX snapshots preserve and resolve shared-formula followers" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Data")) workbook.set_cell_formula_opts( @@ -644,7 +696,7 @@ test "XLSX structured reads reject shared-formula followers without a master" { } ///| -test "XLSX formatting is deferred and custom-format work is charged" { +async test "XLSX formatting is deferred and custom-format work is charged" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Data")) let style_id = workbook.new_style(@xlsx.Style::number_format("[Red]0.00")) @@ -677,7 +729,7 @@ test "XLSX formatting is deferred and custom-format work is charged" { assert_eq(page.matched_total, 2) assert_eq(page.cells.length(), 0) assert_false(projection.sheets[0].shared_formulas_indexed) - let limited = xlsx_cli_error(() => { + let limited = xlsx_cli_error_async(() => { ignore( collect_xlsx_cell_page( projection, @@ -700,7 +752,7 @@ test "XLSX formatting is deferred and custom-format work is charged" { } ///| -test "XLSX outline and get use office schemas with canonical cell records" { +async test "XLSX outline and get use office schemas with canonical cell records" { let projection = xlsx_read_test_projection() let outline = xlsx_outline_data(projection) guard outline is Object(outline_fields) else { @@ -730,14 +782,14 @@ test "XLSX outline and get use office schemas with canonical cell records" { let missing = resolve_xlsx_selector( projection, "/xlsx/sheet[name=\"Data\"]/cell[D4]", ) - let missing_error = xlsx_cli_error(() => { + let missing_error = xlsx_cli_error_async(() => { ignore(xlsx_get_data(projection, missing)) }) assert_eq(missing_error.code, "office.xlsx.selector_not_found") } ///| -test "XLSX singleton used ranges remain canonical two-endpoint ranges" { +async test "XLSX singleton used ranges remain canonical two-endpoint ranges" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Only")) workbook.set_cell_str("Only", "A1", "one") @@ -882,7 +934,7 @@ test "XLSX style parsing rejects non-finite JSON-facing numbers" { } ///| -test "XLSX displayed text preserves formula results formatted to blank" { +async test "XLSX displayed text preserves formula results formatted to blank" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Data")) let style_id = workbook.new_style(@xlsx.Style::number_format(";;;")) @@ -927,7 +979,7 @@ test "XLSX displayed text preserves formula results formatted to blank" { } ///| -test "XLSX cached empty formula results remain distinct from a missing cache" { +async test "XLSX cached empty formula results remain distinct from a missing cache" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Data")) workbook.set_cell_formula("Data", "A1", "1+0") @@ -1010,7 +1062,7 @@ test "XLSX cached empty formula results remain distinct from a missing cache" { } ///| -test "XLSX text and query scan fully while retaining only the requested page" { +async test "XLSX text and query scan fully while retaining only the requested page" { let projection = xlsx_read_test_projection() let under = Some( resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), @@ -1043,7 +1095,7 @@ test "XLSX text and query scan fully while retaining only the requested page" { } ///| -test "XLSX JSON and human output enforce format-specific stdout ceilings" { +async test "XLSX JSON and human output enforce format-specific stdout ceilings" { let projection = xlsx_read_test_projection() let payload = xlsx_outline_payload(projection, 100_000) let json = checked_office_json_output(payload) @@ -1062,12 +1114,12 @@ test "XLSX JSON and human output enforce format-specific stdout ceilings" { } ///| -test "XLSX all-sheet scans reject sparse far extents before visiting cells" { +async test "XLSX all-sheet scans reject sparse far extents before visiting cells" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Sparse")) workbook.set_cell_str("Sparse", "K10", "far") let projection = make_xlsx_projection("sparse.xlsx", workbook, 100) - let error = xlsx_cli_error(() => { + let error = xlsx_cli_error_async(() => { ignore(xlsx_text_data(projection, None, 0, 10)) }) assert_eq(error.code, "office.xlsx.resource_limit") diff --git a/xlsx/io.mbt b/xlsx/io.mbt index 8aaa3f60..c39dc7e1 100644 --- a/xlsx/io.mbt +++ b/xlsx/io.mbt @@ -242,7 +242,6 @@ fn read_workbook_from_bytes( options : Options, limits : ReadLimits, io_context : WorkbookIOContext, - cancelled : () -> Bool, ) -> Workbook raise XlsxError { let resolved_password = if password == "" { options.password @@ -250,18 +249,16 @@ fn read_workbook_from_bytes( password } let resolved_options = options_with_password(options, resolved_password) - let workbook = if resolved_password == "" { + if resolved_password == "" { match io_context.charset_transcoder { Some(value) => read_zip_bytes( bytes, options=resolved_options, limits~, - cancelled~, transcoder=value, ) - None => - read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) + None => read_zip_bytes(bytes, options=resolved_options, limits~) } } else { match io_context.charset_transcoder { @@ -271,7 +268,6 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, - cancelled~, transcoder=value, ) None => @@ -280,11 +276,9 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, - cancelled~, ) } } - checked_read_workbook(workbook, cancelled) } ///| @@ -304,15 +298,9 @@ pub async fn[R : @async/io.Reader] open_reader( ) let io_context = read_io_context(transcoder) let workbook = read_workbook_from_bytes( - bytes, - password, - options, - limits, - io_context, - () => @async.is_being_cancelled(), + bytes, password, options, limits, io_context, ) workbook.set_io_context(io_context) - @async.pause() workbook } @@ -325,13 +313,11 @@ pub async fn[R : @async/io.Reader] read_zip_reader( limits? : ReadLimits = ReadLimits::new(), transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook { - let workbook = match transcoder { + match transcoder { Some(value) => open_reader(reader, password~, options~, limits~, transcoder=value) None => open_reader(reader, password~, options~, limits~) } - @async.pause() - workbook } ///| @@ -361,15 +347,9 @@ pub async fn[R : @async/io.Reader] Workbook::read_zip_reader( resource="package_bytes", ) let workbook = read_workbook_from_bytes( - bytes, - password, - resolved_options, - limits, - io_context, - () => @async.is_being_cancelled(), + bytes, password, resolved_options, limits, io_context, ) workbook.set_io_context(io_context) - @async.pause() workbook } @@ -395,15 +375,9 @@ pub async fn open_file( Some(path), ) let workbook = read_workbook_from_bytes( - bytes, - password, - options, - limits, - io_context, - () => @async.is_being_cancelled(), + bytes, password, options, limits, io_context, ) workbook.set_io_context(io_context) - @async.pause() workbook } From ca5f5adb2786db28b4bfa893393fc105d93706a5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 10:51:48 +0800 Subject: [PATCH 047/150] test: refresh bounded Office read error --- office/cmd/office/cram/cli.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index fdbff05a..15bba7c0 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -193,7 +193,7 @@ Extension/content mismatches and malformed input fail non-zero. [1] $ printf 'not zip' > broken.docx; office.exe identify broken.docx - office: invalid Office package: archive is not a readable ZIP + office: invalid Office package: archive is not a readable bounded ZIP [1] JSON business and operational failures are one parseable envelope and retain a From f727a663719ad36d0438ae7071b6cce9571bf7bd Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 11:30:16 +0800 Subject: [PATCH 048/150] fix(ooxml): reject malformed processing markup --- ooxml/start_tag_scanner.mbt | 259 ++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index f38cb8e7..9d410f87 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -221,6 +221,236 @@ fn find_xml_sequence( None } +///| +fn xml_span_equals_ascii_case_insensitive( + xml : StringView, + start : Int, + end : Int, + expected : StringView, +) -> Bool { + if end - start != expected.length() { + return false + } + for offset in 0..= ('A' : UInt16).to_int() && + actual <= ('Z' : UInt16).to_int() { + actual + 32 + } else { + actual + } + let wanted_lower = if wanted >= ('A' : UInt16).to_int() && + wanted <= ('Z' : UInt16).to_int() { + wanted + 32 + } else { + wanted + } + if actual_lower != wanted_lower { + return false + } + } + true +} + +///| +fn validate_xml_character_span( + xml : StringView, + start : Int, + end : Int, + cancelled : () -> Bool, +) -> Unit raise ParseXmlError { + let mut index = start + let mut next_checkpoint = start + while index < end { + if index >= next_checkpoint { + check_xml_cancelled(cancelled) + next_checkpoint = index + 4096 + } + let unit = xml[index] + if unit < 0x20 && + unit != ('\t' : UInt16) && + unit != ('\n' : UInt16) && + unit != ('\r' : UInt16) { + raise InvalidXml(msg="XML markup contains an invalid character") + } + if unit == 0xfffe || unit == 0xffff || unit.is_trailing_surrogate() { + raise InvalidXml(msg="XML markup contains an invalid character") + } + if unit.is_leading_surrogate() { + if index + 1 >= end || !xml[index + 1].is_trailing_surrogate() { + raise InvalidXml(msg="XML markup contains an invalid character") + } + index = index + 2 + } else { + index = index + 1 + } + } + check_xml_cancelled(cancelled) +} + +///| +fn validate_xml_encoding_name(value : StringView) -> Bool { + if value.length() == 0 || + !((value[0] >= ('A' : UInt16) && value[0] <= ('Z' : UInt16)) || + (value[0] >= ('a' : UInt16) && value[0] <= ('z' : UInt16))) { + return false + } + for unit in value[1:] { + let valid = (unit >= 'A' && unit <= 'Z') || + (unit >= 'a' && unit <= 'z') || + (unit >= '0' && unit <= '9') || + unit == '.' || + unit == '_' || + unit == '-' + if !valid { + return false + } + } + true +} + +///| +fn validate_xml_declaration( + xml : StringView, + attributes_start : Int, + end : Int, + cancelled : () -> Bool, +) -> Unit raise ParseXmlError { + if attributes_start >= end || !is_attr_space(xml[attributes_start]) { + raise InvalidXml(msg="XML declaration version is missing") + } + let mut next = attributes_start + let mut position = 0 + let mut saw_encoding = false + let mut saw_standalone = false + while next_xml_attribute(xml, next, end, cancelled~) is Some(attribute) { + validate_xml_character_span( + xml, + attribute.value_start, + attribute.value_end, + cancelled, + ) + if position == 0 { + if !xml_span_equals( + xml, + attribute.name_start, + attribute.name_end, + "version", + ) || + ( + !xml_span_equals( + xml, + attribute.value_start, + attribute.value_end, + "1.0", + ) && + !xml_span_equals( + xml, + attribute.value_start, + attribute.value_end, + "1.1", + ) + ) { + raise InvalidXml(msg="XML declaration version is invalid") + } + } else if xml_span_equals( + xml, + attribute.name_start, + attribute.name_end, + "encoding", + ) { + if saw_encoding || + saw_standalone || + !validate_xml_encoding_name( + xml[attribute.value_start:attribute.value_end], + ) { + raise InvalidXml(msg="XML declaration encoding is invalid") + } + saw_encoding = true + } else if xml_span_equals( + xml, + attribute.name_start, + attribute.name_end, + "standalone", + ) { + if saw_standalone || + ( + !xml_span_equals( + xml, + attribute.value_start, + attribute.value_end, + "yes", + ) && + !xml_span_equals( + xml, + attribute.value_start, + attribute.value_end, + "no", + ) + ) { + raise InvalidXml(msg="XML declaration standalone value is invalid") + } + saw_standalone = true + } else { + raise InvalidXml(msg="XML declaration attribute is invalid") + } + position = position + 1 + next = attribute.next + } + if position == 0 { + raise InvalidXml(msg="XML declaration version is missing") + } +} + +///| +fn XmlStartTagScanner::validate_processing_instruction( + self : XmlStartTagScanner, + start : Int, + end : Int, +) -> Unit raise ParseXmlError { + let target_start = start + 2 + let mut target_end = target_start + while target_end < end && !is_attr_space(self.xml[target_end]) { + target_end = target_end + 1 + } + ignore(validate_xml_qname(self.xml, target_start, target_end)) + let reserved = xml_span_equals_ascii_case_insensitive( + self.xml, + target_start, + target_end, + "xml", + ) + if reserved { + if start != 0 || + !xml_span_equals(self.xml, target_start, target_end, "xml") || + self.depth != 0 || + self.root_seen || + self.root_closed { + raise InvalidXml(msg="XML declaration is misplaced") + } + validate_xml_declaration(self.xml, target_end, end, self.cancelled) + } else { + validate_xml_character_span(self.xml, target_end, end, self.cancelled) + } +} + +///| +fn XmlStartTagScanner::validate_comment( + self : XmlStartTagScanner, + start : Int, + end : Int, +) -> Unit raise ParseXmlError { + let content_start = start + 4 + validate_xml_character_span(self.xml, content_start, end, self.cancelled) + for index in content_start.. value None => raise InvalidXml(msg="XML processing instruction is not closed") } + self.validate_processing_instruction(start, end) if self.canonicalize_markup { self.record_rewrite(start, end + 2, "") } @@ -1121,6 +1352,7 @@ pub fn XmlStartTagScanner::next( Some(value) => value None => raise InvalidXml(msg="XML comment is not closed") } + self.validate_comment(start, end) if self.canonicalize_markup { self.record_rewrite(start, end + 3, "") } @@ -1673,6 +1905,33 @@ test "start tag scanner validates unused attributes and document structure" { } } +///| +test "start tag scanner validates declarations, PI targets, and comments" { + let valid = XmlStartTagScanner::new( + "", + ) + assert_true(valid.next()) + assert_false(valid.next()) + + for + xml in [ + "", "", "", + "", "", "", + ] { + let scanner = XmlStartTagScanner::new(xml) + try { + while scanner.next() { + + } + } catch { + InvalidXml(_) => () + _ => fail("unexpected markup validation cancellation") + } noraise { + _ => fail("expected invalid XML markup") + } + } +} + ///| test "start tag scanner rejects a misplaced self-closing slash" { let scanner = XmlStartTagScanner::new("x") From 061f73b173793a151fe96edf930531e412483d6e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 11:30:30 +0800 Subject: [PATCH 049/150] fix(xlsx): close read correctness gaps --- office/cmd/office/xlsx_read_wbtest.mbt | 42 +++++++++++++++ xlsx/cell_value_test.mbt | 31 +++++++++++ xlsx/iterators.mbt | 74 ++++++++++++++++++++++---- xlsx/read.mbt | 25 ++++++++- xlsx/read_budget.mbt | 53 +++++++++++++++--- xlsx/value_format.mbt | 67 +++++++++++++++-------- xlsx/workbook.mbt | 20 ++++++- 7 files changed, 271 insertions(+), 41 deletions(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 42cdbfbc..fa06de24 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -978,6 +978,48 @@ async test "XLSX displayed text preserves formula results formatted to blank" { assert_eq(cell.get("formula"), Some(Json::string("1+0"))) } +///| +async test "XLSX get text and query honor the 1904 date system" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Dates")) + workbook.set_workbook_props( + Some(@xlsx.WorkbookPropsOptions::with_values(date_1904=true)), + ) + workbook.set_cell_value("Dates", "A1", Numeric(0.0)) + let style_id = workbook.new_style(@xlsx.Style::number_format("yyyy-mm-dd")) + workbook.set_cell_style("Dates", "A1", style_id) + let projection = make_xlsx_projection("dates-1904.xlsx", workbook, 10) + let cell_selector = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Dates\"]/cell[A1]", + ) + guard xlsx_get_data(projection, cell_selector) is Object(get) && + get.get("cell") is Some(Object(cell)) else { + fail("expected 1904 get result") + } + assert_eq(cell.get("value"), Some(Json::string("1904-01-01"))) + + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Dates\"]"), + ) + guard xlsx_text_data(projection, under, 0, 10) is Object(text) && + text.get("entries") is Some(Array([Object(text_entry), ..])) else { + fail("expected 1904 text result") + } + assert_eq(text_entry.get("text"), Some(Json::string("1904-01-01"))) + + let spec : XlsxQuerySpec = { + selector: "cell[type=number]", + predicates: parse_xlsx_query_selector("cell[type=number]"), + offset: 0, + limit: 10, + } + guard xlsx_query_data(projection, under, spec) is Object(query) && + query.get("matches") is Some(Array([Object(query_cell), ..])) else { + fail("expected 1904 query result") + } + assert_eq(query_cell.get("value"), Some(Json::string("1904-01-01"))) +} + ///| async test "XLSX cached empty formula results remain distinct from a missing cache" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/cell_value_test.mbt b/xlsx/cell_value_test.mbt index 0b530b7a..8839fe6f 100644 --- a/xlsx/cell_value_test.mbt +++ b/xlsx/cell_value_test.mbt @@ -82,6 +82,37 @@ test "formatted percent and date" { ) } +///| +test "formatted dates honor the workbook 1904 date system" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + workbook.set_workbook_props( + Some(@xlsx.WorkbookPropsOptions::with_values(date_1904=true)), + ) + workbook.set_cell_value("Sheet1", "A1", Numeric(0.0)) + let date_style = workbook.add_style(@xlsx.Style::number_format("yyyy-mm-dd")) + workbook.set_cell_style("Sheet1", "A1", date_style) + debug_inspect( + workbook.get_cell_value("Sheet1", "A1"), + content="Some(\"1904-01-01\")", + ) + guard workbook.sheet("Sheet1") is Some(sheet) else { + fail("expected date-system worksheet") + } + debug_inspect( + workbook.get_cell_value_styled_from_worksheet_rc(sheet, 1, 1, date_style), + content="Some(\"1904-01-01\")", + ) + let rows = workbook.rows("Sheet1") + assert_true(rows.next()) + debug_inspect(rows.columns(), content="[\"1904-01-01\"]") + let columns = workbook.cols("Sheet1") + assert_true(columns.next()) + debug_inspect(columns.rows(), content="[\"1904-01-01\"]") + debug_inspect(workbook.get_rows("Sheet1"), content="[[\"1904-01-01\"]]") + debug_inspect(workbook.get_cols("Sheet1"), content="[[\"1904-01-01\"]]") +} + ///| test "typed setters and cell type" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/iterators.mbt b/xlsx/iterators.mbt index 100e7e94..6948ecd5 100644 --- a/xlsx/iterators.mbt +++ b/xlsx/iterators.mbt @@ -36,6 +36,7 @@ pub struct Rows { mut index : Int styles : Array[Style]? options : Options + use_1904_format : Bool } ///| @@ -87,8 +88,16 @@ fn Rows::new_with_options( sheet : Worksheet, styles : Array[Style]?, options : Options, + use_1904_format? : Bool = false, ) -> Rows { - { sheet, entries: collect_row_cells(sheet), index: 0, styles, options } + { + sheet, + entries: collect_row_cells(sheet), + index: 0, + styles, + options, + use_1904_format, + } } ///| @@ -119,7 +128,11 @@ pub fn Rows::columns( Some(styles) => for cell in entry.cells { let formatted = format_cell_value_for_workbook( - styles, cell, false, resolved_options, + styles, + cell, + false, + resolved_options, + use_1904_format=self.use_1904_format, ) values[cell.col - 1] = formatted } @@ -182,6 +195,7 @@ pub struct Cols { mut index : Int styles : Array[Style]? options : Options + use_1904_format : Bool mut err : XlsxError? } @@ -215,6 +229,7 @@ fn collect_row_entries_with_options( sheet : Worksheet, styles : ArrayView[Style], options : Options, + use_1904_format : Bool, ) -> Array[RowValues] raise XlsxError { let cells = sheet.sorted_cells() let row_cells : Map[Int, Array[Cell]] = Map([]) @@ -251,7 +266,11 @@ fn collect_row_entries_with_options( let values : Array[String] = Array::make(max_col, "") for cell in row_entries { let formatted = format_cell_value_for_workbook( - styles, cell, false, options, + styles, + cell, + false, + options, + use_1904_format~, ) values[cell.col - 1] = formatted } @@ -265,6 +284,7 @@ fn collect_columns_with_options( sheet : Worksheet, styles : ArrayView[Style], options : Options, + use_1904_format : Bool, ) -> Array[Array[String]] raise XlsxError { let max_col = sheet.max_col() if max_col <= 0 { @@ -273,7 +293,13 @@ fn collect_columns_with_options( let max_row = max_row_for_columns(sheet) let col_cells : Map[Int, Array[(Int, String)]] = Map([]) for cell in sheet.cells { - let formatted = format_cell_value_for_workbook(styles, cell, false, options) + let formatted = format_cell_value_for_workbook( + styles, + cell, + false, + options, + use_1904_format~, + ) let entries = col_cells.get(cell.col).unwrap_or([]) entries.push((cell.row, formatted)) col_cells[cell.col] = entries @@ -302,6 +328,7 @@ fn Cols::new_with_options( sheet : Worksheet, styles : Array[Style]?, options : Options, + use_1904_format? : Bool = false, ) -> Cols { let max_col = sheet.max_col() let columns = if max_col <= 0 { @@ -310,7 +337,16 @@ fn Cols::new_with_options( collect_column_cells(sheet, max_col) } let max_row = if max_col <= 0 { 0 } else { max_row_for_columns(sheet) } - { sheet, columns, max_row, index: 0, styles, options, err: None } + { + sheet, + columns, + max_row, + index: 0, + styles, + options, + use_1904_format, + err: None, + } } ///| @@ -341,7 +377,11 @@ pub fn Cols::rows(self : Cols, options? : Options) -> Array[String] { continue } let formatted = format_cell_value_for_workbook( - styles, cell, false, resolved_options, + styles, + cell, + false, + resolved_options, + use_1904_format=self.use_1904_format, ) catch { err => { if self.err is None { @@ -443,7 +483,12 @@ pub fn Workbook::rows( Some(value) => value None => raise SheetNotFound(name=sheet_name.to_owned()) } - Rows::new_with_options(sheet, Some(self.styles), self.options) + Rows::new_with_options( + sheet, + Some(self.styles), + self.options, + use_1904_format=self.uses_1904_date_system(), + ) } ///| @@ -456,7 +501,12 @@ pub fn Workbook::cols( Some(value) => value None => raise SheetNotFound(name=sheet_name.to_owned()) } - Cols::new_with_options(sheet, Some(self.styles), self.options) + Cols::new_with_options( + sheet, + Some(self.styles), + self.options, + use_1904_format=self.uses_1904_date_system(), + ) } ///| @@ -475,6 +525,7 @@ pub fn Workbook::get_rows( sheet, self.styles, resolved_options, + self.uses_1904_date_system(), ) let results : Array[Array[String]] = [] let mut max_with_values = 0 @@ -509,5 +560,10 @@ pub fn Workbook::get_cols( Some(value) => value None => raise SheetNotFound(name=sheet_name.to_owned()) } - collect_columns_with_options(sheet, self.styles, resolved_options) + collect_columns_with_options( + sheet, + self.styles, + resolved_options, + self.uses_1904_date_system(), + ) } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 06e604e6..97028220 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3084,7 +3084,11 @@ fn first_existing_rel_target_path( ) -> String? raise XlsxError { for _, target in targets { let path = actual_relationship_target_path(source_part, target, part_names) - if part_names.contains(@ooxml.package_part_name_key(path)) { + // `path` is the physical ZIP spelling when the target exists. The archive + // index is keyed in logical OPC PartName space, so restore percent-encoded + // Unicode before testing identity. + let logical_path = logical_archive_part_path(path) + if part_names.contains(@ooxml.package_part_name_key(logical_path)) { return Some(path) } } @@ -3156,6 +3160,25 @@ test "read archive index maps physical Unicode OPC names to logical identity" { ) } +///| +test "relationship selection recognizes physical Unicode chart parts" { + let archive = @zip.Archive::new() + archive.add(content_types_part_path, b"manifest") + archive.add("xl/charts/%E6%96%87.xml", b"unicode chart") + archive.add("xl/charts/chart2.xml", b"other chart") + let part_names = archive_part_name_index(archive) + let targets : Map[String, String] = Map([ + ("unicode", "../charts/文.xml"), + ("other", "../charts/chart2.xml"), + ]) + debug_inspect( + first_existing_rel_target_path( + targets, "xl/drawings/drawing1.xml", part_names, + ), + content="Some(\"xl/charts/%E6%96%87.xml\")", + ) +} + ///| test "read archive index rejects ambiguous and misspelled reserved entries" { let duplicate = @zip.Archive::new() diff --git a/xlsx/read_budget.mbt b/xlsx/read_budget.mbt index 97c453a0..bc0b7925 100644 --- a/xlsx/read_budget.mbt +++ b/xlsx/read_budget.mbt @@ -19,11 +19,15 @@ fn ReadBudget::new( } ///| -fn read_budget_actual(current : Int, amount : Int, limit : Int) -> Int { +fn read_budget_actual(current : Int, amount : Int, limit : Int) -> (Int, Bool) { if amount > limit - current { - limit + 1 + // `ReadLimits` accepts every positive Int. Keep the reported value + // representable when the ceiling is Int::max while carrying overflow as a + // separate bit; wrapping `limit + 1` would otherwise turn the rejection + // into a negative, apparently in-budget value. + (if limit < 0x7fffffff { limit + 1 } else { limit }, true) } else { - current + amount + (current + amount, false) } } @@ -35,12 +39,12 @@ fn ReadBudget::charge_items( if amount <= 0 { return } - let actual = read_budget_actual( + let (actual, exceeded) = read_budget_actual( self.parser_items, amount, self.limits.max_parser_items, ) - if actual > self.limits.max_parser_items { + if exceeded { raise ResourceLimitExceeded( kind="parser_items", limit=self.limits.max_parser_items, @@ -58,12 +62,12 @@ fn ReadBudget::charge_work( if amount <= 0 { return } - let actual = read_budget_actual( + let (actual, exceeded) = read_budget_actual( self.parser_work_units, amount, self.limits.max_parser_work_units, ) - if actual > self.limits.max_parser_work_units { + if exceeded { raise ResourceLimitExceeded( kind="parser_work_units", limit=self.limits.max_parser_work_units, @@ -310,3 +314,38 @@ test "read budget ignores element-like text in comments and CDATA" { ) assert_eq(budget.parser_items, 2) } + +///| +test "read budget rejects overflow at maximum Int limits" { + let limits = ReadLimits::with_values( + max_parser_items=0x7fffffff, + max_parser_work_units=0x7fffffff, + ) + let item_budget = ReadBudget::new(limits) + item_budget.parser_items = 0x7ffffffe + try item_budget.charge_items(2) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "parser_items") + assert_eq(limit, 0x7fffffff) + assert_eq(actual, 0x7fffffff) + assert_eq(item_budget.parser_items, 0x7ffffffe) + } + _ => fail("unexpected parser item overflow error") + } noraise { + _ => fail("expected parser item overflow rejection") + } + + let work_budget = ReadBudget::new(limits) + work_budget.parser_work_units = 0x7ffffffe + try work_budget.charge_work(2) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "parser_work_units") + assert_eq(limit, 0x7fffffff) + assert_eq(actual, 0x7fffffff) + assert_eq(work_budget.parser_work_units, 0x7ffffffe) + } + _ => fail("unexpected parser work overflow error") + } noraise { + _ => fail("expected parser work overflow rejection") + } +} diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index fbf7f4ba..e5093f45 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -4,6 +4,7 @@ fn format_cell_value( raw : String, style : Style?, options : Options, + use_1904_format? : Bool = false, ) -> String { match value_type { String => @@ -21,7 +22,8 @@ fn format_cell_value( match style { Some(style) => match style.number_format { - Some(format) => format_number_with_format(raw, format, options) + Some(format) => + format_number_with_format(raw, format, options, use_1904_format) None => raw } None => raw @@ -47,15 +49,21 @@ fn format_number_with_format( raw : String, format : NumberFormat, options : Options, + use_1904_format : Bool, ) -> String { match format { - Builtin(id) => format_number_builtin(raw, id, options) - Custom(code) => format_number_code(raw, code, options) + Builtin(id) => format_number_builtin(raw, id, options, use_1904_format) + Custom(code) => format_number_code(raw, code, options, use_1904_format~) } } ///| -fn format_number_builtin(raw : String, id : Int, options : Options) -> String { +fn format_number_builtin( + raw : String, + id : Int, + options : Options, + use_1904_format : Bool, +) -> String { let short_date = options.short_date_pattern let long_date = options.long_date_pattern let long_time = options.long_time_pattern @@ -69,37 +77,37 @@ fn format_number_builtin(raw : String, id : Int, options : Options) -> String { 10 => format_number_simple(raw, 2, false, true) 14 => if short_date != "" { - format_excel_date(raw, short_date) + format_excel_date(raw, short_date, use_1904_format~) } else { - format_excel_date(raw, "mm-dd-yy") + format_excel_date(raw, "mm-dd-yy", use_1904_format~) } 15 => if long_date != "" { - format_excel_date(raw, long_date) + format_excel_date(raw, long_date, use_1904_format~) } else { - format_excel_date(raw, "d-mmm-yy") + format_excel_date(raw, "d-mmm-yy", use_1904_format~) } - 16 => format_excel_date(raw, "d-mmm") - 17 => format_excel_date(raw, "mmm-yy") - 18 => format_excel_date(raw, "h:mm AM/PM") - 19 => format_excel_date(raw, "h:mm:ss AM/PM") + 16 => format_excel_date(raw, "d-mmm", use_1904_format~) + 17 => format_excel_date(raw, "mmm-yy", use_1904_format~) + 18 => format_excel_date(raw, "h:mm AM/PM", use_1904_format~) + 19 => format_excel_date(raw, "h:mm:ss AM/PM", use_1904_format~) 20 => if long_time != "" { - format_excel_date(raw, long_time) + format_excel_date(raw, long_time, use_1904_format~) } else { - format_excel_date(raw, "hh:mm") + format_excel_date(raw, "hh:mm", use_1904_format~) } 21 => if long_time != "" { - format_excel_date(raw, long_time) + format_excel_date(raw, long_time, use_1904_format~) } else { - format_excel_date(raw, "hh:mm:ss") + format_excel_date(raw, "hh:mm:ss", use_1904_format~) } 22 => if short_date != "" { - format_excel_date(raw, "\{short_date} hh:mm") + format_excel_date(raw, "\{short_date} hh:mm", use_1904_format~) } else { - format_excel_date(raw, "m/d/yy hh:mm") + format_excel_date(raw, "m/d/yy hh:mm", use_1904_format~) } // language-glyph builtin IDs resolve through the workbook culture 27..=36 | 50..=62 | 67..=81 => @@ -108,7 +116,7 @@ fn format_number_builtin(raw : String, id : Int, options : Options) -> String { if code == "" { raw } else { - format_number_code(raw, code, options) + format_number_code(raw, code, options, use_1904_format~) } None => raw } @@ -117,7 +125,12 @@ fn format_number_builtin(raw : String, id : Int, options : Options) -> String { } ///| -fn format_number_code(raw : String, code : String, options : Options) -> String { +fn format_number_code( + raw : String, + code : String, + options : Options, + use_1904_format? : Bool = false, +) -> String { let _ = options let sections = split_format_sections(code) if sections.length() == 0 { @@ -143,7 +156,7 @@ fn format_number_code(raw : String, code : String, options : Options) -> String Some(tag) => if !is_supported_locale(tag) { return raw } None => () } - let formatted = format_excel_date(raw, pattern) + let formatted = format_excel_date(raw, pattern, use_1904_format~) let output = currency_prefix + formatted return if add_minus { "-" + output } else { output } } @@ -1030,9 +1043,17 @@ fn format_int_grouped_string(text : String) -> String { } ///| -fn format_excel_date(raw : String, pattern : String) -> String { +fn format_excel_date( + raw : String, + pattern : String, + use_1904_format? : Bool = false, +) -> String { let value = @string.parse_double(raw) catch { _ => return raw } - let parts = excel_serial_to_parts(value) + let parts = if use_1904_format { + excel_serial_to_parts_1904(value) + } else { + excel_serial_to_parts(value) + } match parts { None => raw Some((year, month, day, hour, minute, second)) => diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 85c65148..1a8e8881 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1478,6 +1478,7 @@ pub fn Workbook::get_cell_value_styled( cell.value, Some(self.styles[style_id]), resolved_options, + use_1904_format=self.uses_1904_date_system(), ), ) } @@ -1491,6 +1492,7 @@ fn format_cell_value_for_workbook( cell : Cell, raw : Bool, options : Options, + use_1904_format? : Bool = false, ) -> String raise XlsxError { if raw || options.raw_cell_value { return cell.value @@ -1503,9 +1505,15 @@ fn format_cell_value_for_workbook( cell.value, Some(styles[cell.style_id]), options, + use_1904_format~, ) } +///| +fn Workbook::uses_1904_date_system(self : Workbook) -> Bool { + self.workbook_props.date_1904.unwrap_or(false) +} + ///| fn normalize_cell_float_value( value : Double, @@ -1810,6 +1818,7 @@ pub fn Workbook::get_cell_value( sheet.cells[index], raw, resolved_options, + use_1904_format=self.uses_1904_date_system(), ), ) None => None @@ -1847,6 +1856,7 @@ pub fn Workbook::get_cell_value_from_worksheet_rc( worksheet.cells[index], raw, resolved_options, + use_1904_format=self.uses_1904_date_system(), ), ) None => None @@ -1887,6 +1897,7 @@ pub fn Workbook::get_cell_value_styled_from_worksheet_rc( cell.value, Some(self.styles[style_id]), resolved_options, + use_1904_format=self.uses_1904_date_system(), ), ) } @@ -1934,6 +1945,7 @@ pub fn Workbook::calc_cell_value( raw_value, Some(self.styles[style_id]), resolved_options, + use_1904_format=self.uses_1904_date_system(), ) } @@ -3447,7 +3459,13 @@ pub fn Workbook::get_cell_value_rc( for cell in sheet.cells() { if cell.row == row && cell.col == col { return Some( - format_cell_value_for_workbook(self.styles, cell, raw, resolved_options), + format_cell_value_for_workbook( + self.styles, + cell, + raw, + resolved_options, + use_1904_format=self.uses_1904_date_system(), + ), ) } } From e5b9ee91d45be120fb427fecbfb8adaea67a3c02 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 11:43:36 +0800 Subject: [PATCH 050/150] perf(xlsx): build bounded cell indexes during read --- xlsx/read.mbt | 5 ++- xlsx/workbook.mbt | 7 ++-- xlsx/worksheet_cell_index.mbt | 71 ++++++++++++++++++++++++++++++----- xlsx/worksheet_types.mbt | 4 +- 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 97028220..eb94c84d 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4183,13 +4183,14 @@ fn read_zip_archive_core_unchecked( Some(xml) => parse_form_controls_vml(xml) None => [] } + let cell_index = build_bounded_worksheet_cell_index(cells, read_budget) let sheet = { name, sheet_views, dimension_ref, cells, - cell_index: Map([]), - cell_index_valid: false, + cell_index, + cell_index_valid: true, merged_cells, auto_filter, page_margins, diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 1a8e8881..5aa6c23a 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1306,13 +1306,14 @@ fn clone_worksheet( name : String, state : SheetState, ) -> Worksheet { + let cells = clone_cells(sheet.cells) { name, sheet_views: clone_sheet_views(sheet.sheet_views), dimension_ref: sheet.dimension_ref, - cells: clone_cells(sheet.cells), - cell_index: Map([]), - cell_index_valid: false, + cells, + cell_index: build_worksheet_cell_index(cells), + cell_index_valid: true, merged_cells: clone_strings(sheet.merged_cells), hyperlinks: clone_hyperlinks(sheet.hyperlinks), tables: clone_tables(sheet.tables), diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index d96cafd0..6b74fe61 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -9,20 +9,58 @@ fn Worksheet::invalidate_cell_index(self : Worksheet) -> Unit { } ///| -fn Worksheet::ensure_cell_index(self : Worksheet) -> Unit { - if self.cell_index_valid { - return - } - self.cell_index.clear() - for i, cell in self.cells { +fn populate_worksheet_cell_index( + index : Map[String, Int], + cells : Array[Cell], +) -> Unit { + for i, cell in cells { let key = cell_index_key(cell.row, cell.col) // First occurrence wins: a noncanonical file can carry duplicate // coordinates, and the historical (pre-index) getters returned the - // first stored cell — keep that semantic for every indexed getter. - if !self.cell_index.contains(key) { - self.cell_index[key] = i + // first stored cell. + if !index.contains(key) { + index[key] = i } } +} + +///| +fn build_worksheet_cell_index(cells : Array[Cell]) -> Map[String, Int] { + let index : Map[String, Int] = Map([]) + populate_worksheet_cell_index(index, cells) + index +} + +///| +fn build_bounded_worksheet_cell_index( + cells : Array[Cell], + budget : ReadBudget, +) -> Map[String, Int] raise XlsxError { + // Index construction is part of workbook parsing, not deferred command + // work. Charge it before allocating and retain cancellation checkpoints for + // large but valid sheets. + budget.charge_work(cells.length()) + let index : Map[String, Int] = Map([]) + for i, cell in cells { + if i % 4096 == 0 { + budget.checkpoint() + } + let key = cell_index_key(cell.row, cell.col) + if !index.contains(key) { + index[key] = i + } + } + budget.checkpoint() + index +} + +///| +fn Worksheet::ensure_cell_index(self : Worksheet) -> Unit { + if self.cell_index_valid { + return + } + self.cell_index.clear() + populate_worksheet_cell_index(self.cell_index, self.cells) self.cell_index_valid = true } @@ -102,3 +140,18 @@ test "resolved worksheet reads avoid repeated names and preserve style precedenc assert_eq(sheet.effective_style_id_rc(3, 1), column_style) assert_eq(sheet.effective_style_id_rc(3, 2), 0) } + +///| +test "parsed worksheets leave coordinate indexes ready for bounded reads" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("Parsed") + sheet.set_cell_value_rc(1, 1, String("first")) + sheet.set_cell_value_rc(2, 2, Numeric(2.0)) + let parsed = read(write(workbook)) + guard parsed.sheet("Parsed") is Some(parsed_sheet) else { + fail("expected parsed sheet") + } + assert_true(parsed_sheet.cell_index_valid) + assert_eq(parsed_sheet.cell_index.length(), 2) + assert_eq(parsed.get_cell_value("Parsed", "B2"), Some("2")) +} diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index fa36f93d..1d03a5bd 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -145,7 +145,9 @@ pub fn Worksheet::new(name : String) -> Worksheet { dimension_ref: None, cells: [], cell_index: Map([]), - cell_index_valid: false, + // Empty worksheets begin with a valid index. Normal mutators update it as + // cells are added, avoiding a deferred whole-sheet rebuild on first read. + cell_index_valid: true, merged_cells: [], hyperlinks: [], tables: [], From 244821ad8c93d51cd7ffea79f8f7e5228a3a355f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 11:43:43 +0800 Subject: [PATCH 051/150] fix(office): bound cooperative XLSX output --- office/cmd/office/xlsx_query.mbt | 89 +++++- office/cmd/office/xlsx_read_output.mbt | 417 +++++++++++++++++++------ office/cmd/office/xlsx_read_wbtest.mbt | 131 ++++++-- 3 files changed, 507 insertions(+), 130 deletions(-) diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt index e11e2c74..bb13fbd8 100644 --- a/office/cmd/office/xlsx_query.mbt +++ b/office/cmd/office/xlsx_query.mbt @@ -7,6 +7,9 @@ let xlsx_cli_max_query_predicates : Int = 16 ///| let xlsx_cli_max_query_value_chars : Int = 1024 +///| +let xlsx_cli_query_yield_work_units : Int = 4096 + ///| enum XlsxCellKind { Formula @@ -375,11 +378,79 @@ fn prepare_xlsx_query_predicates( } ///| -fn PreparedXlsxCellPredicate::matches( +async fn xlsx_query_strings_equal_cooperative( + value : String, + expected : String, + work : OfficeQueryWorkBudget, +) -> Bool { + if value.length() != expected.length() { + // String lengths are already known, so unequal-length values require no + // linear comparison even though the conservative work budget retains the + // previous upper-bound accounting. + work.charge(value.length().min(expected.length()) + 1) + return false + } + let mut pending_work = 1 + for index in 0..= xlsx_cli_query_yield_work_units { + work.charge(pending_work) + pending_work = 0 + @async.pause() + } + pending_work += 1 + if value[index] != expected[index] { + work.charge(pending_work) + return false + } + } + work.charge(pending_work) + true +} + +///| +async fn OfficeLinearPattern::is_in_cooperative( + self : OfficeLinearPattern, + value : String, + work : OfficeQueryWorkBudget, +) -> Bool { + if self.characters.is_empty() { + return true + } + let mut matched = 0 + let mut pending_work = 1 + for raw_character in value { + // Supplementary scalars occupy two UTF-16 units. This preserves the + // original conservative accounting while charging and yielding in small + // quanta rather than monopolizing one scheduler turn for a large cell. + let scalar_work = if raw_character.to_int() > 0xffff { 6 } else { 3 } + if pending_work > xlsx_cli_query_yield_work_units - scalar_work { + work.charge(pending_work) + pending_work = 0 + @async.pause() + } + pending_work += scalar_work + let character = query_fold_character(raw_character, self.ignore_case) + while matched > 0 && character != self.characters[matched] { + matched = self.failure[matched - 1] + } + if character == self.characters[matched] { + matched += 1 + if matched == self.characters.length() { + work.charge(pending_work) + return true + } + } + } + work.charge(pending_work) + false +} + +///| +async fn PreparedXlsxCellPredicate::matches( self : PreparedXlsxCellPredicate, cell : XlsxCellSnapshot, work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { match self { PreparedHasType(kind) => { work.charge(1) @@ -404,7 +475,7 @@ fn PreparedXlsxCellPredicate::matches( } PreparedFormulaContains(pattern) => match cell.formula { - Some(formula) => pattern.is_in(formula, work) + Some(formula) => pattern.is_in_cooperative(formula, work) None => { work.charge(1) false @@ -412,10 +483,8 @@ fn PreparedXlsxCellPredicate::matches( } PreparedTextEquals(expected) => match cell.raw { - Some(String(value)) => { - work.charge(value.length().min(expected.length()) + 1) - value == expected - } + Some(String(value)) => + xlsx_query_strings_equal_cooperative(value, expected, work) _ => { work.charge(1) false @@ -423,7 +492,7 @@ fn PreparedXlsxCellPredicate::matches( } PreparedTextContains(pattern) => match cell.raw { - Some(String(value)) => pattern.is_in(value, work) + Some(String(value)) => pattern.is_in_cooperative(value, work) _ => { work.charge(1) false @@ -433,11 +502,11 @@ fn PreparedXlsxCellPredicate::matches( } ///| -fn xlsx_cell_matches( +async fn xlsx_cell_matches( cell : XlsxCellSnapshot, predicates : Array[PreparedXlsxCellPredicate], work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { work.charge(1) if !cell.is_present() { return false diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 6a89147f..348a6874 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -20,7 +20,48 @@ struct XlsxCellPage { matched_total : Int scanned_cells : Int offset : Int - limit : Int +} + +///| +enum XlsxRetainedJsonKind { + RetainedCellJson + RetainedTextJson +} + +///| +struct XlsxJsonRetention { + budget : OfficeOutputBudget + records : Array[Json] + kind : XlsxRetainedJsonKind +} + +///| +fn xlsx_text_entry_json(cell : XlsxCellSnapshot) -> Json raise CliFailure { + guard xlsx_cell_text(cell) is Some(text) else { + raise xlsx_cli_failure( + "office.xlsx.read_failed", "internal XLSX text retention received a cell without displayed text", + ) + } + Json::object({ + "path": Json::string(cell.path), + "stability": Json::string("snapshot-relative"), + "text": Json::string(text), + }) +} + +///| +fn XlsxJsonRetention::retain( + self : XlsxJsonRetention, + cell : XlsxCellSnapshot, +) -> Unit raise CliFailure { + let record = match self.kind { + RetainedCellJson => xlsx_cell_json(cell) + RetainedTextJson => xlsx_text_entry_json(cell) + } + // Charge before insertion so the live retained result is always bounded by + // the successful-output ceiling, including on Wasm targets. + self.budget.reserve_array_item(record, !self.records.is_empty()) + self.records.push(record) } ///| @@ -203,12 +244,54 @@ fn xlsx_outline_payload( max_output_chars : Int, ) -> BoundedOfficePayload raise CliFailure { projection.checkpoint() + let sheets : Array[Json] = [] + let defined_names : Array[Json] = [] + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_OUTLINE), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string("/xlsx/workbook"), + "sheet_count": Json::number(projection.sheets.length().to_double()), + "sheets": Json::array(sheets), + "defined_names": Json::array(defined_names), + "limits": Json::object({ + "max_scan_cells": Json::number(projection.max_scan_cells.to_double()), + "max_metadata_items": Json::number( + xlsx_cli_max_metadata_items.to_double(), + ), + "max_metadata_string_chars": Json::number( + xlsx_cli_max_metadata_string_chars.to_double(), + ), + }), + } let payload = bounded_office_payload( - xlsx_outline_data(projection), + Json::object(fields), [], max_output_chars, "xlsx", ) + retain_xlsx_outline_records(projection, sheets, defined_names, payload.budget) + if !projection.sheets.is_empty() { + projection.checkpoint() + let active_index = projection.workbook.get_active_sheet_index() + guard projection.sheets.get(active_index) is Some(sheet) else { + raise xlsx_cli_failure( + "office.xlsx.read_failed", + "workbook active-sheet index is outside the tab order", + details=Json::object({ + "active_sheet_index": Json::number(active_index.to_double()), + "sheet_count": Json::number(projection.sheets.length().to_double()), + }), + ) + } + let active = Json::object({ + "path": Json::string(sheet.path), + "name": Json::string(sheet.name), + "index": Json::number((sheet.index + 1).to_double()), + }) + payload.budget.reserve_json_field("active_sheet", active, true) + fields["active_sheet"] = active + } projection.checkpoint() payload } @@ -270,8 +353,19 @@ fn xlsx_styles_json( projection : XlsxProjection, cells : Array[XlsxCellSnapshot], ) -> Json raise CliFailure { - projection.checkpoint() let fields : Map[String, Json] = Map([]) + retain_xlsx_styles(projection, cells, fields) + Json::object(fields) +} + +///| +fn retain_xlsx_styles( + projection : XlsxProjection, + cells : Array[XlsxCellSnapshot], + fields : Map[String, Json], + retention_budget? : OfficeOutputBudget, +) -> Unit raise CliFailure { + projection.checkpoint() for cell in cells { projection.checkpoint() if cell.style_id != 0 { @@ -280,12 +374,17 @@ fn xlsx_styles_json( let style = xlsx_call(projection.file, () => { projection.workbook.get_style(cell.style_id) }) - fields[key] = @inspect.style_to_json(style) + let value = @inspect.style_to_json(style) + match retention_budget { + Some(budget) => + budget.reserve_json_field(key, value, !fields.is_empty()) + None => () + } + fields[key] = value } } } projection.checkpoint() - Json::object(fields) } ///| @@ -329,10 +428,10 @@ fn XlsxCellFilter::needs_formatted(self : XlsxCellFilter) -> Bool { } ///| -fn XlsxCellFilter::matches( +async fn XlsxCellFilter::matches( self : XlsxCellFilter, cell : XlsxCellSnapshot, -) -> Bool raise CliFailure { +) -> Bool { match self { PresentCells => cell.is_present() TextCells => xlsx_cell_text(cell) is Some(_) @@ -367,6 +466,7 @@ async fn collect_xlsx_cell_page( offset : Int, limit : Int, format_work_maximum? : Int = xlsx_cli_max_format_work_units, + retention? : XlsxJsonRetention, ) -> XlsxCellPage { projection.checkpoint() @async.pause() @@ -396,6 +496,10 @@ async fn collect_xlsx_cell_page( if filter.matches(cell) { if matched >= offset && cells.length() < limit { cell.ensure_formatted(projection, scan) + match retention { + Some(value) => value.retain(cell) + None => () + } cells.push(cell) } matched += 1 @@ -412,21 +516,64 @@ async fn collect_xlsx_cell_page( } projection.checkpoint() @async.pause() - { cells, matched_total: matched, scanned_cells: scan.scanned, offset, limit } + { cells, matched_total: matched, scanned_cells: scan.scanned, offset } } ///| -fn xlsx_page_fields(page : XlsxCellPage) -> Map[String, Json] { - { - "matched_total": Json::number(page.matched_total.to_double()), - "offset": Json::number(page.offset.to_double()), - "limit": Json::number(page.limit.to_double()), - "returned": Json::number(page.cells.length().to_double()), - "truncated": Json::boolean( - page.offset + page.cells.length() < page.matched_total, - ), - "scanned_cells": Json::number(page.scanned_cells.to_double()), +fn retain_xlsx_outline_records( + projection : XlsxProjection, + sheets : Array[Json], + defined_names : Array[Json], + budget : OfficeOutputBudget, +) -> Unit raise CliFailure { + let metadata = XlsxMetadataBudget::new() + for sheet in projection.sheets { + projection.checkpoint() + let record = xlsx_sheet_summary_json(projection, sheet, metadata) + budget.reserve_array_item(record, !sheets.is_empty()) + sheets.push(record) + } + for defined in projection.workbook.get_defined_names() { + projection.checkpoint() + metadata.charge_item([ + defined.name, + defined.refers_to, + defined.scope, + defined.comment, + ]) + let record_fields : Map[String, Json] = { + "name": Json::string(defined.name), + "refers_to": Json::string(defined.refers_to), + } + if defined.scope != "" { + record_fields["scope"] = Json::string(defined.scope) + } + if defined.comment != "" { + record_fields["comment"] = Json::string(defined.comment) + } + let record = Json::object(record_fields) + budget.reserve_array_item(record, !defined_names.is_empty()) + defined_names.push(record) } + projection.checkpoint() +} + +///| +fn update_bounded_xlsx_page_fields( + fields : Map[String, Json], + page : XlsxCellPage, + budget : OfficeOutputBudget, +) -> Unit raise CliFailure { + let returned = page.cells.length() + let truncated = page.offset + returned < page.matched_total + budget.reserve_nonnegative_integer_growth(page.matched_total) + budget.reserve_nonnegative_integer_growth(returned) + budget.reserve_nonnegative_integer_growth(page.scanned_cells) + budget.reserve_boolean_growth_from_true(truncated) + fields["matched_total"] = Json::number(page.matched_total.to_double()) + fields["returned"] = Json::number(returned.to_double()) + fields["scanned_cells"] = Json::number(page.scanned_cells.to_double()) + fields["truncated"] = Json::boolean(truncated) } ///| @@ -536,101 +683,144 @@ async fn xlsx_get_payload( max_output_chars : Int, ) -> BoundedOfficePayload { projection.checkpoint() - let payload = bounded_office_payload( - xlsx_get_data(projection, resolved), - [], - max_output_chars, - "xlsx", - ) - projection.checkpoint() - payload + match resolved { + Workbook(path~) => { + let sheets : Array[Json] = [] + let defined_names : Array[Json] = [] + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(path), + "kind": Json::string("workbook"), + "stability": Json::string("stable"), + "sheet_count": Json::number(projection.sheets.length().to_double()), + "sheets": Json::array(sheets), + "defined_names": Json::array(defined_names), + } + let bounded = bounded_office_payload( + Json::object(fields), + [], + max_output_chars, + "xlsx", + ) + retain_xlsx_outline_records( + projection, + sheets, + defined_names, + bounded.budget, + ) + bounded + } + Sheet(_) | Cell(_) => { + let bounded = bounded_office_payload( + xlsx_get_data(projection, resolved), + [], + max_output_chars, + "xlsx", + ) + projection.checkpoint() + bounded + } + Range(sheet, rect, path~) => { + let records : Array[Json] = [] + let styles : Map[String, Json] = Map([]) + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_ELEMENT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "path": Json::string(path), + "kind": Json::string("range"), + "stability": Json::string("snapshot-relative"), + "parent": Json::string(sheet.path), + "reference": Json::string(rect.range_reference(projection.file)), + "cells": Json::array(records), + "styles": Json::object(styles), + "scanned_cells": Json::number(0), + "returned": Json::number(0), + } + let bounded = bounded_office_payload( + Json::object(fields), + [], + max_output_chars, + "xlsx", + ) + let retention : XlsxJsonRetention = { + budget: bounded.budget, + records, + kind: RetainedCellJson, + } + let page = collect_xlsx_cell_page( + projection, + [{ sheet, rect }], + PresentCells, + 0, + projection.max_scan_cells, + retention~, + ) + retain_xlsx_styles( + projection, + page.cells, + styles, + retention_budget=bounded.budget, + ) + bounded.budget.reserve_nonnegative_integer_growth(page.scanned_cells) + bounded.budget.reserve_nonnegative_integer_growth(page.cells.length()) + fields["scanned_cells"] = Json::number(page.scanned_cells.to_double()) + fields["returned"] = Json::number(page.cells.length().to_double()) + projection.checkpoint() + bounded + } + } } ///| -async fn xlsx_text_data( +async fn xlsx_text_payload( projection : XlsxProjection, under : XlsxResolvedSelector?, offset : Int, limit : Int, -) -> Json { + max_output_chars : Int, +) -> BoundedOfficePayload { projection.checkpoint() - let page = collect_xlsx_cell_page( - projection, - xlsx_scan_regions(projection, under), - TextCells, - offset, - limit, - ) let entries : Array[Json] = [] - for cell in page.cells { - projection.checkpoint() - guard xlsx_cell_text(cell) is Some(text) else { continue } - entries.push( - Json::object({ - "path": Json::string(cell.path), - "stability": Json::string("snapshot-relative"), - "text": Json::string(text), - }), - ) + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_TEXT), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "matched_total": Json::number(0), + "offset": Json::number(offset.to_double()), + "limit": Json::number(limit.to_double()), + "returned": Json::number(0), + "truncated": Json::boolean(true), + "scanned_cells": Json::number(0), + "entries": Json::array(entries), } - projection.checkpoint() - let fields = xlsx_page_fields(page) - fields["schema"] = Json::string(@lib.SCHEMA_XLSX_TEXT) - fields["file"] = Json::string(projection.file) - fields["format"] = Json::string("xlsx") - fields["entries"] = Json::array(entries) match under { Some(value) => fields["under"] = Json::string(value.path()) None => () } - Json::object(fields) -} - -///| -async fn xlsx_text_payload( - projection : XlsxProjection, - under : XlsxResolvedSelector?, - offset : Int, - limit : Int, - max_output_chars : Int, -) -> BoundedOfficePayload { - projection.checkpoint() - let payload = bounded_office_payload( - xlsx_text_data(projection, under, offset, limit), + let bounded = bounded_office_payload( + Json::object(fields), [], max_output_chars, "xlsx", ) - projection.checkpoint() - payload -} - -///| -async fn xlsx_query_data( - projection : XlsxProjection, - under : XlsxResolvedSelector?, - spec : XlsxQuerySpec, -) -> Json { - projection.checkpoint() let page = collect_xlsx_cell_page( projection, xlsx_scan_regions(projection, under), - xlsx_query_filter(spec.predicates), - spec.offset, - spec.limit, + TextCells, + offset, + limit, + retention={ + budget: bounded.budget, + records: entries, + kind: RetainedTextJson, + }, ) - let fields = xlsx_page_fields(page) - fields["schema"] = Json::string(@lib.SCHEMA_XLSX_QUERY) - fields["file"] = Json::string(projection.file) - fields["format"] = Json::string("xlsx") - fields["selector"] = Json::string(spec.selector) - fields["matches"] = Json::array(xlsx_cells_json(projection, page.cells)) - fields["styles"] = xlsx_styles_json(projection, page.cells) - match under { - Some(value) => fields["under"] = Json::string(value.path()) - None => () - } - Json::object(fields) + update_bounded_xlsx_page_fields(fields, page, bounded.budget) + projection.checkpoint() + bounded } ///| @@ -641,14 +831,53 @@ async fn xlsx_query_payload( max_output_chars : Int, ) -> BoundedOfficePayload { projection.checkpoint() - let payload = bounded_office_payload( - xlsx_query_data(projection, under, spec), + let matches : Array[Json] = [] + let styles : Map[String, Json] = Map([]) + let fields : Map[String, Json] = { + "schema": Json::string(@lib.SCHEMA_XLSX_QUERY), + "file": Json::string(projection.file), + "format": Json::string("xlsx"), + "selector": Json::string(spec.selector), + "matched_total": Json::number(0), + "offset": Json::number(spec.offset.to_double()), + "limit": Json::number(spec.limit.to_double()), + "returned": Json::number(0), + "truncated": Json::boolean(true), + "scanned_cells": Json::number(0), + "matches": Json::array(matches), + "styles": Json::object(styles), + } + match under { + Some(value) => fields["under"] = Json::string(value.path()) + None => () + } + let bounded = bounded_office_payload( + Json::object(fields), [], max_output_chars, "xlsx", ) + let page = collect_xlsx_cell_page( + projection, + xlsx_scan_regions(projection, under), + xlsx_query_filter(spec.predicates), + spec.offset, + spec.limit, + retention={ + budget: bounded.budget, + records: matches, + kind: RetainedCellJson, + }, + ) + retain_xlsx_styles( + projection, + page.cells, + styles, + retention_budget=bounded.budget, + ) + update_bounded_xlsx_page_fields(fields, page, bounded.budget) projection.checkpoint() - payload + bounded } ///| diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index fa06de24..8f59a76c 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -29,6 +29,25 @@ async fn xlsx_cli_error_async(run : async () -> Unit) -> @lib.ProtocolError { } } +///| +async fn xlsx_text_test_data( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + offset : Int, + limit : Int, +) -> Json { + xlsx_text_payload(projection, under, offset, limit, 100_000).data +} + +///| +async fn xlsx_query_test_data( + projection : XlsxProjection, + under : XlsxResolvedSelector?, + spec : XlsxQuerySpec, +) -> Json { + xlsx_query_payload(projection, under, spec, 100_000).data +} + ///| fn xlsx_bounded_test_source( file : String, @@ -118,13 +137,7 @@ test "XLSX post-projection metadata, styles, and human output observe cancellati ignore( xlsx_page_human( projection, - { - cells: [], - matched_total: 0, - scanned_cells: 0, - offset: 0, - limit: 0, - }, + { cells: [], matched_total: 0, scanned_cells: 0, offset: 0 }, false, 10_000, ), @@ -489,7 +502,7 @@ test "XLSX string and metadata accounting fail before unbounded retention" { } ///| -test "XLSX query selectors are bounded, deterministic predicate conjunctions" { +async test "XLSX query selectors are bounded, deterministic predicate conjunctions" { let projection = xlsx_read_test_projection() guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[B2]") is Cell(sheet, rect, ..) else { @@ -542,7 +555,7 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { ignore(parse_xlsx_query_selector(too_many)) }) assert_eq(predicate_limit.code, "office.xlsx.query_predicate_limit") - let work_limit = xlsx_cli_error(() => { + let work_limit = xlsx_cli_error_async(() => { let limited = OfficeQueryWorkBudget::new(24, format="xlsx") let prepared = prepare_xlsx_query_predicates( parse_xlsx_query_selector("cell[text~=alpha]"), @@ -561,7 +574,7 @@ test "XLSX query selectors are bounded, deterministic predicate conjunctions" { } ///| -test "XLSX query literals preserve whitespace and support JSON quoting" { +async test "XLSX query literals preserve whitespace and support JSON quoting" { let unquoted = parse_xlsx_query_selector("cell[text= leading and trailing ]") guard unquoted[0] is TextEquals(unquoted_text) else { fail("expected exact-text predicate") @@ -623,6 +636,32 @@ test "XLSX query literals preserve whitespace and support JSON quoting" { } } +///| +async test "XLSX large single-cell predicates yield within the comparison" { + let work = OfficeQueryWorkBudget::new( + office_cli_max_query_predicate_work_units, + format="xlsx", + ) + let pattern = OfficeLinearPattern::compile("missing", false, work) + @async.with_task_group() <| group => { + let task = group.spawn(() => { + pattern.is_in_cooperative("a".repeat(1024 * 1024), work) + }) + // A single cell is larger than one comparison quantum. The child must + // yield from inside KMP rather than waiting for the outer cell counter. + @async.pause() + assert_true(task.try_wait() is None) + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + ///| async test "XLSX snapshots preserve and resolve shared-formula followers" { let workbook = @xlsx.Workbook::new() @@ -644,7 +683,7 @@ async test "XLSX snapshots preserve and resolve shared-formula followers" { offset: 0, limit: 10, } - guard xlsx_query_data(projection, under, spec) is Object(fields) && + guard xlsx_query_test_data(projection, under, spec) is Object(fields) && fields.get("matches") is Some(Array(matches)) else { fail("expected shared-formula matches") } @@ -660,7 +699,8 @@ async test "XLSX snapshots preserve and resolve shared-formula followers" { offset: 0, limit: 10, } - guard xlsx_query_data(projection, under, master_only) is Object(master_fields) else { + guard xlsx_query_test_data(projection, under, master_only) + is Object(master_fields) else { fail("expected translated shared-formula query") } assert_eq(master_fields.get("matched_total"), Some(Json::number(1))) @@ -962,7 +1002,7 @@ async test "XLSX displayed text preserves formula results formatted to blank" { let under = Some( resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), ) - guard xlsx_text_data(projection, under, 0, 10) is Object(text) else { + guard xlsx_text_test_data(projection, under, 0, 10) is Object(text) else { fail("expected text result") } assert_eq(text.get("matched_total"), Some(Json::number(0))) @@ -1001,7 +1041,7 @@ async test "XLSX get text and query honor the 1904 date system" { let under = Some( resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Dates\"]"), ) - guard xlsx_text_data(projection, under, 0, 10) is Object(text) && + guard xlsx_text_test_data(projection, under, 0, 10) is Object(text) && text.get("entries") is Some(Array([Object(text_entry), ..])) else { fail("expected 1904 text result") } @@ -1013,7 +1053,7 @@ async test "XLSX get text and query honor the 1904 date system" { offset: 0, limit: 10, } - guard xlsx_query_data(projection, under, spec) is Object(query) && + guard xlsx_query_test_data(projection, under, spec) is Object(query) && query.get("matches") is Some(Array([Object(query_cell), ..])) else { fail("expected 1904 query result") } @@ -1091,7 +1131,7 @@ async test "XLSX cached empty formula results remain distinct from a missing cac Json::object({ "type": Json::string("string"), "value": Json::string("") }), ), ) - guard xlsx_text_data( + guard xlsx_text_test_data( projection, Some(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]")), 0, @@ -1109,7 +1149,7 @@ async test "XLSX text and query scan fully while retaining only the requested pa let under = Some( resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), ) - let text = xlsx_text_data(projection, under, 1, 1) + let text = xlsx_text_test_data(projection, under, 1, 1) guard text is Object(text_fields) && text_fields.get("entries") is Some(Array(entries)) else { fail("expected text entries") @@ -1126,7 +1166,7 @@ async test "XLSX text and query scan fully while retaining only the requested pa offset: 0, limit: 10, } - let query = xlsx_query_data(projection, under, formula_query) + let query = xlsx_query_test_data(projection, under, formula_query) guard query is Object(query_fields) && query_fields.get("matches") is Some(Array(matches)) else { fail("expected query matches") @@ -1139,17 +1179,56 @@ async test "XLSX text and query scan fully while retaining only the requested pa ///| async test "XLSX JSON and human output enforce format-specific stdout ceilings" { let projection = xlsx_read_test_projection() - let payload = xlsx_outline_payload(projection, 100_000) - let json = checked_office_json_output(payload) - assert_true(json.contains("office.output/1")) - assert_true(json.contains("office.xlsx.outline/1")) + let style_id = projection.workbook.new_style( + @xlsx.Style::number_format("0.00"), + ) + projection.workbook.set_cell_style("Data", "B2", style_id) + let under = Some( + resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), + ) + let range = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/range[A1:C3]", + ) + let query_spec : XlsxQuerySpec = { + selector: "cell[formula]", + predicates: parse_xlsx_query_selector("cell[formula]"), + offset: 0, + limit: 10, + } + let payloads = [ + xlsx_outline_payload(projection, 100_000), + xlsx_get_payload(projection, range, 100_000), + xlsx_text_payload(projection, under, 0, 10, 100_000), + xlsx_query_payload(projection, under, query_spec, 100_000), + ] + for payload in payloads { + let json = checked_office_json_output(payload) + assert_true(json.contains("office.output/1")) + let mut characters = 0 + for _ in json { + characters += 1 + } + assert_eq(characters + docx_cli_output_framing_chars, payload.budget.used) + } let too_small = xlsx_cli_error(() => { ignore(xlsx_outline_payload(projection, 8)) }) assert_eq(too_small.code, "office.xlsx.resource_limit") - let under = Some( - resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), - ) + let mut checks = 0 + let limited_projection = { + ..projection, + cancelled: () => { + checks += 1 + false + }, + } + let range_too_small = xlsx_cli_error_async(() => { + ignore(xlsx_get_payload(limited_projection, range, 1)) + }) + assert_eq(range_too_small.code, "office.xlsx.resource_limit") + // Only the command-entry checkpoint runs: the bounded skeleton rejects the + // request before any cell is scanned or retained. + assert_eq(checks, 1) let human = xlsx_text_human(projection, under, 0, 10, 10_000) assert_true(human.contains("/cell[A1]\talpha")) assert_true(human.contains("/cell[C3]\t=NOW()")) @@ -1162,7 +1241,7 @@ async test "XLSX all-sheet scans reject sparse far extents before visiting cells workbook.set_cell_str("Sparse", "K10", "far") let projection = make_xlsx_projection("sparse.xlsx", workbook, 100) let error = xlsx_cli_error_async(() => { - ignore(xlsx_text_data(projection, None, 0, 10)) + ignore(xlsx_text_test_data(projection, None, 0, 10)) }) assert_eq(error.code, "office.xlsx.resource_limit") guard error.details is Some(Object(fields)) else { From 1e816c72cfa40592b0d7dd242d497b3110bfe61c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 11:44:18 +0800 Subject: [PATCH 052/150] chore: refresh generated interfaces --- office/cmd/office/pkg.generated.mbti | 4 ++++ xlsx/pkg.generated.mbti | 2 ++ 2 files changed, 6 insertions(+) diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 77ffbd5d..53ce4539 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -69,6 +69,8 @@ type XlsxCellSnapshot type XlsxFormatBudget +type XlsxJsonRetention + type XlsxMetadataBudget type XlsxNumberOperator @@ -81,6 +83,8 @@ type XlsxRect type XlsxResolvedSelector +type XlsxRetainedJsonKind + type XlsxScanBudget type XlsxScanRegion diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 8d631c15..d646b6db 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -595,6 +595,7 @@ pub struct Cols { mut index : Int styles : Array[Style]? options : Options + use_1904_format : Bool mut err : XlsxError? } pub fn Cols::col_index(Self) -> Int? @@ -1291,6 +1292,7 @@ pub struct Rows { mut index : Int styles : Array[Style]? options : Options + use_1904_format : Bool } pub fn Rows::close(Self) -> Unit pub fn Rows::columns(Self, options? : Options) -> Array[String] raise XlsxError From 9b2e5b61408521af7cfc80054dce0f842e74ded8 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 12:48:01 +0800 Subject: [PATCH 053/150] fix(read): close parser and query review gaps --- office/cmd/office/docx_read_output.mbt | 61 ++++++------- office/cmd/office/docx_read_wbtest.mbt | 67 ++++++++++---- office/cmd/office/query_work.mbt | 117 +++++++++++++++++++++++-- office/cmd/office/xlsx_query.mbt | 73 +-------------- office/cmd/office/xlsx_read_model.mbt | 6 +- ooxml/start_tag_scanner.mbt | 19 +++- xlsx/io.mbt | 48 +++++++++- xlsx/read.mbt | 3 + xlsx/read_worksheet_xml.mbt | 9 +- xlsx/stream.mbt | 1 + xlsx/workbook.mbt | 4 + xlsx/worksheet.mbt | 80 ++++++++++++++++- xlsx/worksheet_types.mbt | 4 + 13 files changed, 355 insertions(+), 137 deletions(-) diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index 241eef54..2ebf90e5 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -52,18 +52,20 @@ struct PreparedDocxQuery { } ///| -fn PreparedDocxQuery::build( +async fn PreparedDocxQuery::build( spec : DocxQuerySpec, under : DocxProjectionEntry?, work : OfficeQueryWorkBudget, -) -> PreparedDocxQuery raise CliFailure { +) -> PreparedDocxQuery { let (scope_path, scope_prefix) = match under { Some(root) => (Some(root.path), Some(root.path + "/")) None => (None, None) } let text_pattern = match spec.text { Some(needle) => - Some(OfficeLinearPattern::compile(needle, spec.ignore_case, work)) + Some( + OfficeLinearPattern::compile_cooperative(needle, spec.ignore_case, work), + ) None => None } { spec, scope_path, scope_prefix, text_pattern } @@ -575,15 +577,15 @@ fn query_record_json( } ///| -fn entry_matches_properties( +async fn entry_matches_properties( entry : DocxProjectionEntry, predicates : Array[DocxPropertyPredicate], work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { for predicate in predicates { match entry.properties.get(predicate.name) { Some(value) => - if !query_strings_equal(value, predicate.value, work) { + if !query_strings_equal_cooperative(value, predicate.value, work) { return false } None => { @@ -596,46 +598,35 @@ fn entry_matches_properties( } ///| -fn query_strings_equal( - value : String, - expected : String, - work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { - work.charge(value.length().min(expected.length()) + 1) - value == expected -} - -///| -fn query_entry_is_in_scope( +async fn query_entry_is_in_scope( entry : DocxProjectionEntry, query : PreparedDocxQuery, work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { match (query.scope_path, query.scope_prefix) { (Some(root), Some(prefix)) => if entry.path.length() == root.length() { - query_strings_equal(entry.path, root, work) + query_strings_equal_cooperative(entry.path, root, work) } else { - work.charge(entry.path.length().min(prefix.length()) + 1) - entry.path.has_prefix(prefix) + query_string_has_prefix_cooperative(entry.path, prefix, work) } _ => true } } ///| -fn entry_matches_query( +async fn entry_matches_query( entry : DocxProjectionEntry, query : PreparedDocxQuery, scan_budget : DocxTextBudget, work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { if !query_entry_is_in_scope(entry, query, work) { return false } match query.spec.kind { Some(kind) => - if !query_strings_equal(entry.kind, kind, work) { + if !query_strings_equal_cooperative(entry.kind, kind, work) { return false } None => () @@ -644,7 +635,7 @@ fn entry_matches_query( Some(id) => match entry.stable_id { Some(actual) => - if !query_strings_equal(actual, id, work) { + if !query_strings_equal_cooperative(actual, id, work) { return false } None => { @@ -660,19 +651,19 @@ fn entry_matches_query( match query.text_pattern { Some(pattern) => { let value = entry_text_with_budget(entry, scan_budget) - pattern.is_in(value, work) + pattern.is_in_cooperative(value, work) } None => true } } ///| -fn docx_query_payload( +async fn docx_query_payload( projection : DocxProjection, under : DocxProjectionEntry?, spec : DocxQuerySpec, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { let filter_properties : Array[Json] = [] let filters : Map[String, Json] = { "properties": Json::array(filter_properties), @@ -734,7 +725,12 @@ fn docx_query_payload( ) let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 + let mut visited = 0 for entry in projection.entries { + if (visited & 255) == 0 { + @async.pause() + } + visited += 1 if !entry_matches_query(entry, query, scan_budget, work) { continue } @@ -1178,13 +1174,13 @@ fn docx_text_human( } ///| -fn docx_query_human( +async fn docx_query_human( projection : DocxProjection, under : DocxProjectionEntry?, spec : DocxQuerySpec, warnings : Array[@lib.ProtocolWarning], maximum? : Int = docx_cli_hard_max_output_chars, -) -> String raise CliFailure { +) -> String { let output = DocxHumanWriter::new(maximum) let scan_budget = DocxTextBudget::new( "query text scan characters", docx_cli_max_query_text_scan_chars, @@ -1195,7 +1191,12 @@ fn docx_query_human( let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 let mut returned = 0 + let mut visited = 0 for entry in projection.entries { + if (visited & 255) == 0 { + @async.pause() + } + visited += 1 if !entry_matches_query(entry, query, scan_budget, work) { continue } diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index cc1e26a3..8b21eeaa 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -178,7 +178,7 @@ fn assert_docx_capability_payload_contract( } ///| -test "DOCX capability declarations cover every payload field" { +async test "DOCX capability declarations cover every payload field" { let projection = docx_read_test_projection() assert_docx_capability_payload_contract( "outline", @@ -230,7 +230,7 @@ test "DOCX get distinguishes stable items and positional descendants" { } ///| -test "DOCX text and query paginate in deterministic document order" { +async test "DOCX text and query paginate in deterministic document order" { let projection = docx_read_test_projection() let text = docx_text_payload(projection, None, None, 1, 3, 100_000).data guard text is Object(text_fields) && @@ -282,7 +282,7 @@ test "DOCX text and query paginate in deterministic document order" { } ///| -test "DOCX query uses only its declared property set" { +async test "DOCX query uses only its declared property set" { let projection = docx_read_test_projection() let links = docx_query_payload( projection, @@ -310,7 +310,7 @@ test "DOCX query uses only its declared property set" { } ///| -test "DOCX exact queries cover the full supported XML token length" { +async test "DOCX exact queries cover the full supported XML token length" { let long_id = "i".repeat(@lib.SELECTOR_MAX_VALUE_LENGTH + 1) let long_property = "p".repeat(@lib.SELECTOR_MAX_VALUE_LENGTH + 1) validate_query_value(long_id, "id", false) @@ -370,25 +370,31 @@ test "DOCX exact queries cover the full supported XML token length" { } ///| -test "DOCX query matching is linear-work and Unicode simple-case aware" { +async test "DOCX query matching is linear-work and Unicode simple-case aware" { let work = OfficeQueryWorkBudget::new(100_000) let adversarial = OfficeLinearPattern::compile( "a".repeat(127) + "b", false, work, ) - assert_false(adversarial.is_in("a".repeat(4096), work)) - assert_true(adversarial.is_in("a".repeat(127) + "b", work)) + assert_false(adversarial.is_in_cooperative("a".repeat(4096), work)) + assert_true(adversarial.is_in_cooperative("a".repeat(127) + "b", work)) let unicode = OfficeLinearPattern::compile("école", true, work) - assert_true(unicode.is_in("L'ÉCOLE MoonBit", work)) + assert_true(unicode.is_in_cooperative("L'ÉCOLE MoonBit", work)) let supplementary = OfficeLinearPattern::compile("𐐨", true, work) - assert_true(supplementary.is_in("𐐀", work)) + assert_true(supplementary.is_in_cooperative("𐐀", work)) } ///| -test "DOCX query predicate work has an independent hard budget" { +async test "DOCX query predicate work has an independent hard budget" { let equality_work = OfficeQueryWorkBudget::new(16) - try query_strings_equal("x".repeat(32), "x".repeat(32), equality_work) catch { + try + query_strings_equal_cooperative( + "x".repeat(32), + "x".repeat(32), + equality_work, + ) + catch { CliFailure(error) => { assert_eq(error.code, "office.docx.resource_limit") guard error.details is Some(Object(details)) else { @@ -399,18 +405,46 @@ test "DOCX query predicate work has an independent hard budget" { Some(Json::string("query predicate work units")), ) } + _ => fail("unexpected predicate work failure") } noraise { _ => fail("expected exact comparison work to be bounded") } let pattern_work = OfficeQueryWorkBudget::new(32) - let pattern = OfficeLinearPattern::compile("a", false, pattern_work) - try pattern.is_in("a".repeat(32), pattern_work) catch { + let pattern = OfficeLinearPattern::compile("b", false, pattern_work) + try pattern.is_in_cooperative("a".repeat(32), pattern_work) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected pattern work failure") } noraise { _ => fail("expected substring work to be bounded") } } +///| +async test "DOCX large exact predicates yield within one comparison" { + let work = OfficeQueryWorkBudget::new( + office_cli_max_query_predicate_work_units, + ) + @async.with_task_group() <| group => { + let task = group.spawn(() => { + query_strings_equal_cooperative( + "x".repeat(1024 * 1024), + "x".repeat(1024 * 1024), + work, + ) + }) + @async.pause() + assert_true(task.try_wait() is None) + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + ///| test "DOCX resolver reports malformed, missing, and unsupported stable selectors" { let projection = docx_read_test_projection() @@ -643,7 +677,7 @@ test "compact JSON preflight matches serializer character accounting" { } ///| -test "bounded DOCX payloads account for their exact serialized envelopes" { +async test "bounded DOCX payloads account for their exact serialized envelopes" { let projection = docx_read_test_projection() let paragraph = resolve_projection_selector(projection, "/docx/body/p[1]") let query_spec : DocxQuerySpec = { @@ -712,7 +746,7 @@ test "bounded DOCX payloads account for their exact serialized envelopes" { } ///| -test "DOCX output builders reject growth before returning oversized output" { +async test "DOCX output builders reject growth before returning oversized output" { let projection = docx_read_test_projection() let entry = resolve_projection_selector(projection, "/docx/body/p[1]") try docx_get_human(entry, "x".repeat(1000), [], maximum=40) catch { @@ -751,6 +785,7 @@ test "DOCX output builders reject growth before returning oversized output" { } try docx_query_payload(projection, None, query_spec, 1) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected query output failure") } noraise { _ => fail("expected query skeleton to fail before retaining records") } @@ -1000,7 +1035,7 @@ test "DOCX preflight rejects foreign-namespace officeDocument relationships" { } ///| -test "DOCX human output escapes control sequences in paths, kinds, and text" { +async test "DOCX human output escapes control sequences in paths, kinds, and text" { let projection = docx_read_test_projection() let unsafe_path = "/docx/body/\u{009b}story" let unsafe_kind = "body\u{009b}" diff --git a/office/cmd/office/query_work.mbt b/office/cmd/office/query_work.mbt index faec6437..6b5652c2 100644 --- a/office/cmd/office/query_work.mbt +++ b/office/cmd/office/query_work.mbt @@ -4,6 +4,9 @@ /// limit also covers supplementary characters without under-counting. let office_cli_max_query_predicate_work_units : Int = 128 * 1024 * 1024 +///| +let office_cli_query_yield_work_units : Int = 4096 + ///| struct OfficeQueryWorkBudget { format : String @@ -84,19 +87,121 @@ fn OfficeLinearPattern::compile( } ///| -fn OfficeLinearPattern::is_in( +async fn OfficeLinearPattern::compile_cooperative( + needle : String, + ignore_case : Bool, + work : OfficeQueryWorkBudget, +) -> OfficeLinearPattern { + // Reserve the complete conservative cost before retaining either index. + work.charge(needle.length() * 4 + 1) + let characters : Array[Char] = [] + let mut visited = 0 + for character in needle { + if visited >= office_cli_query_yield_work_units { + visited = 0 + @async.pause() + } + characters.push(query_fold_character(character, ignore_case)) + visited += 1 + } + let failure = Array::make(characters.length(), 0) + let mut matched = 0 + visited = 0 + for at in 1..= office_cli_query_yield_work_units { + visited = 0 + @async.pause() + } + while matched > 0 && characters[at] != characters[matched] { + matched = failure[matched - 1] + } + if characters[at] == characters[matched] { + matched += 1 + } + failure[at] = matched + visited += 1 + } + { characters, failure, ignore_case } +} + +///| +async fn query_strings_equal_cooperative( + value : String, + expected : String, + work : OfficeQueryWorkBudget, +) -> Bool { + if value.length() != expected.length() { + // String lengths are already known, so unequal-length values require no + // linear comparison even though the conservative work budget retains the + // same upper-bound accounting as an equal-length comparison. + work.charge(value.length().min(expected.length()) + 1) + return false + } + let mut pending_work = 1 + for index in 0..= office_cli_query_yield_work_units { + work.charge(pending_work) + pending_work = 0 + @async.pause() + } + pending_work += 1 + if value[index] != expected[index] { + work.charge(pending_work) + return false + } + } + work.charge(pending_work) + true +} + +///| +async fn query_string_has_prefix_cooperative( + value : String, + prefix : String, + work : OfficeQueryWorkBudget, +) -> Bool { + if value.length() < prefix.length() { + work.charge(value.length() + 1) + return false + } + let mut pending_work = 1 + for index in 0..= office_cli_query_yield_work_units { + work.charge(pending_work) + pending_work = 0 + @async.pause() + } + pending_work += 1 + if value[index] != prefix[index] { + work.charge(pending_work) + return false + } + } + work.charge(pending_work) + true +} + +///| +async fn OfficeLinearPattern::is_in_cooperative( self : OfficeLinearPattern, value : String, work : OfficeQueryWorkBudget, -) -> Bool raise CliFailure { +) -> Bool { if self.characters.is_empty() { return true } - // KMP performs at most two pattern comparisons per scalar. The third unit - // covers iteration/folding; UTF-16 length safely over-counts scalars. - work.charge(value.length() * 3 + 1) let mut matched = 0 + let mut pending_work = 1 for raw_character in value { + // Supplementary scalars occupy two UTF-16 units. This preserves the + // conservative accounting while charging and yielding in small quanta. + let scalar_work = if raw_character.to_int() > 0xffff { 6 } else { 3 } + if pending_work > office_cli_query_yield_work_units - scalar_work { + work.charge(pending_work) + pending_work = 0 + @async.pause() + } + pending_work += scalar_work let character = query_fold_character(raw_character, self.ignore_case) while matched > 0 && character != self.characters[matched] { matched = self.failure[matched - 1] @@ -104,9 +209,11 @@ fn OfficeLinearPattern::is_in( if character == self.characters[matched] { matched += 1 if matched == self.characters.length() { + work.charge(pending_work) return true } } } + work.charge(pending_work) false } diff --git a/office/cmd/office/xlsx_query.mbt b/office/cmd/office/xlsx_query.mbt index bb13fbd8..ce5c5a50 100644 --- a/office/cmd/office/xlsx_query.mbt +++ b/office/cmd/office/xlsx_query.mbt @@ -7,9 +7,6 @@ let xlsx_cli_max_query_predicates : Int = 16 ///| let xlsx_cli_max_query_value_chars : Int = 1024 -///| -let xlsx_cli_query_yield_work_units : Int = 4096 - ///| enum XlsxCellKind { Formula @@ -377,74 +374,6 @@ fn prepare_xlsx_query_predicates( predicates.map(predicate => predicate.prepare(work)) } -///| -async fn xlsx_query_strings_equal_cooperative( - value : String, - expected : String, - work : OfficeQueryWorkBudget, -) -> Bool { - if value.length() != expected.length() { - // String lengths are already known, so unequal-length values require no - // linear comparison even though the conservative work budget retains the - // previous upper-bound accounting. - work.charge(value.length().min(expected.length()) + 1) - return false - } - let mut pending_work = 1 - for index in 0..= xlsx_cli_query_yield_work_units { - work.charge(pending_work) - pending_work = 0 - @async.pause() - } - pending_work += 1 - if value[index] != expected[index] { - work.charge(pending_work) - return false - } - } - work.charge(pending_work) - true -} - -///| -async fn OfficeLinearPattern::is_in_cooperative( - self : OfficeLinearPattern, - value : String, - work : OfficeQueryWorkBudget, -) -> Bool { - if self.characters.is_empty() { - return true - } - let mut matched = 0 - let mut pending_work = 1 - for raw_character in value { - // Supplementary scalars occupy two UTF-16 units. This preserves the - // original conservative accounting while charging and yielding in small - // quanta rather than monopolizing one scheduler turn for a large cell. - let scalar_work = if raw_character.to_int() > 0xffff { 6 } else { 3 } - if pending_work > xlsx_cli_query_yield_work_units - scalar_work { - work.charge(pending_work) - pending_work = 0 - @async.pause() - } - pending_work += scalar_work - let character = query_fold_character(raw_character, self.ignore_case) - while matched > 0 && character != self.characters[matched] { - matched = self.failure[matched - 1] - } - if character == self.characters[matched] { - matched += 1 - if matched == self.characters.length() { - work.charge(pending_work) - return true - } - } - } - work.charge(pending_work) - false -} - ///| async fn PreparedXlsxCellPredicate::matches( self : PreparedXlsxCellPredicate, @@ -484,7 +413,7 @@ async fn PreparedXlsxCellPredicate::matches( PreparedTextEquals(expected) => match cell.raw { Some(String(value)) => - xlsx_query_strings_equal_cooperative(value, expected, work) + query_strings_equal_cooperative(value, expected, work) _ => { work.charge(1) false diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 73e00c59..9f61d1dd 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -417,9 +417,9 @@ fn XlsxSheetTarget::worksheet( } ///| -/// Resolves shared-formula masters on demand. Most reads never encounter a -/// shared follower, so they avoid a worksheet-wide formula pass; the first -/// follower builds every master in one pass and later lookups are O(1). +/// Resolves shared-formula masters on demand. Parsed worksheets retain the +/// validated master index built under the XLSX read-work budget, so the first +/// follower copies only the usually-small master map and later lookups are O(1). fn XlsxSheetTarget::shared_formula_master( self : XlsxSheetTarget, projection : XlsxProjection, diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 9d410f87..d09dcb52 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -320,6 +320,13 @@ fn validate_xml_declaration( if attributes_start >= end || !is_attr_space(xml[attributes_start]) { raise InvalidXml(msg="XML declaration version is missing") } + let mut content_end = end + while content_end > attributes_start && is_attr_space(xml[content_end - 1]) { + content_end = content_end - 1 + } + if content_end > attributes_start && xml[content_end - 1] == ('/' : UInt16) { + raise InvalidXml(msg="XML declaration cannot be self-closing") + } let mut next = attributes_start let mut position = 0 let mut saw_encoding = false @@ -504,7 +511,9 @@ fn next_xml_attribute( check_xml_cancelled(cancelled) let mut index = start let mut next_checkpoint = start + 4096 + let mut separated = false while index < tag_end && is_attr_space(xml[index]) { + separated = true if index >= next_checkpoint { check_xml_cancelled(cancelled) next_checkpoint = index + 4096 @@ -520,6 +529,9 @@ fn next_xml_attribute( } return None } + if !separated { + raise InvalidXml(msg="XML attribute is not separated by whitespace") + } let name_start = index while index < tag_end && !is_attr_space(xml[index]) && @@ -637,6 +649,7 @@ pub fn decode_xml_attribute( cancelled? : () -> Bool = () => false, ) -> String raise ParseXmlError { check_xml_cancelled(cancelled) + validate_xml_character_span(value, 0, value.length(), cancelled) let mut needs_decode = false for index, unit in value { if (index & 4095) == 0 { @@ -713,6 +726,7 @@ fn validate_xml_attribute_value( cancelled : () -> Bool, ) -> Unit raise ParseXmlError { check_xml_cancelled(cancelled) + validate_xml_character_span(value, 0, value.length(), cancelled) let mut index = 0 let mut next_checkpoint = 0 while index < value.length() { @@ -1889,7 +1903,7 @@ test "start tag scanner validates unused attributes and document structure" { for xml in [ "", "", "", "text", - "", + "", "", ] { let scanner = XmlStartTagScanner::new(xml) try { @@ -1916,7 +1930,8 @@ test "start tag scanner validates declarations, PI targets, and comments" { for xml in [ "", "", "", - "", "", "", + "", "", "", + "", "", ] { let scanner = XmlStartTagScanner::new(xml) try { diff --git a/xlsx/io.mbt b/xlsx/io.mbt index c39dc7e1..cd0e2c0a 100644 --- a/xlsx/io.mbt +++ b/xlsx/io.mbt @@ -27,6 +27,26 @@ test "bounded file-size telemetry saturates instead of truncating" { inspect(bounded_actual_above_limit(0x7fff_ffff), content="2147483647") } +///| +test "async byte bridge forwards cancellation into semantic parsing" { + let source = Workbook::new() + ignore(source.add_sheet("Sheet1")) + let bytes = write(source) + let result : Result[Workbook, XlsxError] = Ok( + read_workbook_from_bytes( + bytes, + "", + Options::new(), + ReadLimits::new(), + read_io_context(None), + cancelled=() => true, + ), + ) catch { + error => Err(error) + } + inspect(result is Err(ReadCancelled), content="true") +} + ///| async fn[R : @async/io.Reader] read_all_bytes( reader : R, @@ -242,6 +262,7 @@ fn read_workbook_from_bytes( options : Options, limits : ReadLimits, io_context : WorkbookIOContext, + cancelled? : () -> Bool = () => false, ) -> Workbook raise XlsxError { let resolved_password = if password == "" { options.password @@ -256,9 +277,11 @@ fn read_workbook_from_bytes( bytes, options=resolved_options, limits~, + cancelled~, transcoder=value, ) - None => read_zip_bytes(bytes, options=resolved_options, limits~) + None => + read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) } } else { match io_context.charset_transcoder { @@ -268,6 +291,7 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, + cancelled~, transcoder=value, ) None => @@ -276,6 +300,7 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, + cancelled~, ) } } @@ -298,7 +323,12 @@ pub async fn[R : @async/io.Reader] open_reader( ) let io_context = read_io_context(transcoder) let workbook = read_workbook_from_bytes( - bytes, password, options, limits, io_context, + bytes, + password, + options, + limits, + io_context, + cancelled=() => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook @@ -347,7 +377,12 @@ pub async fn[R : @async/io.Reader] Workbook::read_zip_reader( resource="package_bytes", ) let workbook = read_workbook_from_bytes( - bytes, password, resolved_options, limits, io_context, + bytes, + password, + resolved_options, + limits, + io_context, + cancelled=() => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook @@ -375,7 +410,12 @@ pub async fn open_file( Some(path), ) let workbook = read_workbook_from_bytes( - bytes, password, options, limits, io_context, + bytes, + password, + options, + limits, + io_context, + cancelled=() => @async.is_being_cancelled(), ) workbook.set_io_context(io_context) workbook diff --git a/xlsx/read.mbt b/xlsx/read.mbt index eb94c84d..4397fc6a 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3727,6 +3727,7 @@ fn read_zip_archive_core_unchecked( } let ( cells, + shared_formula_masters_index, merged_cells, auto_filter, row_dimensions, @@ -4191,6 +4192,8 @@ fn read_zip_archive_core_unchecked( cells, cell_index, cell_index_valid: true, + shared_formula_masters_index, + shared_formula_masters_index_valid: true, merged_cells, auto_filter, page_margins, diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index dd6373bb..ca3896ed 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -31,6 +31,7 @@ fn parse_worksheet( budget? : ReadBudget, ) -> ( Array[Cell], + Map[UInt, SharedFormulaMaster], Array[String], AutoFilter?, Map[Int, RowDimension], @@ -290,7 +291,7 @@ fn parse_worksheet( } None => () } - ignore(validated_shared_formula_masters(cells)) + let shared_formula_masters_index = validated_shared_formula_masters(cells) worksheet_parse_pass(xml, budget) let merged_cells = parse_merge_cells(xml) worksheet_parse_pass(xml, budget) @@ -320,8 +321,8 @@ fn parse_worksheet( worksheet_parse_pass(xml, budget) let picture_rel_id = parse_picture_rel_id(xml) ( - cells, merged_cells, auto_filter, row_dimensions, col_dimensions, page_margins, - page_layout, header_footer, sheet_protection, row_breaks, col_breaks, sheet_views, - sheet_props, dimension_ref, picture_rel_id, + cells, shared_formula_masters_index, merged_cells, auto_filter, row_dimensions, + col_dimensions, page_margins, page_layout, header_footer, sheet_protection, row_breaks, + col_breaks, sheet_views, sheet_props, dimension_ref, picture_rel_id, ) } diff --git a/xlsx/stream.mbt b/xlsx/stream.mbt index 264d674e..60a52fc5 100644 --- a/xlsx/stream.mbt +++ b/xlsx/stream.mbt @@ -455,6 +455,7 @@ pub fn StreamWriter::set_row_cells( // index (e.g. from a read on the then-empty sheet) would go stale and // make indexed getters miss every streamed cell. self.sheet.invalidate_cell_index() + self.sheet.invalidate_shared_formula_masters_index() self.last_row = row } diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 5aa6c23a..d1349172 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1314,6 +1314,10 @@ fn clone_worksheet( cells, cell_index: build_worksheet_cell_index(cells), cell_index_valid: true, + shared_formula_masters_index: clone_shared_formula_masters_index( + sheet.shared_formula_masters_index, + ), + shared_formula_masters_index_valid: sheet.shared_formula_masters_index_valid, merged_cells: clone_strings(sheet.merged_cells), hyperlinks: clone_hyperlinks(sheet.hyperlinks), tables: clone_tables(sheet.tables), diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index cc85d872..53db4ee7 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -112,6 +112,7 @@ pub fn Worksheet::set_cell( value : String, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) match self.cell_index_of(row, col) { @@ -186,6 +187,7 @@ pub fn Worksheet::set_cell_value( value : CellValue, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) let value_type = cell_value_type(value) @@ -282,6 +284,7 @@ pub fn Worksheet::set_cell_rich_text( runs : ArrayView[RichTextRun], ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let total = rich_text_total_units(runs) if total > max_cell_chars { raise CellTextTooLong(len=total) @@ -452,6 +455,38 @@ fn validated_shared_formula_masters( masters } +///| +fn clone_shared_formula_masters_index( + source : Map[UInt, SharedFormulaMaster], +) -> Map[UInt, SharedFormulaMaster] { + let cloned : Map[UInt, SharedFormulaMaster] = Map([]) + for shared_index, master in source { + cloned[shared_index] = master + } + cloned +} + +///| +fn Worksheet::invalidate_shared_formula_masters_index(self : Worksheet) -> Unit { + self.shared_formula_masters_index.clear() + self.shared_formula_masters_index_valid = false +} + +///| +fn Worksheet::ensure_shared_formula_masters_index( + self : Worksheet, +) -> Unit raise XlsxError { + if self.shared_formula_masters_index_valid { + return + } + let rebuilt = validated_shared_formula_masters(self.cells) + self.shared_formula_masters_index.clear() + for shared_index, master in rebuilt { + self.shared_formula_masters_index[shared_index] = master + } + self.shared_formula_masters_index_valid = true +} + ///| /// Returns non-empty shared-formula masters keyed by OOXML shared index after /// validating that every follower has exactly one master and lies within the @@ -459,7 +494,40 @@ fn validated_shared_formula_masters( pub fn Worksheet::shared_formula_masters( self : Worksheet, ) -> Map[UInt, SharedFormulaMaster] raise XlsxError { - validated_shared_formula_masters(self.cells) + self.ensure_shared_formula_masters_index() + clone_shared_formula_masters_index(self.shared_formula_masters_index) +} + +///| +test "shared formula masters stay eagerly indexed after bounded read" { + let source = Workbook::new() + let source_sheet = source.add_sheet("Data") + source_sheet.set_cell_formula_opts( + "A1", + "C1+1", + opts=FormulaOpts::shared("A1:A2"), + ) + let parsed = read(write(source)) + guard parsed.sheet("Data") is Some(sheet) else { + fail("expected parsed shared-formula worksheet") + } + assert_true(sheet.shared_formula_masters_index_valid) + assert_eq(sheet.shared_formula_masters_index.length(), 1) + let exported = sheet.shared_formula_masters() + exported.clear() + assert_eq(sheet.shared_formula_masters_index.length(), 1) +} + +///| +test "cell mutation invalidates and safely rebuilds shared formula index" { + let sheet = Worksheet::new("Data") + sheet.set_cell_formula_opts("A1", "C1+1", opts=FormulaOpts::shared("A1:A2")) + assert_false(sheet.shared_formula_masters_index_valid) + assert_eq(sheet.shared_formula_masters().length(), 1) + assert_true(sheet.shared_formula_masters_index_valid) + sheet.set_cell("D1", "value") + assert_false(sheet.shared_formula_masters_index_valid) + assert_eq(sheet.shared_formula_masters().length(), 1) } ///| @@ -470,6 +538,7 @@ pub fn Worksheet::set_cell_formula( value? : String = "", ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) match self.cell_index_of(row, col) { @@ -567,6 +636,7 @@ pub fn Worksheet::set_cell_formula_opts( value? : String = "", ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let formula_kind = opts.formula_type match formula_kind { None => self.set_cell_formula(reference, formula, value~) @@ -2457,6 +2527,7 @@ pub fn Worksheet::set_cell_rc( value : String, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let reference = cell_ref_from(row, col) match self.cell_index_of(row, col) { Some(i) => { @@ -2505,6 +2576,7 @@ pub fn Worksheet::set_cell_value_rc( value : CellValue, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let reference = cell_ref_from(row, col) let value_type = cell_value_type(value) let raw_value = cell_value_raw_string(value) @@ -2556,6 +2628,7 @@ pub fn Worksheet::set_cell_formula_rc( value? : String = "", ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() let reference = cell_ref_from(row, col) match self.cell_index_of(row, col) { Some(i) => { @@ -3145,6 +3218,7 @@ fn Worksheet::insert_rows( count : Int, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() if row <= 0 || row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{row}") } @@ -3295,6 +3369,7 @@ fn Worksheet::remove_rows( count : Int, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() if row <= 0 { raise InvalidCellRef(value="\{0}:\{row}") @@ -3607,6 +3682,7 @@ fn Worksheet::duplicate_row_to( target_row : Int, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() if row <= 0 || row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{row}") @@ -3660,6 +3736,7 @@ fn Worksheet::insert_cols( count : Int, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() if col <= 0 || col > cell_ref_max_cols { raise InvalidCellRef(value="\{col}:\{0}") } @@ -3803,6 +3880,7 @@ fn Worksheet::remove_cols( count : Int, ) -> Unit raise XlsxError { self.ensure_stream_idle() + self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() if col <= 0 { raise InvalidCellRef(value="\{col}:\{0}") diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 1d03a5bd..ad701dae 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -98,6 +98,8 @@ pub struct Worksheet { cells : Array[Cell] cell_index : Map[String, Int] mut cell_index_valid : Bool + shared_formula_masters_index : Map[UInt, SharedFormulaMaster] + mut shared_formula_masters_index_valid : Bool merged_cells : Array[String] hyperlinks : Array[Hyperlink] tables : Array[Table] @@ -148,6 +150,8 @@ pub fn Worksheet::new(name : String) -> Worksheet { // Empty worksheets begin with a valid index. Normal mutators update it as // cells are added, avoiding a deferred whole-sheet rebuild on first read. cell_index_valid: true, + shared_formula_masters_index: Map([]), + shared_formula_masters_index_valid: true, merged_cells: [], hyperlinks: [], tables: [], From 279d84fa41198f74457114b821dbb03418f9c3c5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 12:54:47 +0800 Subject: [PATCH 054/150] fix(office): make DOCX reads cooperative --- docx2html/opc/validate.mbt | 17 ++- office/cmd/office/docx_commands.mbt | 8 +- office/cmd/office/docx_projection.mbt | 187 ++++++++++++++++++++----- office/cmd/office/docx_read_model.mbt | 6 +- office/cmd/office/docx_read_output.mbt | 49 ++++--- office/cmd/office/docx_read_wbtest.mbt | 160 +++++++++++++++------ 6 files changed, 320 insertions(+), 107 deletions(-) diff --git a/docx2html/opc/validate.mbt b/docx2html/opc/validate.mbt index f852af84..7d7035e8 100644 --- a/docx2html/opc/validate.mbt +++ b/docx2html/opc/validate.mbt @@ -218,6 +218,7 @@ fn FindingCollector::new_with_xml_limits( max_xml_tokens : Int, max_xml_materialized_chars : Int, max_xml_token_chars : Int, + cancelled : () -> Bool, ) -> FindingCollector { { findings: [], @@ -259,6 +260,7 @@ fn FindingCollector::new_with_xml_limits( } else { 0 }, + cancelled~, ), truncated: false, resource_limit: None, @@ -508,8 +510,15 @@ pub fn validate_docx_package_report_limited( max_xml_token_chars? : Int = DEFAULT_MAX_XML_TOKEN_CHARS, ) -> DocxValidationReport { let findings = FindingCollector::new_with_xml_limits( - max_findings, max_message_chars, max_zip_entry_name_chars, max_zip_total_entry_name_chars, - max_xml_source_units, max_xml_tokens, max_xml_materialized_chars, max_xml_token_chars, + max_findings, + max_message_chars, + max_zip_entry_name_chars, + max_zip_total_entry_name_chars, + max_xml_source_units, + max_xml_tokens, + max_xml_materialized_chars, + max_xml_token_chars, + () => false, ) let archive = @mbtzip.read_limited( data, @@ -576,6 +585,7 @@ pub fn validate_docx_archive_limited( max_xml_tokens? : Int = DEFAULT_MAX_XML_TOKENS, max_xml_materialized_chars? : Int = DEFAULT_MAX_XML_MATERIALIZED_CHARS, max_xml_token_chars? : Int = DEFAULT_MAX_XML_TOKEN_CHARS, + cancelled? : () -> Bool = () => false, ) -> (Array[String], Bool) { let report = validate_docx_archive_report_limited( archive, @@ -587,6 +597,7 @@ pub fn validate_docx_archive_limited( max_xml_tokens~, max_xml_materialized_chars~, max_xml_token_chars~, + cancelled~, ) (report.findings, report.findings_truncated) } @@ -604,10 +615,12 @@ pub fn validate_docx_archive_report_limited( max_xml_tokens? : Int = DEFAULT_MAX_XML_TOKENS, max_xml_materialized_chars? : Int = DEFAULT_MAX_XML_MATERIALIZED_CHARS, max_xml_token_chars? : Int = DEFAULT_MAX_XML_TOKEN_CHARS, + cancelled? : () -> Bool = () => false, ) -> DocxValidationReport { let findings = FindingCollector::new_with_xml_limits( max_findings, max_message_chars, max_zip_entry_name_chars, max_zip_total_entry_name_chars, max_xml_source_units, max_xml_tokens, max_xml_materialized_chars, max_xml_token_chars, + cancelled, ) validate_docx_archive_into(archive, findings) findings.report() diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index d9d0760f..6e7fa0bc 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -103,10 +103,10 @@ fn emit_docx_success(output : String) -> Unit { } ///| -fn resolve_optional_under( +async fn resolve_optional_under( projection : DocxProjection, matches : @argparse.Matches, -) -> (DocxProjectionEntry?, String?) raise CliFailure { +) -> (DocxProjectionEntry?, String?) { match optional_value(matches, "under") { Some(path) => { let entry = resolve_projection_selector(projection, path) @@ -141,6 +141,7 @@ async fn run_docx_outline(matches : @argparse.Matches) -> Unit { file, source.archive, max_elements, + cancelled=office_async_cancelled, ) if matches.flags.get_or_default("json", false) { emit_docx_success( @@ -195,6 +196,7 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { file, source.archive, max_elements, + cancelled=office_async_cancelled, ) let entry = resolve_projection_selector(projection, selector) if matches.flags.get_or_default("json", false) { @@ -266,6 +268,7 @@ async fn run_docx_text(matches : @argparse.Matches) -> Unit { file, source.archive, max_elements, + cancelled=office_async_cancelled, ) let (under, under_path) = resolve_optional_under(projection, matches) if matches.flags.get_or_default("json", false) { @@ -545,6 +548,7 @@ async fn run_docx_query(matches : @argparse.Matches) -> Unit { file, source.archive, max_elements, + cancelled=office_async_cancelled, ) let (under, under_path) = resolve_optional_under(projection, matches) let spec : DocxQuerySpec = { diff --git a/office/cmd/office/docx_projection.mbt b/office/cmd/office/docx_projection.mbt index 72bc027a..4b7b8ce9 100644 --- a/office/cmd/office/docx_projection.mbt +++ b/office/cmd/office/docx_projection.mbt @@ -19,6 +19,9 @@ let docx_cli_hard_max_elements : Int = 200_000 ///| let docx_cli_max_warnings : Int = 128 +///| +let docx_cli_projection_yield_elements : Int = 256 + ///| enum DocxProjectionRole { StoryRoot @@ -49,6 +52,8 @@ struct DocxProjection { entries : Array[DocxProjectionEntry] entry_index : @sorted_map.SortedMap[String, Int] warnings : Array[@lib.ProtocolWarning] + cancelled : () -> Bool + mut work_since_yield : Int } ///| @@ -84,6 +89,8 @@ struct DocxProjectionBuilder { warnings : Array[@lib.ProtocolWarning] warning_keys : Set[String] mut warnings_truncated : Bool + cancelled : () -> Bool + mut work_since_yield : Int } ///| @@ -150,7 +157,10 @@ fn docx_xml_resource_name(limit : @docx_core.DocxXmlResourceLimit) -> String { } ///| -fn DocxProjectionBuilder::new(max_elements : Int) -> DocxProjectionBuilder { +fn DocxProjectionBuilder::new( + max_elements : Int, + cancelled? : () -> Bool = () => false, +) -> DocxProjectionBuilder { { max_elements, entries: [], @@ -158,6 +168,50 @@ fn DocxProjectionBuilder::new(max_elements : Int) -> DocxProjectionBuilder { warnings: [], warning_keys: Set([]), warnings_truncated: false, + cancelled, + work_since_yield: 0, + } +} + +///| +fn DocxProjectionBuilder::check_cancelled( + self : DocxProjectionBuilder, +) -> Unit raise CliFailure { + check_office_read_cancelled(self.cancelled) +} + +///| +async fn DocxProjectionBuilder::cooperate( + self : DocxProjectionBuilder, + work? : Int = 1, +) -> Unit { + self.check_cancelled() + self.work_since_yield += work.max(0) + if self.work_since_yield >= docx_cli_projection_yield_elements { + self.work_since_yield = 0 + @async.pause() + self.check_cancelled() + } +} + +///| +fn DocxProjection::check_cancelled( + self : DocxProjection, +) -> Unit raise CliFailure { + check_office_read_cancelled(self.cancelled) +} + +///| +async fn DocxProjection::cooperate( + self : DocxProjection, + work? : Int = 1, +) -> Unit { + self.check_cancelled() + self.work_since_yield += work.max(0) + if self.work_since_yield >= docx_cli_projection_yield_elements { + self.work_since_yield = 0 + @async.pause() + self.check_cancelled() } } @@ -198,6 +252,7 @@ fn selector_stability_text(stability : @lib.SelectorStability) -> String { fn DocxProjectionBuilder::require_entry_capacity( self : DocxProjectionBuilder, ) -> Unit raise CliFailure { + self.check_cancelled() if self.entries.length() >= self.max_elements { raise docx_resource_failure( "projection elements", @@ -362,12 +417,14 @@ fn canonical_index_path( } ///| -fn canonical_index_paths( +async fn canonical_index_paths( paths : Array[String], path_roots : @sorted_map.SortedMap[String, String], + builder : DocxProjectionBuilder, ) -> Json { let values : Array[Json] = [] for path in paths { + builder.cooperate() match canonical_index_path(path, path_roots) { Some(canonical) => values.push(Json::string(canonical)) None => () @@ -377,12 +434,17 @@ fn canonical_index_paths( } ///| -fn comment_anchor_metadata( +async fn comment_anchor_metadata( anchor : @docx.AnnotationAnchor, path_roots : @sorted_map.SortedMap[String, String], + builder : DocxProjectionBuilder, ) -> Json { let fields : Map[String, Json] = { - "references": canonical_index_paths(anchor.references(), path_roots), + "references": canonical_index_paths( + anchor.references(), + path_roots, + builder, + ), } match canonical_index_path(anchor.story(), path_roots) { Some(canonical) => fields["story"] = Json::string(canonical) @@ -506,11 +568,13 @@ fn stable_annotation_path( } ///| -fn annotation_id_counts( +async fn annotation_id_counts( pairs : Array[(String, Array[@document.DocumentElement])], + builder : DocxProjectionBuilder, ) -> @sorted_map.SortedMap[String, Int] { let counts : @sorted_map.SortedMap[String, Int] = SortedMap([]) for pair in pairs { + builder.cooperate() let (id, _) = pair counts[id] = counts.get(id).unwrap_or(0) + 1 } @@ -518,18 +582,20 @@ fn annotation_id_counts( } ///| -fn annotation_story_plan( +async fn annotation_story_plan( story : String, kind : String, pairs : Array[(String, Array[@document.DocumentElement])], + builder : DocxProjectionBuilder, ) -> AnnotationStoryPlan { let collection_path = "/docx/\{story}" - let counts = annotation_id_counts(pairs) + let counts = annotation_id_counts(pairs, builder) let paths : Array[String] = [] let mut duplicate_ids = 0 let mut missing_ids = 0 let mut unrepresentable_ids = 0 for index, pair in pairs { + builder.cooperate() let ordinal = index + 1 let (id, _) = pair let occurrence_count = counts.get(id).unwrap_or(0) @@ -556,21 +622,25 @@ fn annotation_story_plan( } ///| -fn add_annotation_path_roots( +async fn add_annotation_path_roots( output : @sorted_map.SortedMap[String, String], plan : AnnotationStoryPlan, + builder : DocxProjectionBuilder, ) -> Unit { for index, path in plan.paths { + builder.cooperate() output["/\{plan.story}/\{plan.kind}[\{index + 1}]"] = path } } ///| -fn index_notes( +async fn index_notes( notes : Array[@docx.NoteInfo], + builder : DocxProjectionBuilder, ) -> @sorted_map.SortedMap[String, @docx.NoteInfo] { let output : @sorted_map.SortedMap[String, @docx.NoteInfo] = SortedMap([]) for note in notes { + builder.cooperate() let id = note.id() if !output.contains(id) { output[id] = note @@ -580,19 +650,22 @@ fn index_notes( } ///| -fn index_comment_anchors( +async fn index_comment_anchors( comments : Array[@docx.CommentInfo], path_roots : @sorted_map.SortedMap[String, String], + builder : DocxProjectionBuilder, ) -> @sorted_map.SortedMap[String, Json] { let output : @sorted_map.SortedMap[String, Json] = SortedMap([]) for info in comments { + builder.cooperate() let id = info.id() if output.contains(id) { continue } let anchors : Array[Json] = [] for anchor in info.anchors() { - anchors.push(comment_anchor_metadata(anchor, path_roots)) + builder.cooperate() + anchors.push(comment_anchor_metadata(anchor, path_roots, builder)) } output[id] = Json::array(anchors) } @@ -600,48 +673,56 @@ fn index_comment_anchors( } ///| -fn index_note_references( +async fn index_note_references( notes : @sorted_map.SortedMap[String, @docx.NoteInfo], path_roots : @sorted_map.SortedMap[String, String], + builder : DocxProjectionBuilder, ) -> @sorted_map.SortedMap[String, Json] { let output : @sorted_map.SortedMap[String, Json] = SortedMap([]) for id, note in notes { - output[id] = canonical_index_paths(note.references(), path_roots) + builder.cooperate() + output[id] = canonical_index_paths(note.references(), path_roots, builder) } output } ///| -fn annotation_projection_index( +async fn annotation_projection_index( annotated : @docx.DocxAnnotatedResult, footnotes : AnnotationStoryPlan, endnotes : AnnotationStoryPlan, comments : AnnotationStoryPlan, + builder : DocxProjectionBuilder, ) -> AnnotationProjectionIndex { let path_roots : @sorted_map.SortedMap[String, String] = SortedMap([]) - add_annotation_path_roots(path_roots, footnotes) - add_annotation_path_roots(path_roots, endnotes) - add_annotation_path_roots(path_roots, comments) + add_annotation_path_roots(path_roots, footnotes, builder) + add_annotation_path_roots(path_roots, endnotes, builder) + add_annotation_path_roots(path_roots, comments, builder) let annotations = annotated.annotations() let comment_infos = annotations.comments() - let footnote_infos = index_notes(annotations.footnotes()) - let endnote_infos = index_notes(annotations.endnotes()) + let footnote_infos = index_notes(annotations.footnotes(), builder) + let endnote_infos = index_notes(annotations.endnotes(), builder) { comments: comment_infos, - comment_anchors: index_comment_anchors(comment_infos, path_roots), - footnote_references: index_note_references(footnote_infos, path_roots), - endnote_references: index_note_references(endnote_infos, path_roots), + comment_anchors: index_comment_anchors(comment_infos, path_roots, builder), + footnote_references: index_note_references( + footnote_infos, path_roots, builder, + ), + endnote_references: index_note_references( + endnote_infos, path_roots, builder, + ), } } ///| -fn DocxProjectionBuilder::walk_element_children( +async fn DocxProjectionBuilder::walk_element_children( self : DocxProjectionBuilder, parent : DocxProjectionEntry, element : @document.DocumentElement, -) -> Unit raise CliFailure { +) -> Unit { let counters : Map[String, Int] = Map([]) for child in @docx_paths.element_children(element) { + self.cooperate() match @docx_paths.segment_kind(child) { Some(kind) => { let ordinal = counters.get(kind).unwrap_or(0) + 1 @@ -667,13 +748,13 @@ fn DocxProjectionBuilder::walk_element_children( } ///| -fn DocxProjectionBuilder::add_story( +async fn DocxProjectionBuilder::add_story( self : DocxProjectionBuilder, path : String, kind : String, body : Array[@document.DocumentElement], source : Json, -) -> Unit raise CliFailure { +) -> Unit { let document = @document.document(body) let root = self.add_entry( path, @@ -706,12 +787,12 @@ fn docx_story_source_fields( } ///| -fn DocxProjectionBuilder::add_annotation_story( +async fn DocxProjectionBuilder::add_annotation_story( self : DocxProjectionBuilder, annotation_index : AnnotationProjectionIndex, plan : AnnotationStoryPlan, source : @docx.DocxStoryPartSource?, -) -> Unit raise CliFailure { +) -> Unit { let story = plan.story let kind = plan.kind let pairs = plan.pairs @@ -719,7 +800,10 @@ fn DocxProjectionBuilder::add_annotation_story( let combined_body : Array[@document.DocumentElement] = [] for pair in pairs { let (_, body) = pair - combined_body.append(body) + for element in body { + self.cooperate() + combined_body.push(element) + } } let collection_source = docx_story_source_fields(story, source) let collection = self.add_entry( @@ -734,6 +818,7 @@ fn DocxProjectionBuilder::add_annotation_story( Json::object(collection_source), ) for item_index, pair in pairs { + self.cooperate() self.require_entry_capacity() let ordinal = item_index + 1 let (id, body) = pair @@ -811,30 +896,35 @@ fn document_body_elements( } ///| -fn projection_from_annotated( +async fn projection_from_annotated( file : String, annotated : @docx.DocxAnnotatedResult, max_elements : Int, -) -> DocxProjection raise CliFailure { - let builder = DocxProjectionBuilder::new(max_elements) + cancelled? : () -> Bool = () => false, +) -> DocxProjection { + let builder = DocxProjectionBuilder::new(max_elements, cancelled~) + builder.cooperate(work=docx_cli_projection_yield_elements) let result = annotated.result() let footnotes = annotation_story_plan( "footnotes", "note", annotated.footnote_bodies(), + builder, ) let endnotes = annotation_story_plan( "endnotes", "note", annotated.endnote_bodies(), + builder, ) let comments = annotation_story_plan( "comments", "comment", annotated.comment_bodies(), + builder, ) let annotations = annotation_projection_index( - annotated, footnotes, endnotes, comments, + annotated, footnotes, endnotes, comments, builder, ) let header_sources = annotated.header_story_sources() let footer_sources = annotated.footer_story_sources() @@ -847,6 +937,7 @@ fn projection_from_annotated( ), ) for index, header in result.headers { + builder.cooperate() let source = docx_story_source_fields("header", header_sources.get(index)) source["part_index"] = Json::number((index + 1).to_double()) builder.add_story( @@ -857,6 +948,7 @@ fn projection_from_annotated( ) } for index, footer in result.footers { + builder.cooperate() let source = docx_story_source_fields("footer", footer_sources.get(index)) source["part_index"] = Json::number((index + 1).to_double()) builder.add_story( @@ -882,6 +974,7 @@ fn projection_from_annotated( annotated.comments_story_source(), ) for message in result.messages { + builder.cooperate() match message { Warning(text) => builder.warn("office.docx.reader_warning", text) Error(text) => builder.warn("office.docx.reader_error", text) @@ -893,6 +986,8 @@ fn projection_from_annotated( entries: builder.entries, entry_index: builder.entry_index, warnings: builder.warnings, + cancelled, + work_since_yield: 0, } } @@ -900,7 +995,9 @@ fn projection_from_annotated( fn validate_docx_read_archive( file : String, archive : @zip.Archive, + cancelled? : () -> Bool = () => false, ) -> String raise CliFailure { + check_office_read_cancelled(cancelled) let lower = file.to_lower() if lower.has_suffix(".xlsx") { raise docx_cli_failure( @@ -930,7 +1027,9 @@ fn validate_docx_read_archive( max_xml_tokens=docx_cli_max_xml_tokens, max_xml_materialized_chars=docx_cli_max_xml_materialized_chars, max_xml_token_chars=docx_cli_max_xml_token_chars, + cancelled~, ) + check_office_read_cancelled(cancelled) match report.resource_limit() { Some(limit) => raise docx_cli_failure( @@ -971,11 +1070,13 @@ fn validate_docx_read_archive( } ///| -fn open_docx_projection_archive( +async fn open_docx_projection_archive( file : String, archive : @zip.Archive, max_elements : Int, -) -> DocxProjection raise CliFailure { + cancelled? : () -> Bool = () => false, +) -> DocxProjection { + check_office_read_cancelled(cancelled) if max_elements < 1 || max_elements > docx_cli_hard_max_elements { raise docx_cli_failure( "office.invalid_arguments", @@ -987,12 +1088,17 @@ fn open_docx_projection_archive( }), ) } - let main_document_part = validate_docx_read_archive(file, archive) + @async.pause() + check_office_read_cancelled(cancelled) + let main_document_part = validate_docx_read_archive(file, archive, cancelled~) + @async.pause() + check_office_read_cancelled(cancelled) let xml_budget = @xml.xml_read_budget( max_source_units=docx_cli_max_xml_source_units, max_tokens=docx_cli_max_xml_tokens, max_materialized_chars=docx_cli_max_xml_materialized_chars, max_token_chars=docx_cli_max_xml_token_chars, + cancelled~, ) let annotated = @docx.read_docx_annotated_archive_tolerant_limited( archive, @@ -1001,7 +1107,12 @@ fn open_docx_projection_archive( max_diagnostic_chars=512, expected_main_document_path=main_document_part, ) catch { - error => raise docx_reader_failure(error, file) + error => { + check_office_read_cancelled(cancelled) + raise docx_reader_failure(error, file) + } } - projection_from_annotated(file, annotated, max_elements) + check_office_read_cancelled(cancelled) + @async.pause() + projection_from_annotated(file, annotated, max_elements, cancelled~) } diff --git a/office/cmd/office/docx_read_model.mbt b/office/cmd/office/docx_read_model.mbt index 968df33b..5b3298f7 100644 --- a/office/cmd/office/docx_read_model.mbt +++ b/office/cmd/office/docx_read_model.mbt @@ -354,10 +354,10 @@ fn reject_unsupported_stable_ids( } ///| -fn resolve_projection_selector( +async fn resolve_projection_selector( projection : DocxProjection, input : String, -) -> DocxProjectionEntry raise CliFailure { +) -> DocxProjectionEntry { let selector = @lib.parse_selector(input) catch { error => raise selector_failure(error) } @@ -383,7 +383,9 @@ fn resolve_projection_selector( Some((story, kind, id)) => { let parent = "/docx/\{story}" let mut matches = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() if entry.parent == Some(parent) && entry.kind == kind && entry.stable_id == Some(id) { diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index 2ebf90e5..6215375f 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -413,11 +413,11 @@ fn checked_docx_human_output( } ///| -fn get_docx_payload( +async fn get_docx_payload( projection : DocxProjection, entry : DocxProjectionEntry, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { let children : Array[Json] = [] let fields = entry_base_json(entry, children) fields["schema"] = Json::string(@lib.SCHEMA_DOCX_ELEMENT) @@ -433,7 +433,9 @@ fn get_docx_payload( projection.warnings, max_output_chars, ) + projection.cooperate(work=docx_cli_projection_yield_elements) for path in entry.children { + projection.cooperate() let child = child_reference_json(projection, path) bounded.budget.reserve_array_item(child, !children.is_empty()) children.push(child) @@ -469,14 +471,14 @@ fn text_record_json(entry : DocxProjectionEntry, text : String) -> Json { } ///| -fn docx_text_payload( +async fn docx_text_payload( projection : DocxProjection, under : DocxProjectionEntry?, under_path : String?, offset : Int, limit : Int, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { let entries : Array[Json] = [] let fields : Map[String, Json] = { "schema": Json::string(@lib.SCHEMA_DOCX_TEXT), @@ -502,7 +504,9 @@ fn docx_text_payload( max_output_chars, ) let mut matched = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() let in_scope = match under { Some(root) => path_is_within(entry.path, root.path) None => true @@ -725,12 +729,9 @@ async fn docx_query_payload( ) let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 - let mut visited = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { - if (visited & 255) == 0 { - @async.pause() - } - visited += 1 + projection.cooperate() if !entry_matches_query(entry, query, scan_budget, work) { continue } @@ -870,15 +871,17 @@ fn reader_diagnostics( } ///| -fn docx_outline_payload( +async fn docx_outline_payload( projection : DocxProjection, max_output_chars? : Int = docx_cli_default_max_output_chars, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { ignore(checked_output_limit(max_output_chars)) let counts : Map[String, Int] = Map([]) let mut footnotes = 0 let mut endnotes = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() counts[entry.kind] = counts.get(entry.kind).unwrap_or(0) + 1 if entry.kind == "note" && entry.parent == Some("/docx/footnotes") { footnotes += 1 @@ -930,7 +933,9 @@ fn docx_outline_payload( projection.warnings, max_output_chars, ) + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() if entry.role is (StoryRoot | AnnotationCollection) { let record = Json::object({ "path": Json::string(entry.path), @@ -994,6 +999,7 @@ fn docx_outline_payload( } } for section in result.sections { + projection.cooperate() sections.push( bounded_section_json(section, bounded.budget, !sections.is_empty()), ) @@ -1135,18 +1141,20 @@ fn DocxHumanWriter::write_warnings( } ///| -fn docx_text_human( +async fn docx_text_human( projection : DocxProjection, under : DocxProjectionEntry?, offset : Int, limit : Int, warnings : Array[@lib.ProtocolWarning], maximum? : Int = docx_cli_hard_max_output_chars, -) -> String raise CliFailure { +) -> String { let output = DocxHumanWriter::new(maximum) let mut matched = 0 let mut returned = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() let in_scope = match under { Some(root) => path_is_within(entry.path, root.path) None => true @@ -1191,12 +1199,9 @@ async fn docx_query_human( let query = PreparedDocxQuery::build(spec, under, work) let mut matched = 0 let mut returned = 0 - let mut visited = 0 + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { - if (visited & 255) == 0 { - @async.pause() - } - visited += 1 + projection.cooperate() if !entry_matches_query(entry, query, scan_budget, work) { continue } @@ -1241,10 +1246,10 @@ fn docx_get_human( } ///| -fn docx_outline_human( +async fn docx_outline_human( projection : DocxProjection, maximum? : Int = docx_cli_hard_max_output_chars, -) -> String raise CliFailure { +) -> String { let counts : Map[String, Int] = Map([]) let output = DocxHumanWriter::new(maximum) output.begin_line() @@ -1252,7 +1257,9 @@ fn docx_outline_human( output.write_plain(raw_human_text(projection.file, 160)) output.begin_line() output.write_plain("scanned elements: \{projection.entries.length()}") + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() counts[entry.kind] = counts.get(entry.kind).unwrap_or(0) + 1 if entry.role is (StoryRoot | AnnotationCollection) { output.begin_line() @@ -1266,7 +1273,9 @@ fn docx_outline_human( output.write_plain( "paragraphs: \{counts.get("p").unwrap_or(0)}; tables: \{counts.get("tbl").unwrap_or(0)}; images: \{counts.get("image").unwrap_or(0)}; comments: \{counts.get("comment").unwrap_or(0)}; footnotes/endnotes: \{counts.get("note").unwrap_or(0)}", ) + projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { + projection.cooperate() match entry.element { Some(element) => match docx_heading_level(element) { diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index 8b21eeaa..8f70ba0d 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -52,11 +52,11 @@ fn docx_read_test_bytes() -> Bytes raise { } ///| -fn open_docx_projection( +async fn open_docx_projection( file : String, data : BytesView, max_elements : Int, -) -> DocxProjection raise CliFailure { +) -> DocxProjection { let archive = @zip.read_limited( data, max_package_bytes=office_read_max_package_bytes, @@ -71,12 +71,12 @@ fn open_docx_projection( } ///| -fn docx_read_test_projection() -> DocxProjection raise { +async fn docx_read_test_projection() -> DocxProjection { open_docx_projection("fixture.docx", docx_read_test_bytes(), 1000) } ///| -test "DOCX projection emits canonical round-tripping story paths" { +async test "DOCX projection emits canonical round-tripping story paths" { let projection = docx_read_test_projection() let paths = projection.entries.map(entry => entry.path) assert_true(paths.contains("/docx/body/p[1]")) @@ -105,7 +105,7 @@ test "DOCX projection emits canonical round-tripping story paths" { } ///| -test "DOCX outline reports semantic annotation counts" { +async test "DOCX outline reports semantic annotation counts" { let payload = docx_outline_payload(docx_read_test_projection()).data guard payload is Object(fields) && fields.get("counts") is Some(Object(counts)) else { @@ -117,7 +117,7 @@ test "DOCX outline reports semantic annotation counts" { } ///| -test "DOCX projection reports physical story sources and authority" { +async test "DOCX projection reports physical story sources and authority" { let projection = docx_read_test_projection() for expected in [ @@ -138,7 +138,7 @@ test "DOCX projection reports physical story sources and authority" { } ///| -test "DOCX annotation anchors expose canonical story selectors" { +async test "DOCX annotation anchors expose canonical story selectors" { let projection = docx_read_test_projection() let comment = resolve_projection_selector( projection, "/docx/comments/comment[id=\"0\"]", @@ -212,7 +212,7 @@ async test "DOCX capability declarations cover every payload field" { } ///| -test "DOCX get distinguishes stable items and positional descendants" { +async test "DOCX get distinguishes stable items and positional descendants" { let projection = docx_read_test_projection() let comment = resolve_projection_selector( projection, "/docx/comments/comment[id=\"0\"]", @@ -318,11 +318,16 @@ async test "DOCX exact queries cover the full supported XML token length" { let annotated = @docx.read_docx_annotated(docx_read_test_bytes()) let builder = DocxProjectionBuilder::new(100) let body = [@document.paragraph([@document.run([@document.text("x")])])] - let footnotes = annotation_story_plan("footnotes", "note", []) - let endnotes = annotation_story_plan("endnotes", "note", []) - let comments = annotation_story_plan("comments", "comment", [(long_id, body)]) + let footnotes = annotation_story_plan("footnotes", "note", [], builder) + let endnotes = annotation_story_plan("endnotes", "note", [], builder) + let comments = annotation_story_plan( + "comments", + "comment", + [(long_id, body)], + builder, + ) let annotation_index = annotation_projection_index( - annotated, footnotes, endnotes, comments, + annotated, footnotes, endnotes, comments, builder, ) builder.add_annotation_story(annotation_index, comments, None) let item = { ..builder.entries[1], properties: { "author": long_property } } @@ -446,7 +451,59 @@ async test "DOCX large exact predicates yield within one comparison" { } ///| -test "DOCX resolver reports malformed, missing, and unsupported stable selectors" { +async test "DOCX projection construction yields within its element ceiling" { + let body : Array[@document.DocumentElement] = [] + for index in 0..<2000 { + body.push( + @document.paragraph([ + @document.run([@document.text("paragraph \{index}")]), + ]), + ) + } + let builder = DocxProjectionBuilder::new(10_000) + @async.with_task_group() <| group => { + let task = group.spawn(() => { + builder.add_story("/docx/body", "body", body, Json::empty_object()) + }) + @async.pause() + assert_true(task.try_wait() is None) + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + +///| +async test "DOCX validation forwards parser cancellation callbacks" { + let archive = @zip.read_limited( + docx_read_test_bytes(), + max_package_bytes=office_read_max_package_bytes, + max_entries=office_read_max_zip_entries, + max_entry_uncompressed_bytes=office_read_max_zip_entry_bytes, + max_total_uncompressed_bytes=office_read_max_zip_total_bytes, + max_total_preserved_source_bytes=office_read_max_preserved_source_bytes, + ) + let mut checks = 0 + try + validate_docx_read_archive("fixture.docx", archive, cancelled=() => { + checks += 1 + checks >= 3 + }) + catch { + CliFailure(error) => assert_eq(error.code, "office.cancelled") + } noraise { + _ => fail("expected parser cancellation") + } + assert_true(checks >= 3) +} + +///| +async test "DOCX resolver reports malformed, missing, and unsupported stable selectors" { let projection = docx_read_test_projection() for case in [ @@ -457,6 +514,7 @@ test "DOCX resolver reports malformed, missing, and unsupported stable selectors let (path, expected) = case try resolve_projection_selector(projection, path) catch { CliFailure(error) => assert_eq(error.code, expected) + _ => fail("unexpected selector failure") } noraise { _ => fail("expected selector failure for \{path}") } @@ -466,24 +524,27 @@ test "DOCX resolver reports malformed, missing, and unsupported stable selectors catch { CliFailure(error) => assert_eq(error.code, "office.selector.unsupported_predicate") + _ => fail("unexpected malformed selector failure") } noraise { _ => fail("expected malformed selector failure") } } ///| -test "duplicate annotation ids stay positional and stable lookup is ambiguous" { +async test "duplicate annotation ids stay positional and stable lookup is ambiguous" { let annotated = @docx.read_docx_annotated(docx_read_test_bytes()) let builder = DocxProjectionBuilder::new(100) let body = [@document.paragraph([@document.run([@document.text("x")])])] - let footnotes = annotation_story_plan("footnotes", "note", []) - let endnotes = annotation_story_plan("endnotes", "note", []) - let comments = annotation_story_plan("comments", "comment", [ - ("dup", body), - ("dup", body), - ]) + let footnotes = annotation_story_plan("footnotes", "note", [], builder) + let endnotes = annotation_story_plan("endnotes", "note", [], builder) + let comments = annotation_story_plan( + "comments", + "comment", + [("dup", body), ("dup", body)], + builder, + ) let annotation_index = annotation_projection_index( - annotated, footnotes, endnotes, comments, + annotated, footnotes, endnotes, comments, builder, ) builder.add_annotation_story(annotation_index, comments, None) assert_eq(builder.entries[1].path, "/docx/comments/comment[1]") @@ -499,6 +560,8 @@ test "duplicate annotation ids stay positional and stable lookup is ambiguous" { entries: builder.entries, entry_index: builder.entry_index, warnings: builder.warnings, + cancelled: () => false, + work_since_yield: 0, } try resolve_projection_selector( @@ -507,13 +570,14 @@ test "duplicate annotation ids stay positional and stable lookup is ambiguous" { catch { CliFailure(error) => assert_eq(error.code, "office.docx.selector_ambiguous_id") + _ => fail("unexpected duplicate id failure") } noraise { _ => fail("expected duplicate id ambiguity") } } ///| -test "delimiter-rich unique annotation ids remain canonical and stable" { +async test "delimiter-rich unique annotation ids remain canonical and stable" { let id = "review/one]=\"ready\"" let path = stable_annotation_path("comments", "comment", id) assert_eq( @@ -525,9 +589,10 @@ test "delimiter-rich unique annotation ids remain canonical and stable" { } ///| -test "missing annotation ids remain empty and positional" { +async test "missing annotation ids remain empty and positional" { let body = [@document.paragraph([@document.run([@document.text("x")])])] - let plan = annotation_story_plan("comments", "comment", [("", body)]) + let builder = DocxProjectionBuilder::new(100) + let plan = annotation_story_plan("comments", "comment", [("", body)], builder) assert_eq(plan.paths, ["/docx/comments/comment[1]"]) assert_eq(plan.missing_ids, 1) assert_eq(plan.duplicate_ids, 0) @@ -536,27 +601,31 @@ test "missing annotation ids remain empty and positional" { } ///| -test "annotation metadata paths rewrite to emitted stable roots" { +async test "annotation metadata paths rewrite to emitted stable roots" { let annotated = @docx.read_docx_annotated(docx_read_test_bytes()) + let builder = DocxProjectionBuilder::new(1000) let footnotes = annotation_story_plan( "footnotes", "note", annotated.footnote_bodies(), + builder, ) let endnotes = annotation_story_plan( "endnotes", "note", annotated.endnote_bodies(), + builder, ) let comments = annotation_story_plan( "comments", "comment", annotated.comment_bodies(), + builder, ) let path_roots : @sorted_map.SortedMap[String, String] = SortedMap([]) - add_annotation_path_roots(path_roots, footnotes) - add_annotation_path_roots(path_roots, endnotes) - add_annotation_path_roots(path_roots, comments) + add_annotation_path_roots(path_roots, footnotes, builder) + add_annotation_path_roots(path_roots, endnotes, builder) + add_annotation_path_roots(path_roots, comments, builder) assert_eq( canonical_index_path("/comments/comment[1]/p[1]", path_roots), Some("/docx/comments/comment[id=\"0\"]/p[1]"), @@ -583,10 +652,11 @@ test "annotation metadata paths rewrite to emitted stable roots" { } ///| -test "projection and successful command output limits fail explicitly" { +async test "projection and successful command output limits fail explicitly" { let annotated = @docx.read_docx_annotated(docx_read_test_bytes()) try projection_from_annotated("small.docx", annotated, 2) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected projection failure") } noraise { _ => fail("expected projection resource limit") } @@ -604,7 +674,7 @@ test "projection and successful command output limits fail explicitly" { } ///| -test "successful command output accounting includes its trailing line feed" { +async test "successful command output accounting includes its trailing line feed" { assert_eq(checked_docx_human_output("abc", 4), "abc") try checked_docx_human_output("abc", 3) catch { CliFailure(error) => { @@ -625,7 +695,7 @@ test "successful command output accounting includes its trailing line feed" { } ///| -test "deep generated DOCX paths are classified as resource limits" { +async test "deep generated DOCX paths are classified as resource limits" { let mut nested = @word.paragraph([@word.run([@word.text("deep")])]) for _ in 0..<40 { nested = @word.table([@word.table_row([@word.table_cell([nested])])]) @@ -648,13 +718,14 @@ test "deep generated DOCX paths are classified as resource limits" { Some(Json::number(@lib.SELECTOR_MAX_DEPTH.to_double())), ) } + _ => fail("unexpected generated path failure") } noraise { _ => fail("expected deep generated selector paths to hit a resource limit") } } ///| -test "compact JSON preflight matches serializer character accounting" { +async test "compact JSON preflight matches serializer character accounting" { let value = Json::object({ "escaped": Json::string("quote \" slash \\ controls \b\t\n\f\r \u{0001}"), "unicode": Json::string("月🌙"), @@ -756,6 +827,7 @@ async test "DOCX output builders reject growth before returning oversized output } try docx_outline_payload(projection, max_output_chars=40) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected outline output failure") } noraise { _ => fail("expected bounded outline retention to fail") } @@ -765,11 +837,13 @@ async test "DOCX output builders reject growth before returning oversized output } try get_docx_payload(projection, wide, 1) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected get output failure") } noraise { _ => fail("expected get skeleton to fail before retaining child records") } try docx_text_payload(projection, None, None, 0, 10_000, 1) catch { CliFailure(error) => assert_eq(error.code, "office.docx.resource_limit") + _ => fail("unexpected text output failure") } noraise { _ => fail("expected text skeleton to fail before retaining records") } @@ -792,7 +866,7 @@ async test "DOCX output builders reject growth before returning oversized output } ///| -test "DOCX heading classification shares inspect semantics and bounds folding" { +async test "DOCX heading classification shares inspect semantics and bounds folding" { let paragraph = @word.paragraph( [@word.run([@word.text("not a heading")])], properties=@word.paragraph_properties( @@ -830,7 +904,7 @@ test "DOCX heading classification shares inspect semantics and bounds folding" { } ///| -test "DOCX projection lookup uses its canonical path index" { +async test "DOCX projection lookup uses its canonical path index" { let projection = docx_read_test_projection() for index, entry in projection.entries { assert_eq(projection.entry_index.get(entry.path), Some(index)) @@ -843,7 +917,7 @@ test "DOCX projection lookup uses its canonical path index" { } ///| -test "DOCX warning retention and deduplication sets share one hard cap" { +async test "DOCX warning retention and deduplication sets share one hard cap" { let builder = DocxProjectionBuilder::new(1) for index in 0..<1000 { builder.warn("office.docx.reader_warning", "diagnostic \{index}") @@ -861,7 +935,7 @@ test "DOCX warning retention and deduplication sets share one hard cap" { } ///| -test "DOCX preview truncation excludes discarded paragraph separators" { +async test "DOCX preview truncation excludes discarded paragraph separators" { let exact = @word.paragraph([@word.run([@word.text("x".repeat(159))])]) let (exact_text, exact_truncated) = collect_docx_text_prefix(exact, 160) assert_eq(exact_text, "x".repeat(159)) @@ -883,7 +957,7 @@ test "DOCX preview truncation excludes discarded paragraph separators" { } ///| -test "literal newlines consume scan budgets even when presentation trims them" { +async test "literal newlines consume scan budgets even when presentation trims them" { let newline_only = @word.paragraph([@word.run([@word.text("\n".repeat(161))])]) let (preview, truncated) = collect_docx_text_prefix(newline_only, 160) assert_eq(preview, "") @@ -908,7 +982,7 @@ test "literal newlines consume scan budgets even when presentation trims them" { } ///| -test "DOCX text budgets retain an independent per-element ceiling" { +async test "DOCX text budgets retain an independent per-element ceiling" { let projection = docx_read_test_projection() let oversized = { ..projection.entries[0], @@ -943,7 +1017,7 @@ test "DOCX text budgets retain an independent per-element ceiling" { } ///| -test "DOCX reader classifies only typed parser guard failures as resources" { +async test "DOCX reader classifies only typed parser guard failures as resources" { let CliFailure(resource) = docx_reader_failure( @docx_core.docx_xml_resource_limit_error(DocxXmlTokens), "fixture.docx", @@ -971,7 +1045,7 @@ test "DOCX reader classifies only typed parser guard failures as resources" { } ///| -test "DOCX OPC classification cannot be forged by an entry name" { +async test "DOCX OPC classification cannot be forged by an entry name" { let archive = try! @zip.read(docx_read_test_bytes()) archive.add("word/XML source budget exceeded.bin", b"ordinary") try validate_docx_read_archive("spoof.docx", archive) catch { @@ -982,7 +1056,7 @@ test "DOCX OPC classification cannot be forged by an entry name" { } ///| -test "DOCX structured reads accept explicit ZIP directory records" { +async test "DOCX structured reads accept explicit ZIP directory records" { let archive = try! @zip.read(docx_read_test_bytes()) archive.add("word/", b"") archive.add("word/media/", b"") @@ -995,7 +1069,7 @@ test "DOCX structured reads accept explicit ZIP directory records" { } ///| -test "DOCX structured reads warn and continue on broken section references" { +async test "DOCX structured reads warn and continue on broken section references" { let archive = try! @zip.read(docx_read_test_bytes()) guard archive.get("word/document.xml") is Some(document_bytes) else { fail("fixture has no main document part") @@ -1021,7 +1095,7 @@ test "DOCX structured reads warn and continue on broken section references" { } ///| -test "DOCX preflight rejects foreign-namespace officeDocument relationships" { +async test "DOCX preflight rejects foreign-namespace officeDocument relationships" { let archive = try! @zip.read(docx_read_test_bytes()) let root_relationships = #| From 19622679d81d3ed582a33592d02db1cee634a17d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 13:08:42 +0800 Subject: [PATCH 055/150] fix(xlsx): honor 1904 dates in formula calculation --- xlsx/calc_1904_date_test.mbt | 65 +++ xlsx/formula_builtins.mbt | 252 +++++++---- xlsx/formula_builtins_financial.mbt | 630 ++++++++++++++++++++-------- xlsx/formula_eval.mbt | 168 ++++++-- xlsx/formula_eval_types.mbt | 1 + xlsx/formula_parse.mbt | 3 +- xlsx/workbook.mbt | 4 +- 7 files changed, 837 insertions(+), 286 deletions(-) create mode 100644 xlsx/calc_1904_date_test.mbt diff --git a/xlsx/calc_1904_date_test.mbt b/xlsx/calc_1904_date_test.mbt new file mode 100644 index 00000000..e6860bf9 --- /dev/null +++ b/xlsx/calc_1904_date_test.mbt @@ -0,0 +1,65 @@ +///| +test "calculated dates honor the workbook 1904 date system" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Sheet1") + workbook.set_workbook_props( + Some(@xlsx.WorkbookPropsOptions::with_values(date_1904=true)), + ) + let date_style = workbook.add_style(@xlsx.Style::number_format("yyyy-mm-dd")) + sheet.set_cell_formula("A1", "DATE(1904,1,1)") + workbook.set_cell_style("Sheet1", "A1", date_style) + inspect(workbook.calc_cell_value("Sheet1", "A1", raw=true), content="0") + inspect(workbook.calc_cell_value("Sheet1", "A1"), content="1904-01-01") + debug_inspect( + workbook.calc_cell_typed("Sheet1", "A1"), + content="Some(Numeric(0))", + ) + + let cases : Array[(String, String)] = [ + ("DATEVALUE(\"1904-01-02\")", "1"), + ("TEXT(DATE(1904,1,1),\"yyyy-mm-dd\")", "1904-01-01"), + ("YEAR(0)", "1904"), + ("MONTH(0)", "1"), + ("DAY(0)", "1"), + ("TEXT(EDATE(DATE(1904,1,31),1),\"yyyy-mm-dd\")", "1904-02-29"), + ("DAYS(DATE(1904,1,3),DATE(1904,1,1))", "2"), + ("NETWORKDAYS(DATE(1904,1,1),DATE(1904,1,4))", "2"), + ("TEXT(WORKDAY(DATE(1904,1,1),1),\"yyyy-mm-dd\")", "1904-01-04"), + ("INT(NOW())=TODAY()", "TRUE"), + ("TODAY()=DATE(YEAR(TODAY()),MONTH(TODAY()),DAY(TODAY()))", "TRUE"), + ("DATE(1903,12,31)", "#NUM!"), + ] + for index, case in cases { + let (formula, expected) = case + let cell = "B\{index + 1}" + sheet.set_cell_formula(cell, formula) + inspect(workbook.calc_cell_value("Sheet1", cell), content=expected) + } +} + +///| +test "date-aware financial formulas are invariant across date systems" { + let workbook_1900 = @xlsx.Workbook::new() + let sheet_1900 = workbook_1900.add_sheet("Sheet1") + let workbook_1904 = @xlsx.Workbook::new() + let sheet_1904 = workbook_1904.add_sheet("Sheet1") + workbook_1904.set_workbook_props( + Some(@xlsx.WorkbookPropsOptions::with_values(date_1904=true)), + ) + let formulas : Array[String] = [ + "ACCRINT(DATE(2020,1,1),DATE(2020,7,1),DATE(2020,3,1),0.05,1000,2,0)", "AMORLINC(1000,DATE(2020,1,1),DATE(2020,12,31),100,1,0.1,0)", + "COUPDAYS(DATE(2020,1,15),DATE(2021,1,15),2,0)", "DURATION(DATE(2020,1,15),DATE(2021,1,15),0.05,0.04,2,0)", + "PRICE(DATE(2020,1,15),DATE(2021,1,15),0.05,0.04,100,2,0)", "ODDFPRICE(DATE(2017,2,1),DATE(2021,3,31),DATE(2016,12,1),DATE(2017,3,31),5.5%,3.5%,100,2)", + "ODDLPRICE(DATE(2008,4,20),DATE(2008,6,15),DATE(2007,12,24),3.75%,99.875,100,2)", + "TEXT(COUPNCD(DATE(2020,1,15),DATE(2021,1,15),2,0),\"yyyy-mm-dd\")", + ] + for index, formula in formulas { + let cell = "A\{index + 1}" + sheet_1900.set_cell_formula(cell, formula) + sheet_1904.set_cell_formula(cell, formula) + assert_eq( + workbook_1904.calc_cell_value("Sheet1", cell, raw=true), + workbook_1900.calc_cell_value("Sheet1", cell, raw=true), + ) + } +} diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index 83b54165..b4fe4dcc 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -1495,27 +1495,29 @@ fn eval_function( "FTEST" => ftest_values(workbook, sheet_name, args, values, ctx) "FdotTEST" | "F.TEST" | "_XLFN.F.TEST" => ftest_values(workbook, sheet_name, args, values, ctx) - "ACCRINT" => accrint_values(values) - "ACCRINTM" => accrintm_values(values) - "AMORDEGRC" => amordegrc_values(values) - "AMORLINC" => amorlinc_values(values) - "COUPDAYBS" => coupdaybs_values(values) - "COUPDAYS" => coupdays_values(values) - "COUPDAYSNC" => coupdaysnc_values(values) - "COUPNCD" => coupncd_values(values) - "COUPNUM" => coupnum_values(values) - "COUPPCD" => couppcd_values(values) + "ACCRINT" => accrint_values(values, use_1904_dates=ctx.use_1904_dates) + "ACCRINTM" => accrintm_values(values, use_1904_dates=ctx.use_1904_dates) + "AMORDEGRC" => amordegrc_values(values, use_1904_dates=ctx.use_1904_dates) + "AMORLINC" => amorlinc_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPDAYBS" => coupdaybs_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPDAYS" => coupdays_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPDAYSNC" => coupdaysnc_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPNCD" => coupncd_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPNUM" => coupnum_values(values, use_1904_dates=ctx.use_1904_dates) + "COUPPCD" => couppcd_values(values, use_1904_dates=ctx.use_1904_dates) "CUMIPMT" => cumip_values("CUMIPMT", values) "CUMPRINC" => cumip_values("CUMPRINC", values) - "DURATION" => duration_values(values) + "DURATION" => duration_values(values, use_1904_dates=ctx.use_1904_dates) "DB" => db_values(values) "DDB" => ddb_values(values) "FV" => fv_values(values) "FVSCHEDULE" => fvschedule_values(values) - "DISC" => disc_intrate_values("DISC", values) - "INTRATE" => disc_intrate_values("INTRATE", values) + "DISC" => + disc_intrate_values("DISC", values, use_1904_dates=ctx.use_1904_dates) + "INTRATE" => + disc_intrate_values("INTRATE", values, use_1904_dates=ctx.use_1904_dates) "IRR" => irr_values(values) - "MDURATION" => mduration_values(values) + "MDURATION" => mduration_values(values, use_1904_dates=ctx.use_1904_dates) "MIRR" => mirr_values(values) "DOLLAR" => dollar_values(values) "DOLLARDE" => dollar_fraction_values("DOLLARDE", values) @@ -1528,30 +1530,34 @@ fn eval_function( "NOMINAL" => nominal_values(values) "NPER" => nper_values(values) "NPV" => npv_values(values) - "ODDFPRICE" => oddfprice_values(values) - "ODDFYIELD" => oddfyield_values(values) - "ODDLPRICE" => oddl_values("ODDLPRICE", values) - "ODDLYIELD" => oddl_values("ODDLYIELD", values) + "ODDFPRICE" => oddfprice_values(values, use_1904_dates=ctx.use_1904_dates) + "ODDFYIELD" => oddfyield_values(values, use_1904_dates=ctx.use_1904_dates) + "ODDLPRICE" => + oddl_values("ODDLPRICE", values, use_1904_dates=ctx.use_1904_dates) + "ODDLYIELD" => + oddl_values("ODDLYIELD", values, use_1904_dates=ctx.use_1904_dates) "PDURATION" => pduration_values(values) "PMT" => pmt_values(values) "PPMT" => ipmt_values("PPMT", values) - "PRICE" => price_yield_values("PRICE", values) - "PRICEDISC" => pricedisc_values(values) - "PRICEMAT" => pricemat_values(values) + "PRICE" => + price_yield_values("PRICE", values, use_1904_dates=ctx.use_1904_dates) + "PRICEDISC" => pricedisc_values(values, use_1904_dates=ctx.use_1904_dates) + "PRICEMAT" => pricemat_values(values, use_1904_dates=ctx.use_1904_dates) "PV" => pv_values(values) "RATE" => rate_values(values) - "RECEIVED" => received_values(values) + "RECEIVED" => received_values(values, use_1904_dates=ctx.use_1904_dates) "RRI" => rri_values(values) "SLN" => sln_values(values) "SYD" => syd_values(values) - "TBILLEQ" => tbilleq_values(values) - "TBILLPRICE" => tbillprice_values(values) - "TBILLYIELD" => tbillyield_values(values) + "TBILLEQ" => tbilleq_values(values, use_1904_dates=ctx.use_1904_dates) + "TBILLPRICE" => tbillprice_values(values, use_1904_dates=ctx.use_1904_dates) + "TBILLYIELD" => tbillyield_values(values, use_1904_dates=ctx.use_1904_dates) "XIRR" => xirr_values(values) "XNPV" => xnpv_values(values) - "YIELD" => price_yield_values("YIELD", values) - "YIELDDISC" => yielddisc_values(values) - "YIELDMAT" => yieldmat_values(values) + "YIELD" => + price_yield_values("YIELD", values, use_1904_dates=ctx.use_1904_dates) + "YIELDDISC" => yielddisc_values(values, use_1904_dates=ctx.use_1904_dates) + "YIELDMAT" => yieldmat_values(values, use_1904_dates=ctx.use_1904_dates) "VDB" => vdb_values(values) "TdotDIST" | "T.DIST" | "_XLFN.T.DIST" => tdist_values(values) "TdotDISTdot2T" | "T.DIST.2T" | "_XLFN.T.DIST.2T" => tdist_2t_values(values) @@ -3351,6 +3357,7 @@ fn eval_function( format_number(num), format_text, Options::new(), + use_1904_format=ctx.use_1904_dates, ), ) Err(err) => err @@ -3590,7 +3597,13 @@ fn eval_function( value_as_int(values[2]), ) { (Ok(year), Ok(month), Ok(day)) => - match excel_serial_from_date(year, month, day) { + match + excel_serial_from_date( + year, + month, + day, + use_1904_dates=ctx.use_1904_dates, + ) { Some(serial) => Number(serial) None => Error(formula_error_num) } @@ -3629,7 +3642,13 @@ fn eval_function( } match parse_date_parts(text) { Some((year, month, day)) => - match excel_serial_from_date(year, month, day) { + match + excel_serial_from_date( + year, + month, + day, + use_1904_dates=ctx.use_1904_dates, + ) { Some(serial) => Number(serial) None => Error(formula_error_value) } @@ -3655,11 +3674,13 @@ fn eval_function( } "DATEDIF" => if values.length() == 3 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let end_serial = match value_as_date_serial(values[1]) { + let end_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -3673,8 +3694,14 @@ fn eval_function( Err(err) => return err } // value_as_date_serial rejects negatives; serial-to-date conversion is total on non-negative serials. - let (sy, sm, sd) = date_parts_from_serial(start_serial).unwrap() - let (ey, em, ed) = date_parts_from_serial(end_serial).unwrap() + let (sy, sm, sd) = date_parts_from_serial( + start_serial, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() + let (ey, em, ed) = date_parts_from_serial( + end_serial, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() match unit { "y" => { let mut diff = ey - sy @@ -3706,8 +3733,18 @@ fn eval_function( if em < sm || (em == sm && ed < sd) { target_year = sy + 1 } - let start_base = excel_serial_from_date(sy, sm, sd).unwrap() - let end_base = excel_serial_from_date(target_year, em, ed).unwrap() + let start_base = excel_serial_from_date( + sy, + sm, + sd, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() + let end_base = excel_serial_from_date( + target_year, + em, + ed, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() Number(end_base - start_base) } "md" => { @@ -3728,11 +3765,13 @@ fn eval_function( } "DAYS" => if values.length() == 2 { - let end_serial = match value_as_date_serial(values[0]) { + let end_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let start_serial = match value_as_date_serial(values[1]) { + let start_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -3742,16 +3781,24 @@ fn eval_function( } "DAYS360" => if values.length() == 2 || values.length() == 3 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let end_serial = match value_as_date_serial(values[1]) { + let end_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let (sy, sm, sd) = date_parts_from_serial(start_serial).unwrap() - let (ey, em, ed) = date_parts_from_serial(end_serial).unwrap() + let (sy, sm, sd) = date_parts_from_serial( + start_serial, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() + let (ey, em, ed) = date_parts_from_serial( + end_serial, + use_1904_dates=ctx.use_1904_dates, + ).unwrap() let mut start_day = sd let mut end_day = ed let end_year = ey @@ -3794,11 +3841,13 @@ fn eval_function( } "ISOWEEKNUM" => if values.length() == 1 { - let (year, month, day) = match date_parts_from_value(values[0]) { + let (year, month, day) = match + date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(parts) => parts Err(err) => return err } - match iso_week_number(year, month, day) { + match + iso_week_number(year, month, day, use_1904_dates=ctx.use_1904_dates) { Some(week) => Number(Double::from_int(week)) None => Error(formula_error_num) } @@ -3807,7 +3856,8 @@ fn eval_function( } "EDATE" => if values.length() == 2 { - let (year, month, day) = match date_parts_from_value(values[0]) { + let (year, month, day) = match + date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(parts) => parts Err(err) => return err } @@ -3818,7 +3868,13 @@ fn eval_function( let (new_year, new_month) = normalize_year_month(year, month + months) let max_day = days_in_month(new_year, new_month) let new_day = if day > max_day { max_day } else { day } - match excel_serial_from_date(new_year, new_month, new_day) { + match + excel_serial_from_date( + new_year, + new_month, + new_day, + use_1904_dates=ctx.use_1904_dates, + ) { Some(serial) => Number(serial) None => Error(formula_error_num) } @@ -3827,7 +3883,8 @@ fn eval_function( } "EOMONTH" => if values.length() == 2 { - let (year, month, _) = match date_parts_from_value(values[0]) { + let (year, month, _) = match + date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(parts) => parts Err(err) => return err } @@ -3837,7 +3894,13 @@ fn eval_function( } let (new_year, new_month) = normalize_year_month(year, month + months) let new_day = days_in_month(new_year, new_month) - match excel_serial_from_date(new_year, new_month, new_day) { + match + excel_serial_from_date( + new_year, + new_month, + new_day, + use_1904_dates=ctx.use_1904_dates, + ) { Some(serial) => Number(serial) None => Error(formula_error_num) } @@ -3846,11 +3909,13 @@ fn eval_function( } "YEARFRAC" => if values.length() == 2 || values.length() == 3 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let end_serial = match value_as_date_serial(values[1]) { + let end_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -3862,13 +3927,19 @@ fn eval_function( } else { 0 } - yearfrac_value(start_serial, end_serial, basis) + yearfrac_value( + start_serial, + end_serial, + basis, + use_1904_dates=ctx.use_1904_dates, + ) } else { Error(formula_error_value) } "WEEKDAY" => if values.length() == 1 || values.length() == 2 { - let (year, month, day) = match date_parts_from_value(values[0]) { + let (year, month, day) = match + date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(parts) => parts Err(err) => return err } @@ -3899,7 +3970,8 @@ fn eval_function( } "WEEKNUM" => if values.length() == 1 || values.length() == 2 { - let (year, month, day) = match date_parts_from_value(values[0]) { + let (year, month, day) = match + date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(parts) => parts Err(err) => return err } @@ -3946,30 +4018,40 @@ fn eval_function( } "NETWORKDAYS" => if values.length() == 2 || values.length() == 3 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let end_serial = match value_as_date_serial(values[1]) { + let end_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } let holidays = if values.length() == 3 { - collect_holidays(values[2]) + collect_holidays(values[2], use_1904_dates=ctx.use_1904_dates) } else { [] } - networkdays_intl_value(start_serial, end_serial, Number(1.0), holidays) + networkdays_intl_value( + start_serial, + end_serial, + Number(1.0), + holidays, + use_1904_dates=ctx.use_1904_dates, + ) } else { Error(formula_error_value) } "NETWORKDAYS.INTL" => if values.length() >= 2 && values.length() <= 4 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } - let end_serial = match value_as_date_serial(values[1]) { + let end_serial = match + value_as_date_serial(values[1], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -3979,19 +4061,24 @@ fn eval_function( Number(1.0) } let holidays = if values.length() == 4 { - collect_holidays(values[3]) + collect_holidays(values[3], use_1904_dates=ctx.use_1904_dates) } else { [] } networkdays_intl_value( - start_serial, end_serial, weekend_value, holidays, + start_serial, + end_serial, + weekend_value, + holidays, + use_1904_dates=ctx.use_1904_dates, ) } else { Error(formula_error_value) } "WORKDAY" => if values.length() == 2 || values.length() == 3 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -4000,17 +4087,24 @@ fn eval_function( Err(err) => return err } let holidays = if values.length() == 3 { - collect_holidays(values[2]) + collect_holidays(values[2], use_1904_dates=ctx.use_1904_dates) } else { [] } - workday_intl_value(start_serial, days, Number(1.0), holidays) + workday_intl_value( + start_serial, + days, + Number(1.0), + holidays, + use_1904_dates=ctx.use_1904_dates, + ) } else { Error(formula_error_value) } "WORKDAY.INTL" => if values.length() >= 2 && values.length() <= 4 { - let start_serial = match value_as_date_serial(values[0]) { + let start_serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { Ok(value) => value Err(err) => return err } @@ -4024,11 +4118,17 @@ fn eval_function( Number(1.0) } let holidays = if values.length() == 4 { - collect_holidays(values[3]) + collect_holidays(values[3], use_1904_dates=ctx.use_1904_dates) } else { [] } - workday_intl_value(start_serial, days, weekend_value, holidays) + workday_intl_value( + start_serial, + days, + weekend_value, + holidays, + use_1904_dates=ctx.use_1904_dates, + ) } else { Error(formula_error_value) } @@ -4038,7 +4138,8 @@ fn eval_function( let total_seconds = UInt64::to_double(now_ms) / 1000.0 let days = Double::floor(total_seconds / 86400.0) let seconds_in_day = total_seconds - days * 86400.0 - let serial = days + 25569.0 + seconds_in_day / 86400.0 + let epoch_serial = if ctx.use_1904_dates { 24107.0 } else { 25569.0 } + let serial = days + epoch_serial + seconds_in_day / 86400.0 Number(serial) } else { Error(formula_error_value) @@ -4048,13 +4149,14 @@ fn eval_function( let now_ms = @env.now() let total_seconds = UInt64::to_double(now_ms) / 1000.0 let days = Double::floor(total_seconds / 86400.0) - Number(days + 25569.0) + let epoch_serial = if ctx.use_1904_dates { 24107.0 } else { 25569.0 } + Number(days + epoch_serial) } else { Error(formula_error_value) } "YEAR" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((year, _, _, _, _, _)) => Number(Double::from_int(year)) Err(err) => err } @@ -4063,7 +4165,7 @@ fn eval_function( } "MONTH" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((_, month, _, _, _, _)) => Number(Double::from_int(month)) Err(err) => err } @@ -4072,7 +4174,7 @@ fn eval_function( } "DAY" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((_, _, day, _, _, _)) => Number(Double::from_int(day)) Err(err) => err } @@ -4081,7 +4183,7 @@ fn eval_function( } "HOUR" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((_, _, _, hour, _, _)) => Number(Double::from_int(hour)) Err(err) => err } @@ -4090,7 +4192,7 @@ fn eval_function( } "MINUTE" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((_, _, _, _, minute, _)) => Number(Double::from_int(minute)) Err(err) => err } @@ -4099,7 +4201,7 @@ fn eval_function( } "SECOND" => if values.length() == 1 { - match excel_serial_parts(values[0]) { + match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { Ok((_, _, _, _, _, second)) => Number(Double::from_int(second)) Err(err) => err } diff --git a/xlsx/formula_builtins_financial.mbt b/xlsx/formula_builtins_financial.mbt index b351f77f..3e8807c9 100644 --- a/xlsx/formula_builtins_financial.mbt +++ b/xlsx/formula_builtins_financial.mbt @@ -63,14 +63,15 @@ fn coupdays_between( from_serial : Double, to_serial : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Double { let (from_year, from_month, from_day_raw) = match - date_parts_from_serial(from_serial) { + date_parts_from_serial(from_serial, use_1904_dates~) { Some(parts) => parts None => return 0.0 } let (to_year, to_month, to_day_raw) = match - date_parts_from_serial(to_serial) { + date_parts_from_serial(to_serial, use_1904_dates~) { Some(parts) => parts None => return 0.0 } @@ -80,11 +81,12 @@ fn coupdays_between( let mut to_day = get_day_on_basis(to_year, to_month, to_day_raw, basis) if !is_30_basis_method(basis) { let from_adj = match - excel_serial_from_date(from_year, from_month, from_day) { + excel_serial_from_date(from_year, from_month, from_day, use_1904_dates~) { Some(serial) => serial None => return 0.0 } - let to_adj = match excel_serial_from_date(to_year, to_month, to_day) { + let to_adj = match + excel_serial_from_date(to_year, to_month, to_day, use_1904_dates~) { Some(serial) => serial None => return 0.0 } @@ -133,14 +135,15 @@ fn coupon_date_serial( settlement_serial : Double, maturity_serial : Double, frequency : Double, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { let (settle_year, settle_month, settle_day) = match - date_parts_from_serial(settlement_serial) { + date_parts_from_serial(settlement_serial, use_1904_dates~) { Some(parts) => parts None => return Err(Error(formula_error_value)) } let (mat_year, mat_month, mat_day) = match - date_parts_from_serial(maturity_serial) { + date_parts_from_serial(maturity_serial, use_1904_dates~) { Some(parts) => parts None => return Err(Error(formula_error_value)) } @@ -168,7 +171,7 @@ fn coupon_date_serial( } else if day > 27 && day > days { day = days } - match excel_serial_from_date(year, month, day) { + match excel_serial_from_date(year, month, day, use_1904_dates~) { Some(serial) => Ok(serial) None => Err(Error(formula_error_num)) } @@ -177,15 +180,16 @@ fn coupon_date_serial( ///| fn prepare_coupon_args( values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> Result[(Double, Double, Double, Int), FormulaValue] { if values.length() != 3 && values.length() != 4 { return Err(Error(formula_error_value)) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -216,13 +220,20 @@ fn coupdaybs_value( maturity : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { let pcd = match - coupon_date_serial("COUPPCD", settlement, maturity, frequency) { + coupon_date_serial( + "COUPPCD", + settlement, + maturity, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return Err(err) } - Ok(coupdays_between(pcd, settlement, basis)) + Ok(coupdays_between(pcd, settlement, basis, use_1904_dates~)) } ///| @@ -231,29 +242,37 @@ fn coupdays_value( maturity : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { if basis == 1 { let pcd = match - coupon_date_serial("COUPPCD", settlement, maturity, frequency) { + coupon_date_serial( + "COUPPCD", + settlement, + maturity, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return Err(err) } let months = 12 / Double::to_int(trunc_double(frequency)) - let next = match date_parts_from_serial(pcd) { + let next = match date_parts_from_serial(pcd, use_1904_dates~) { Some((year, month, day)) => { let (new_year, new_month, new_day) = normalize_date_parts( year, month + months, day, ) - match excel_serial_from_date(new_year, new_month, new_day) { + match + excel_serial_from_date(new_year, new_month, new_day, use_1904_dates~) { Some(serial) => serial None => return Err(Error(formula_error_num)) } } None => return Err(Error(formula_error_value)) } - return Ok(coupdays_between(pcd, next, basis)) + return Ok(coupdays_between(pcd, next, basis, use_1904_dates~)) } let year_days = get_year_days(0, basis) Ok(Double::from_int(year_days) / frequency) @@ -265,13 +284,20 @@ fn coupdaysnc_value( maturity : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { let ncd = match - coupon_date_serial("COUPNCD", settlement, maturity, frequency) { + coupon_date_serial( + "COUPNCD", + settlement, + maturity, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return Err(err) } - Ok(coupdays_between(settlement, ncd, basis)) + Ok(coupdays_between(settlement, ncd, basis, use_1904_dates~)) } ///| @@ -279,8 +305,9 @@ fn coupnum_value( settlement : Double, maturity : Double, frequency : Double, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { - match yearfrac_value(settlement, maturity, 0) { + match yearfrac_value(settlement, maturity, 0, use_1904_dates~) { Number(num) => Ok(Double::ceil(num * frequency)) Error(err) => Err(Error(err)) _ => Err(Error(formula_error_value)) @@ -292,8 +319,9 @@ fn yearfrac_number( start_serial : Double, end_serial : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { - match yearfrac_value(start_serial, end_serial, basis) { + match yearfrac_value(start_serial, end_serial, basis, use_1904_dates~) { Number(num) => Ok(num) Error(err) => Err(Error(err)) _ => Err(Error(formula_error_value)) @@ -310,7 +338,10 @@ fn round_half_up(value : Double) -> Double { } ///| -fn amor_date_serial(value : FormulaValue) -> Result[Double, FormulaValue] { +fn amor_date_serial( + value : FormulaValue, + use_1904_dates? : Bool = false, +) -> Result[Double, FormulaValue] { let normalized = normalize_scalar(value) match normalized { String(text) => { @@ -318,10 +349,10 @@ fn amor_date_serial(value : FormulaValue) -> Result[Double, FormulaValue] { if trimmed == "" { Err(Error(formula_error_value)) } else { - value_as_date_serial(String(trimmed)) + value_as_date_serial(String(trimmed), use_1904_dates~) } } - _ => value_as_date_serial(normalized) + _ => value_as_date_serial(normalized, use_1904_dates~) } } @@ -339,6 +370,7 @@ priv struct AmorArgs { ///| fn prepare_amor_args( values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> Result[AmorArgs, FormulaValue] { if values.length() != 6 && values.length() != 7 { return Err(Error(formula_error_value)) @@ -350,11 +382,11 @@ fn prepare_amor_args( if cost < 0.0 { return Err(Error(formula_error_value)) } - let date_purchased = match amor_date_serial(values[1]) { + let date_purchased = match amor_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let first_period = match amor_date_serial(values[2]) { + let first_period = match amor_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -402,8 +434,11 @@ fn prepare_amor_args( } ///| -fn amordegrc_values(values : ArrayView[FormulaValue]) -> FormulaValue { - let args = match prepare_amor_args(values) { +fn amordegrc_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { + let args = match prepare_amor_args(values, use_1904_dates~) { Ok(value) => value Err(err) => return err } @@ -421,7 +456,12 @@ fn amordegrc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } let rate = args.rate * amor_coeff let frac = match - yearfrac_number(args.date_purchased, args.first_period, args.basis) { + yearfrac_number( + args.date_purchased, + args.first_period, + args.basis, + use_1904_dates~, + ) { Ok(num) => num Err(err) => return err } @@ -450,13 +490,21 @@ fn amordegrc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn amorlinc_values(values : ArrayView[FormulaValue]) -> FormulaValue { - let args = match prepare_amor_args(values) { +fn amorlinc_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { + let args = match prepare_amor_args(values, use_1904_dates~) { Ok(value) => value Err(err) => return err } let frac = match - yearfrac_number(args.date_purchased, args.first_period, args.basis) { + yearfrac_number( + args.date_purchased, + args.first_period, + args.basis, + use_1904_dates~, + ) { Ok(num) => num Err(err) => return err } @@ -2398,15 +2446,18 @@ fn pduration_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn accrint_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn accrint_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() < 6 || values.length() > 8 { return Error(formula_error_value) } - let issue = match value_as_date_serial(values[0]) { + let issue = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let settlement = match value_as_date_serial(values[2]) { + let settlement = match value_as_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2439,7 +2490,7 @@ fn accrint_values(values : ArrayView[FormulaValue]) -> FormulaValue { Err(err) => return err } } - let frac = match yearfrac_number(issue, settlement, basis) { + let frac = match yearfrac_number(issue, settlement, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2448,15 +2499,18 @@ fn accrint_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn accrintm_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn accrintm_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 4 && values.length() != 5 { return Error(formula_error_value) } - let issue = match value_as_date_serial(values[0]) { + let issue = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let settlement = match value_as_date_serial(values[1]) { + let settlement = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2482,7 +2536,7 @@ fn accrintm_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let frac = match yearfrac_number(issue, settlement, basis) { + let frac = match yearfrac_number(issue, settlement, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2491,13 +2545,17 @@ fn accrintm_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn coupdaybs_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn coupdaybs_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } - let result = match coupdaybs_value(settlement, maturity, frequency, basis) { + let result = match + coupdaybs_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2505,13 +2563,17 @@ fn coupdaybs_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn coupdays_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn coupdays_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } - let result = match coupdays_value(settlement, maturity, frequency, basis) { + let result = match + coupdays_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2519,13 +2581,17 @@ fn coupdays_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn coupdaysnc_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn coupdaysnc_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } - let result = match coupdaysnc_value(settlement, maturity, frequency, basis) { + let result = match + coupdaysnc_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2533,14 +2599,23 @@ fn coupdaysnc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn coupncd_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn coupncd_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, _basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } let result = match - coupon_date_serial("COUPNCD", settlement, maturity, frequency) { + coupon_date_serial( + "COUPNCD", + settlement, + maturity, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return err } @@ -2548,13 +2623,17 @@ fn coupncd_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn coupnum_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn coupnum_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, _basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } - let result = match coupnum_value(settlement, maturity, frequency) { + let result = match + coupnum_value(settlement, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2562,14 +2641,23 @@ fn coupnum_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn couppcd_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn couppcd_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, frequency, _basis) = match - prepare_coupon_args(values) { + prepare_coupon_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } let result = match - coupon_date_serial("COUPPCD", settlement, maturity, frequency) { + coupon_date_serial( + "COUPPCD", + settlement, + maturity, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return err } @@ -2580,15 +2668,16 @@ fn couppcd_values(values : ArrayView[FormulaValue]) -> FormulaValue { fn disc_intrate_values( name : String, values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> FormulaValue { if values.length() != 4 && values.length() != 5 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2617,7 +2706,8 @@ fn disc_intrate_values( } else { 0 } - let frac = match yearfrac_number(settlement, maturity, basis) { + let frac = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2630,15 +2720,18 @@ fn disc_intrate_values( } ///| -fn received_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn received_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 4 && values.length() != 5 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2664,7 +2757,8 @@ fn received_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let frac = match yearfrac_number(settlement, maturity, basis) { + let frac = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2675,15 +2769,16 @@ fn received_values(values : ArrayView[FormulaValue]) -> FormulaValue { ///| fn prepare_duration_args( values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> Result[(Double, Double, Double, Double, Double, Int), FormulaValue] { if values.length() != 5 && values.length() != 6 { return Err(Error(formula_error_value)) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -2730,12 +2825,15 @@ fn duration_calc( yld : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { - let frac = match yearfrac_number(settlement, maturity, basis) { + let frac = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let coups = match coupnum_value(settlement, maturity, frequency) { + let coups = match + coupnum_value(settlement, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -2762,14 +2860,25 @@ fn duration_calc( } ///| -fn duration_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn duration_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, coupon, yld, frequency, basis) = match - prepare_duration_args(values) { + prepare_duration_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } let result = match - duration_calc(settlement, maturity, coupon, yld, frequency, basis) { + duration_calc( + settlement, + maturity, + coupon, + yld, + frequency, + basis, + use_1904_dates~, + ) { Ok(num) => num Err(err) => return err } @@ -2777,14 +2886,25 @@ fn duration_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn mduration_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn mduration_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { let (settlement, maturity, coupon, yld, frequency, basis) = match - prepare_duration_args(values) { + prepare_duration_args(values, use_1904_dates~) { Ok(args) => args Err(err) => return err } let duration = match - duration_calc(settlement, maturity, coupon, yld, frequency, basis) { + duration_calc( + settlement, + maturity, + coupon, + yld, + frequency, + basis, + use_1904_dates~, + ) { Ok(num) => num Err(err) => return err } @@ -2801,6 +2921,7 @@ fn price_value( redemption : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { if settlement >= maturity { return Err(Error(formula_error_num)) @@ -2808,19 +2929,23 @@ fn price_value( if basis < 0 || basis > 4 { return Err(Error(formula_error_num)) } - let e = match coupdays_value(settlement, maturity, frequency, basis) { + let e = match + coupdays_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let dsc = match coupdaysnc_value(settlement, maturity, frequency, basis) { + let dsc = match + coupdaysnc_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num / e Err(err) => return Err(err) } - let n = match coupnum_value(settlement, maturity, frequency) { + let n = match + coupnum_value(settlement, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let a = match coupdaybs_value(settlement, maturity, frequency, basis) { + let a = match + coupdaybs_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -2857,6 +2982,7 @@ fn yield_value( redemption : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> FormulaValue { if settlement >= maturity { return Error(formula_error_num) @@ -2865,14 +2991,28 @@ fn yield_value( let mut yield2 = 1.0 let mut price1 = match price_value( - settlement, maturity, rate, yield1, redemption, frequency, basis, + settlement, + maturity, + rate, + yield1, + redemption, + frequency, + basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err } let mut price2 = match price_value( - settlement, maturity, rate, yield2, redemption, frequency, basis, + settlement, + maturity, + rate, + yield2, + redemption, + frequency, + basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err @@ -2883,7 +3023,14 @@ fn yield_value( while iter < 100 && price_n != pr { price_n = match price_value( - settlement, maturity, rate, yield_n, redemption, frequency, basis, + settlement, + maturity, + rate, + yield_n, + redemption, + frequency, + basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err @@ -2898,7 +3045,14 @@ fn yield_value( yield2 = yield2 * 2.0 price2 = match price_value( - settlement, maturity, rate, yield2, redemption, frequency, basis, + settlement, + maturity, + rate, + yield2, + redemption, + frequency, + basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err @@ -2948,15 +3102,16 @@ fn check_price_yield_args( fn price_yield_values( name : String, values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> FormulaValue { if values.length() != 6 && values.length() != 7 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -2992,26 +3147,45 @@ fn price_yield_values( if name == "PRICE" { let result = match price_value( - settlement, maturity, rate, pr_yld, redemption, frequency, basis, + settlement, + maturity, + rate, + pr_yld, + redemption, + frequency, + basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err } return number_or_num_error(round_significant_digits(result, 15)) } - yield_value(settlement, maturity, rate, pr_yld, redemption, frequency, basis) + yield_value( + settlement, + maturity, + rate, + pr_yld, + redemption, + frequency, + basis, + use_1904_dates~, + ) } ///| -fn pricedisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn pricedisc_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 4 && values.length() != 5 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3040,7 +3214,8 @@ fn pricedisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let frac = match yearfrac_number(settlement, maturity, basis) { + let frac = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3049,19 +3224,22 @@ fn pricedisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn pricemat_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn pricemat_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 5 && values.length() != 6 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let issue = match value_as_date_serial(values[2]) { + let issue = match value_as_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3093,15 +3271,16 @@ fn pricemat_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let dsm = match yearfrac_number(settlement, maturity, basis) { + let dsm = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } - let dis = match yearfrac_number(issue, settlement, basis) { + let dis = match yearfrac_number(issue, settlement, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } - let dim = match yearfrac_number(issue, maturity, basis) { + let dim = match yearfrac_number(issue, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3110,15 +3289,18 @@ fn pricemat_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn tbilleq_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn tbilleq_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 3 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3138,15 +3320,18 @@ fn tbilleq_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn tbillprice_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn tbillprice_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 3 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3166,15 +3351,18 @@ fn tbillprice_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn tbillyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn tbillyield_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 3 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3194,15 +3382,18 @@ fn tbillyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn yielddisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn yielddisc_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 4 && values.length() != 5 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3228,7 +3419,8 @@ fn yielddisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let frac = match yearfrac_number(settlement, maturity, basis) { + let frac = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3237,19 +3429,22 @@ fn yielddisc_values(values : ArrayView[FormulaValue]) -> FormulaValue { } ///| -fn yieldmat_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn yieldmat_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 5 && values.length() != 6 { return Error(formula_error_value) } - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return err } - let issue = match value_as_date_serial(values[2]) { + let issue = match value_as_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3278,15 +3473,16 @@ fn yieldmat_values(values : ArrayView[FormulaValue]) -> FormulaValue { } else { 0 } - let dim = match yearfrac_number(issue, maturity, basis) { + let dim = match yearfrac_number(issue, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } - let dis = match yearfrac_number(issue, settlement, basis) { + let dis = match yearfrac_number(issue, settlement, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } - let dsm = match yearfrac_number(settlement, maturity, basis) { + let dsm = match + yearfrac_number(settlement, maturity, basis, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3330,8 +3526,10 @@ fn change_month_serial( serial : Double, num_months : Double, return_last_month : Bool, + use_1904_dates? : Bool = false, ) -> Double { - let (year, month, day) = match date_parts_from_serial(serial) { + let (year, month, day) = match + date_parts_from_serial(serial, use_1904_dates~) { Some(parts) => parts None => return serial } @@ -3346,12 +3544,13 @@ fn change_month_serial( ) if return_last_month { let last_day = days_in_month(new_year, new_month) - return match excel_serial_from_date(new_year, new_month, last_day) { + return match + excel_serial_from_date(new_year, new_month, last_day, use_1904_dates~) { Some(serial_value) => serial_value None => serial } } - match excel_serial_from_date(new_year, new_month, new_day) { + match excel_serial_from_date(new_year, new_month, new_day, use_1904_dates~) { Some(serial_value) => serial_value None => serial } @@ -3365,6 +3564,7 @@ fn dates_aggregate( f : (Double, Double) -> Double, acc : Double, return_last_month : Bool, + use_1904_dates? : Bool = false, ) -> (Double, Double, Double) { let mut front_date = start_date let mut trailing_date = end_date @@ -3376,7 +3576,12 @@ fn dates_aggregate( let mut result = acc while !stop { trailing_date = front_date - front_date = change_month_serial(front_date, num_months, return_last_month) + front_date = change_month_serial( + front_date, + num_months, + return_last_month, + use_1904_dates~, + ) result = result + f(front_date, trailing_date) stop = if num_months > 0.0 { front_date >= end_date @@ -3392,13 +3597,15 @@ fn coup_number( maturity : Double, settlement : Double, num_months : Double, + use_1904_dates? : Bool = false, ) -> Double { - let (mat_year, mat_month, mat_day) = match date_parts_from_serial(maturity) { + let (mat_year, mat_month, mat_day) = match + date_parts_from_serial(maturity, use_1904_dates~) { Some(parts) => parts None => return 0.0 } let (settle_year, settle_month, settle_day) = match - date_parts_from_serial(settlement) { + date_parts_from_serial(settlement, use_1904_dates~) { Some(parts) => parts None => return 0.0 } @@ -3410,15 +3617,31 @@ fn coup_number( mat_day < days_in_month(mat_year, mat_month) { end_of_month = days_in_month(settle_year, settle_month) == settle_day } - let start_date = change_month_serial(settlement, 0.0, end_of_month) + let start_date = change_month_serial( + settlement, + 0.0, + end_of_month, + use_1904_dates~, + ) let mut coupons = 0.0 if start_date > settlement { coupons = coupons + 1.0 } - let date = change_month_serial(start_date, num_months, end_of_month) + let date = change_month_serial( + start_date, + num_months, + end_of_month, + use_1904_dates~, + ) let f = fn(_pcd : Double, _ncd : Double) -> Double { 1.0 } let (_, _, result) = dates_aggregate( - date, maturity, num_months, f, coupons, end_of_month, + date, + maturity, + num_months, + f, + coupons, + end_of_month, + use_1904_dates~, ) result } @@ -3445,23 +3668,24 @@ fn prepare_odd_yld_or_pr_arg( fn prepare_oddf_args( name : String, values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> Result[ (Double, Double, Double, Double, Double, Double, Double, Double, Int), FormulaValue, ] { - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let issue = match value_as_date_serial(values[2]) { + let issue = match value_as_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let first_coupon = match value_as_date_serial(values[3]) { + let first_coupon = match value_as_date_serial(values[3], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -3526,37 +3750,52 @@ fn oddfprice_calc( redemption : Double, frequency : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { if basis < 0 || basis > 4 { return Err(Error(formula_error_num)) } - let (mat_year, mat_month, mat_day) = match date_parts_from_serial(maturity) { + let (mat_year, mat_month, mat_day) = match + date_parts_from_serial(maturity, use_1904_dates~) { Some(parts) => parts None => return Err(Error(formula_error_value)) } let return_last_month = days_in_month(mat_year, mat_month) == mat_day let num_months = 12.0 / frequency let num_months_neg = -num_months - let mat = change_month_serial(maturity, num_months_neg, return_last_month) + let mat = change_month_serial( + maturity, + num_months_neg, + return_last_month, + use_1904_dates~, + ) let f = fn(_d1 : Double, _d2 : Double) -> Double { 0.0 } let (pcd, _, _) = dates_aggregate( - mat, first_coupon, num_months_neg, f, 0.0, return_last_month, + mat, + first_coupon, + num_months_neg, + f, + 0.0, + return_last_month, + use_1904_dates~, ) if pcd != first_coupon { return Err(Error(formula_error_num)) } - let e = match coupdays_value(settlement, maturity, frequency, basis) { + let e = match + coupdays_value(settlement, maturity, frequency, basis, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let n = match coupnum_value(settlement, maturity, frequency) { + let n = match + coupnum_value(settlement, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let dfc = coupdays_between(issue, first_coupon, basis) + let dfc = coupdays_between(issue, first_coupon, basis, use_1904_dates~) if dfc < e { - let dsc = coupdays_between(settlement, first_coupon, basis) - let a = coupdays_between(issue, settlement, basis) + let dsc = coupdays_between(settlement, first_coupon, basis, use_1904_dates~) + let a = coupdays_between(issue, settlement, basis, use_1904_dates~) let x = yld / frequency + 1.0 let y = dsc / e let p3 = @math.pow(x, n - 1.0 + y) @@ -3571,7 +3810,8 @@ fn oddfprice_calc( let term4 = a / e * (rate / frequency) * 100.0 return Ok(term1 + term2 + term3[0] - term4) } - let nc = match coupnum_value(issue, first_coupon, frequency) { + let nc = match + coupnum_value(issue, first_coupon, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -3581,7 +3821,7 @@ fn oddfprice_calc( index : Double, ) -> Array[Double] { let (last_year, last_month, last_day) = match - date_parts_from_serial(last_coupon) { + date_parts_from_serial(last_coupon, use_1904_dates~) { Some(parts) => parts None => return acc } @@ -3591,15 +3831,20 @@ fn oddfprice_calc( last_day, ) let early_coupon = match - excel_serial_from_date(early_year, early_month, early_day) { + excel_serial_from_date( + early_year, + early_month, + early_day, + use_1904_dates~, + ) { Some(serial) => serial None => return acc } let mut nl = e if basis == 1 { - nl = coupdays_between(early_coupon, last_coupon, basis) + nl = coupdays_between(early_coupon, last_coupon, basis, use_1904_dates~) } - let mut dci = coupdays_between(issue, last_coupon, basis) + let mut dci = coupdays_between(issue, last_coupon, basis, use_1904_dates~) if index > 1.0 { dci = nl } @@ -3609,7 +3854,7 @@ fn oddfprice_calc( } else { last_coupon } - let a = coupdays_between(start_date, end_date, basis) + let a = coupdays_between(start_date, end_date, basis, use_1904_dates~) last_coupon = early_coupon let dcnl = acc[0] + dci / nl let anl = acc[1] + a / nl @@ -3619,22 +3864,35 @@ fn oddfprice_calc( let anl = ag[1] let dsc = if basis == 2 || basis == 3 { let ncd = match - coupon_date_serial("COUPNCD", settlement, first_coupon, frequency) { + coupon_date_serial( + "COUPNCD", + settlement, + first_coupon, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return Err(err) } - coupdays_between(settlement, ncd, basis) + coupdays_between(settlement, ncd, basis, use_1904_dates~) } else { let pcd = match - coupon_date_serial("COUPPCD", settlement, first_coupon, frequency) { + coupon_date_serial( + "COUPPCD", + settlement, + first_coupon, + frequency, + use_1904_dates~, + ) { Ok(serial) => serial Err(err) => return Err(err) } - let a = coupdays_between(pcd, settlement, basis) + let a = coupdays_between(pcd, settlement, basis, use_1904_dates~) e - a } - let nq = coup_number(first_coupon, settlement, num_months) - let n_final = match coupnum_value(first_coupon, maturity, frequency) { + let nq = coup_number(first_coupon, settlement, num_months, use_1904_dates~) + let n_final = match + coupnum_value(first_coupon, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -3654,7 +3912,10 @@ fn oddfprice_calc( } ///| -fn oddfprice_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn oddfprice_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 8 && values.length() != 9 { return Error(formula_error_value) } @@ -3668,14 +3929,22 @@ fn oddfprice_values(values : ArrayView[FormulaValue]) -> FormulaValue { redemption, frequency, basis, - ) = match prepare_oddf_args("ODDFPRICE", values) { + ) = match prepare_oddf_args("ODDFPRICE", values, use_1904_dates~) { Ok(args) => args Err(err) => return err } let result = match oddfprice_calc( - settlement, maturity, issue, first_coupon, rate, yld, redemption, frequency, + settlement, + maturity, + issue, + first_coupon, + rate, + yld, + redemption, + frequency, basis, + use_1904_dates~, ) { Ok(num) => num Err(err) => return err @@ -3703,7 +3972,10 @@ fn get_oddfprice( } ///| -fn oddfyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { +fn oddfyield_values( + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 8 && values.length() != 9 { return Error(formula_error_value) } @@ -3717,14 +3989,14 @@ fn oddfyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { redemption, frequency, basis, - ) = match prepare_oddf_args("ODDFYIELD", values) { + ) = match prepare_oddf_args("ODDFYIELD", values, use_1904_dates~) { Ok(args) => args Err(err) => return err } if basis < 0 || basis > 4 { return Error(formula_error_num) } - let years = coupdays_between(settlement, maturity, basis) + let years = coupdays_between(settlement, maturity, basis, use_1904_dates~) let px = pr - 100.0 let num = rate * years * 100.0 - px let denum = px / 4.0 + years * px / 2.0 + years * 100.0 @@ -3732,8 +4004,16 @@ fn oddfyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { let f = fn(yld : Double) -> Double { match oddfprice_calc( - settlement, maturity, issue, first_coupon, rate, yld, redemption, frequency, + settlement, + maturity, + issue, + first_coupon, + rate, + yld, + redemption, + frequency, basis, + use_1904_dates~, ) { Ok(price) => pr - price Err(_) => 0.0 / 0.0 @@ -3751,19 +4031,20 @@ fn oddfyield_values(values : ArrayView[FormulaValue]) -> FormulaValue { fn prepare_oddl_args( name : String, values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, ) -> Result[ (Double, Double, Double, Double, Double, Double, Double, Int), FormulaValue, ] { - let settlement = match value_as_date_serial(values[0]) { + let settlement = match value_as_date_serial(values[0], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let maturity = match value_as_date_serial(values[1]) { + let maturity = match value_as_date_serial(values[1], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } - let last_interest = match value_as_date_serial(values[2]) { + let last_interest = match value_as_date_serial(values[2], use_1904_dates~) { Ok(num) => num Err(err) => return Err(err) } @@ -3815,7 +4096,11 @@ fn prepare_oddl_args( } ///| -fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue { +fn oddl_values( + name : String, + values : ArrayView[FormulaValue], + use_1904_dates? : Bool = false, +) -> FormulaValue { if values.length() != 7 && values.length() != 8 { return Error(formula_error_value) } @@ -3828,7 +4113,7 @@ fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue redemption, frequency, basis, - ) = match prepare_oddl_args(name, values) { + ) = match prepare_oddl_args(name, values, use_1904_dates~) { Ok(args) => args Err(err) => return err } @@ -3836,7 +4121,8 @@ fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue return Error(formula_error_num) } let num_months = 12.0 / frequency - let nc = match coupnum_value(last_interest, maturity, frequency) { + let nc = match + coupnum_value(last_interest, maturity, frequency, use_1904_dates~) { Ok(num) => num Err(err) => return err } @@ -3845,9 +4131,19 @@ fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue acc : Array[Double], index : Double, ) -> Array[Double] { - let late_coupon = change_month_serial(early_coupon, num_months, false) - let nl = coupdays_between(early_coupon, late_coupon, basis) - let mut dci = coupdays_between(early_coupon, maturity, basis) + let late_coupon = change_month_serial( + early_coupon, + num_months, + false, + use_1904_dates~, + ) + let nl = coupdays_between(early_coupon, late_coupon, basis, use_1904_dates~) + let mut dci = coupdays_between( + early_coupon, + maturity, + basis, + use_1904_dates~, + ) if index < nc { dci = nl } @@ -3855,7 +4151,7 @@ fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue if late_coupon < settlement { a = dci } else if early_coupon < settlement { - a = coupdays_between(early_coupon, settlement, basis) + a = coupdays_between(early_coupon, settlement, basis, use_1904_dates~) } let start_date = if settlement > early_coupon { settlement @@ -3863,7 +4159,7 @@ fn oddl_values(name : String, values : ArrayView[FormulaValue]) -> FormulaValue early_coupon } let end_date = if maturity < late_coupon { maturity } else { late_coupon } - let dsc = coupdays_between(start_date, end_date, basis) + let dsc = coupdays_between(start_date, end_date, basis, use_1904_dates~) early_coupon = late_coupon let dcnl = acc[0] + dci / nl let anl = acc[1] + a / nl diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index 835f64b8..a44cde04 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -5879,7 +5879,12 @@ fn days_before_month(year : Int, month : Int) -> Int { } ///| -fn excel_serial_from_date(year : Int, month : Int, day : Int) -> Double? { +fn excel_serial_from_date( + year : Int, + month : Int, + day : Int, + use_1904_dates? : Bool = false, +) -> Double? { if year < 0 || year > 9999 { return None } @@ -5887,12 +5892,17 @@ fn excel_serial_from_date(year : Int, month : Int, day : Int) -> Double? { if y < 0 || y > 9999 { return None } - let mut days = days_before_year(y) - days = days + days_before_month(y, m) - days = days + (d - 1) - let mut serial = days + 1 - if serial >= 60 { - serial = serial + 1 + let serial = if use_1904_dates { + days_from_civil(y, m, d) - days_from_civil(1904, 1, 1) + } else { + let mut days = days_before_year(y) + days = days + days_before_month(y, m) + days = days + (d - 1) + let mut value = days + 1 + if value >= 60 { + value = value + 1 + } + value } if serial < 0 { None @@ -5914,27 +5924,45 @@ fn excel_time_fraction(hours : Int, minutes : Int, seconds : Int) -> Double? { ///| fn excel_serial_parts( value : FormulaValue, + use_1904_dates? : Bool = false, ) -> Result[(Int, Int, Int, Int, Int, Int), FormulaValue] { match value_as_number(value) { - Ok(num) => - match excel_serial_to_parts(num) { + Ok(num) => { + let parts = if use_1904_dates { + excel_serial_to_parts_1904(num) + } else { + excel_serial_to_parts(num) + } + match parts { Some(parts) => Ok(parts) None => Err(Error(formula_error_num)) } + } Err(err) => Err(err) } } ///| -fn date_parts_from_serial(serial : Double) -> (Int, Int, Int)? { - match excel_serial_to_parts(serial) { +fn date_parts_from_serial( + serial : Double, + use_1904_dates? : Bool = false, +) -> (Int, Int, Int)? { + let parts = if use_1904_dates { + excel_serial_to_parts_1904(serial) + } else { + excel_serial_to_parts(serial) + } + match parts { Some((year, month, day, _, _, _)) => Some((year, month, day)) None => None } } ///| -fn value_as_date_serial(value : FormulaValue) -> Result[Double, FormulaValue] { +fn value_as_date_serial( + value : FormulaValue, + use_1904_dates? : Bool = false, +) -> Result[Double, FormulaValue] { match normalize_scalar(value) { Number(num) => if num < 0.0 { @@ -5948,7 +5976,7 @@ fn value_as_date_serial(value : FormulaValue) -> Result[Double, FormulaValue] { let trimmed = text.trim().to_owned() match parse_date_parts(trimmed) { Some((year, month, day)) => - match excel_serial_from_date(year, month, day) { + match excel_serial_from_date(year, month, day, use_1904_dates~) { Some(serial) => Ok(serial) None => Err(Error(formula_error_value)) } @@ -5972,10 +6000,11 @@ fn value_as_date_serial(value : FormulaValue) -> Result[Double, FormulaValue] { ///| fn date_parts_from_value( value : FormulaValue, + use_1904_dates? : Bool = false, ) -> Result[(Int, Int, Int), FormulaValue] { - match value_as_date_serial(value) { + match value_as_date_serial(value, use_1904_dates~) { Ok(serial) => - match date_parts_from_serial(serial) { + match date_parts_from_serial(serial, use_1904_dates~) { Some(parts) => Ok(parts) None => Err(Error(formula_error_num)) } @@ -6020,18 +6049,25 @@ fn weekday_sun1(year : Int, month : Int, day : Int) -> Int { } ///| -fn iso_week_number(year : Int, month : Int, day : Int) -> Int? { - let serial = match excel_serial_from_date(year, month, day) { +fn iso_week_number( + year : Int, + month : Int, + day : Int, + use_1904_dates? : Bool = false, +) -> Int? { + let serial = match excel_serial_from_date(year, month, day, use_1904_dates~) { Some(value) => value None => return None } let weekday = weekday_monday1(year, month, day) let thursday_serial = serial + Double::from_int(4 - weekday) - let (iso_year, _, _) = match date_parts_from_serial(thursday_serial) { + let (iso_year, _, _) = match + date_parts_from_serial(thursday_serial, use_1904_dates~) { Some(parts) => parts None => return None } - let week1_serial = match excel_serial_from_date(iso_year, 1, 4) { + let week1_serial = match + excel_serial_from_date(iso_year, 1, 4, use_1904_dates~) { Some(value) => value None => return None } @@ -6119,8 +6155,12 @@ fn weekend_mask_from_value( } ///| -fn is_workday_mask(weekend_mask : Array[Int], serial : Double) -> Bool { - match date_parts_from_serial(serial) { +fn is_workday_mask( + weekend_mask : Array[Int], + serial : Double, + use_1904_dates? : Bool = false, +) -> Bool { + match date_parts_from_serial(serial, use_1904_dates~) { Some((year, month, day)) => { let weekday = weekday_monday1(year, month, day) weekend_mask[weekday - 1] == 0 @@ -6130,10 +6170,13 @@ fn is_workday_mask(weekend_mask : Array[Int], serial : Double) -> Bool { } ///| -fn collect_holidays(value : FormulaValue) -> Array[Int] { +fn collect_holidays( + value : FormulaValue, + use_1904_dates? : Bool = false, +) -> Array[Int] { let holidays : Array[Int] = [] for item in flatten_values([value]) { - match value_as_date_serial(item) { + match value_as_date_serial(item, use_1904_dates~) { Ok(serial) => holidays.push(Double::to_int(Double::ceil(serial))) Err(_) => () } @@ -6148,6 +6191,7 @@ fn workday_intl_adjust( holidays : Array[Int], weekend_mask : Array[Int], start_serial : Double, + use_1904_dates? : Bool = false, ) -> Int { let mut adjusted = end_date for holiday in holidays { @@ -6156,9 +6200,17 @@ fn workday_intl_adjust( break } if holiday > Double::to_int(Double::ceil(start_serial)) { - if is_workday_mask(weekend_mask, Double::from_int(holiday)) { + if is_workday_mask( + weekend_mask, + Double::from_int(holiday), + use_1904_dates~, + ) { adjusted = adjusted + sign - while !is_workday_mask(weekend_mask, Double::from_int(adjusted)) { + while !is_workday_mask( + weekend_mask, + Double::from_int(adjusted), + use_1904_dates~, + ) { adjusted = adjusted + sign } } @@ -6168,9 +6220,17 @@ fn workday_intl_adjust( continue } if holiday < Double::to_int(Double::ceil(start_serial)) { - if is_workday_mask(weekend_mask, Double::from_int(holiday)) { + if is_workday_mask( + weekend_mask, + Double::from_int(holiday), + use_1904_dates~, + ) { adjusted = adjusted + sign - while !is_workday_mask(weekend_mask, Double::from_int(adjusted)) { + while !is_workday_mask( + weekend_mask, + Double::from_int(adjusted), + use_1904_dates~, + ) { adjusted = adjusted + sign } } @@ -6186,6 +6246,7 @@ fn networkdays_intl_value( end_serial : Double, weekend_value : FormulaValue, holidays : Array[Int], + use_1904_dates? : Bool = false, ) -> FormulaValue { holidays.sort() let (weekend_mask, workdays) = match weekend_mask_from_value(weekend_value) { @@ -6209,14 +6270,18 @@ fn networkdays_intl_value( let mut count = weeks * workdays let mut days_mod = Double::to_int(Double::floor(offset)) % 7 while days_mod >= 0 { - if is_workday_mask(weekend_mask, end - Double::from_int(days_mod)) { + if is_workday_mask( + weekend_mask, + end - Double::from_int(days_mod), + use_1904_dates~, + ) { count = count + 1 } days_mod = days_mod - 1 } for holiday in holidays { let holiday_serial = Double::from_int(holiday) - if is_workday_mask(weekend_mask, holiday_serial) && + if is_workday_mask(weekend_mask, holiday_serial, use_1904_dates~) && holiday_serial >= start && holiday_serial <= end { count = count - 1 @@ -6231,6 +6296,7 @@ fn workday_intl_value( days : Double, weekend_value : FormulaValue, holidays : Array[Int], + use_1904_dates? : Bool = false, ) -> FormulaValue { holidays.sort() if days == 0.0 { @@ -6252,13 +6318,21 @@ fn workday_intl_value( let mut days_mod = days_int % workdays let mut end_date = Double::to_int(Double::ceil(start_serial)) + offset * 7 if days_mod == 0 { - while !is_workday_mask(weekend_mask, Double::from_int(end_date)) { + while !is_workday_mask( + weekend_mask, + Double::from_int(end_date), + use_1904_dates~, + ) { end_date = end_date - sign } } else { while days_mod != 0 { end_date = end_date + sign - if is_workday_mask(weekend_mask, Double::from_int(end_date)) { + if is_workday_mask( + weekend_mask, + Double::from_int(end_date), + use_1904_dates~, + ) { if days_mod < 0 { days_mod = days_mod + 1 } else { @@ -6268,7 +6342,12 @@ fn workday_intl_value( } } let adjusted = workday_intl_adjust( - end_date, sign, holidays, weekend_mask, start_serial, + end_date, + sign, + holidays, + weekend_mask, + start_serial, + use_1904_dates~, ) Number(Double::from_int(adjusted)) } @@ -6290,12 +6369,14 @@ fn yearfrac_basis_cond( fn yearfrac_basis0( start_serial : Double, end_serial : Double, + use_1904_dates? : Bool = false, ) -> (Double, Double) { - let (sy, sm, sd) = match date_parts_from_serial(start_serial) { + let (sy, sm, sd) = match + date_parts_from_serial(start_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } - let (ey, em, ed) = match date_parts_from_serial(end_serial) { + let (ey, em, ed) = match date_parts_from_serial(end_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } @@ -6320,12 +6401,14 @@ fn yearfrac_basis0( fn yearfrac_basis1( start_serial : Double, end_serial : Double, + use_1904_dates? : Bool = false, ) -> (Double, Double) { - let (sy, sm, sd) = match date_parts_from_serial(start_serial) { + let (sy, sm, sd) = match + date_parts_from_serial(start_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } - let (ey, em, ed) = match date_parts_from_serial(end_serial) { + let (ey, em, ed) = match date_parts_from_serial(end_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } @@ -6352,12 +6435,14 @@ fn yearfrac_basis1( fn yearfrac_basis4( start_serial : Double, end_serial : Double, + use_1904_dates? : Bool = false, ) -> (Double, Double) { - let (sy, sm, sd) = match date_parts_from_serial(start_serial) { + let (sy, sm, sd) = match + date_parts_from_serial(start_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } - let (ey, em, ed) = match date_parts_from_serial(end_serial) { + let (ey, em, ed) = match date_parts_from_serial(end_serial, use_1904_dates~) { Some(parts) => parts None => return (0.0, 0.0) } @@ -6378,16 +6463,17 @@ fn yearfrac_value( start_serial : Double, end_serial : Double, basis : Int, + use_1904_dates? : Bool = false, ) -> FormulaValue { if start_serial == end_serial { return Number(0.0) } let (day_diff, days_in_year) = match basis { - 0 => yearfrac_basis0(start_serial, end_serial) - 1 => yearfrac_basis1(start_serial, end_serial) + 0 => yearfrac_basis0(start_serial, end_serial, use_1904_dates~) + 1 => yearfrac_basis1(start_serial, end_serial, use_1904_dates~) 2 => (end_serial - start_serial, 360.0) 3 => (end_serial - start_serial, 365.0) - 4 => yearfrac_basis4(start_serial, end_serial) + 4 => yearfrac_basis4(start_serial, end_serial, use_1904_dates~) _ => return Error(formula_error_num) } if days_in_year == 0.0 { diff --git a/xlsx/formula_eval_types.mbt b/xlsx/formula_eval_types.mbt index 94ba7837..9762e068 100644 --- a/xlsx/formula_eval_types.mbt +++ b/xlsx/formula_eval_types.mbt @@ -168,6 +168,7 @@ priv struct CalcContext { max_depth : Int mut current_sheet : String mut current_ref : String + use_1904_dates : Bool } ///| diff --git a/xlsx/formula_parse.mbt b/xlsx/formula_parse.mbt index fb1d610b..e0ab9d7d 100644 --- a/xlsx/formula_parse.mbt +++ b/xlsx/formula_parse.mbt @@ -1,11 +1,12 @@ ///| -fn CalcContext::new() -> CalcContext { +fn CalcContext::new(use_1904_dates? : Bool = false) -> CalcContext { { visiting: Map([]), depth: 0, max_depth: 128, current_sheet: "", current_ref: "", + use_1904_dates, } } diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index d1349172..fa258137 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1929,7 +1929,7 @@ pub fn Workbook::calc_cell_value( } let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) - let ctx = CalcContext::new() + let ctx = CalcContext::new(use_1904_dates=self.uses_1904_date_system()) let value = resolve_cell_value(self, sheet.name(), canonical, ctx) let (value_type, raw_value) = formula_value_to_cell(value) if raw || resolved_options.raw_cell_value { @@ -1992,7 +1992,7 @@ pub fn Workbook::calc_cell_typed( } let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) - let ctx = CalcContext::new() + let ctx = CalcContext::new(use_1904_dates=self.uses_1904_date_system()) formula_value_to_cell_value( resolve_cell_value(self, sheet.name(), canonical, ctx), ) From d4917f7b281d13c151b6eeda226bd1b3f25e4c64 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 13:09:20 +0800 Subject: [PATCH 056/150] chore: refresh generated interfaces --- docx2html/opc/pkg.generated.mbti | 4 ++-- xlsx/pkg.generated.mbti | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docx2html/opc/pkg.generated.mbti b/docx2html/opc/pkg.generated.mbti index d7ff4cb6..22ab6ea5 100644 --- a/docx2html/opc/pkg.generated.mbti +++ b/docx2html/opc/pkg.generated.mbti @@ -34,9 +34,9 @@ pub fn resolve_target(String, String) -> String? pub fn validate_docx_archive(@zip.Archive) -> Array[String] -pub fn validate_docx_archive_limited(@zip.Archive, max_findings~ : Int, max_message_chars~ : Int, max_zip_entry_name_chars? : Int, max_zip_total_entry_name_chars? : Int, max_xml_source_units? : Int, max_xml_tokens? : Int, max_xml_materialized_chars? : Int, max_xml_token_chars? : Int) -> (Array[String], Bool) +pub fn validate_docx_archive_limited(@zip.Archive, max_findings~ : Int, max_message_chars~ : Int, max_zip_entry_name_chars? : Int, max_zip_total_entry_name_chars? : Int, max_xml_source_units? : Int, max_xml_tokens? : Int, max_xml_materialized_chars? : Int, max_xml_token_chars? : Int, cancelled? : () -> Bool) -> (Array[String], Bool) -pub fn validate_docx_archive_report_limited(@zip.Archive, max_findings~ : Int, max_message_chars~ : Int, max_zip_entry_name_chars? : Int, max_zip_total_entry_name_chars? : Int, max_xml_source_units? : Int, max_xml_tokens? : Int, max_xml_materialized_chars? : Int, max_xml_token_chars? : Int) -> DocxValidationReport +pub fn validate_docx_archive_report_limited(@zip.Archive, max_findings~ : Int, max_message_chars~ : Int, max_zip_entry_name_chars? : Int, max_zip_total_entry_name_chars? : Int, max_xml_source_units? : Int, max_xml_tokens? : Int, max_xml_materialized_chars? : Int, max_xml_token_chars? : Int, cancelled? : () -> Bool) -> DocxValidationReport pub fn validate_docx_package(BytesView) -> Array[String] diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index d646b6db..2b964a1c 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -2045,6 +2045,8 @@ pub struct Worksheet { cells : Array[Cell] cell_index : Map[String, Int] mut cell_index_valid : Bool + shared_formula_masters_index : Map[UInt, SharedFormulaMaster] + mut shared_formula_masters_index_valid : Bool merged_cells : Array[String] hyperlinks : Array[Hyperlink] tables : Array[Table] From 23b25e30c67c55e710e53c8efa0a75e8e76a81e8 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 13:47:08 +0800 Subject: [PATCH 057/150] fix(xlsx): terminate workday at date bounds --- xlsx/calc_1904_date_test.mbt | 29 +++++++++++++++ xlsx/formula_eval.mbt | 69 ++++++++++++++++++++++++++++++------ 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/xlsx/calc_1904_date_test.mbt b/xlsx/calc_1904_date_test.mbt index e6860bf9..9167daa0 100644 --- a/xlsx/calc_1904_date_test.mbt +++ b/xlsx/calc_1904_date_test.mbt @@ -63,3 +63,32 @@ test "date-aware financial formulas are invariant across date systems" { ) } } + +///| +test "workday formulas terminate at supported date-system boundaries" { + let workbook_1904 = @xlsx.Workbook::new() + let sheet_1904 = workbook_1904.add_sheet("Sheet1") + workbook_1904.set_workbook_props( + Some(@xlsx.WorkbookPropsOptions::with_values(date_1904=true)), + ) + let cases_1904 : Array[String] = [ + "WORKDAY(DATE(1904,1,1),-1)", "WORKDAY.INTL(DATE(1904,1,1),-1)", "WORKDAY.INTL(DATE(1904,1,4),-1,1,DATE(1904,1,1))", + "WORKDAY(DATE(9999,12,31),1)", + ] + for index, formula in cases_1904 { + let cell = "A\{index + 1}" + sheet_1904.set_cell_formula(cell, formula) + inspect(workbook_1904.calc_cell_value("Sheet1", cell), content="#NUM!") + } + + let workbook_1900 = @xlsx.Workbook::new() + let sheet_1900 = workbook_1900.add_sheet("Sheet1") + let cases_1900 : Array[String] = [ + "WORKDAY(DATE(1900,1,1),-2)", "WORKDAY(DATE(9999,12,31),1)", + ] + for index, formula in cases_1900 { + let cell = "A\{index + 1}" + sheet_1900.set_cell_formula(cell, formula) + inspect(workbook_1900.calc_cell_value("Sheet1", cell), content="#NUM!") + } +} diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index a44cde04..d2ff33a6 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -6169,6 +6169,19 @@ fn is_workday_mask( } } +///| +fn maximum_supported_formula_date_serial(use_1904_dates : Bool) -> Int { + Double::to_int(excel_serial_from_date(9999, 12, 31, use_1904_dates~).unwrap()) +} + +///| +fn formula_date_serial_is_supported( + serial : Int, + use_1904_dates : Bool, +) -> Bool { + serial >= 0 && serial <= maximum_supported_formula_date_serial(use_1904_dates) +} + ///| fn collect_holidays( value : FormulaValue, @@ -6192,7 +6205,7 @@ fn workday_intl_adjust( weekend_mask : Array[Int], start_serial : Double, use_1904_dates? : Bool = false, -) -> Int { +) -> Int? { let mut adjusted = end_date for holiday in holidays { if sign > 0 { @@ -6206,12 +6219,18 @@ fn workday_intl_adjust( use_1904_dates~, ) { adjusted = adjusted + sign + if !formula_date_serial_is_supported(adjusted, use_1904_dates) { + return None + } while !is_workday_mask( weekend_mask, Double::from_int(adjusted), use_1904_dates~, ) { adjusted = adjusted + sign + if !formula_date_serial_is_supported(adjusted, use_1904_dates) { + return None + } } } } @@ -6226,18 +6245,24 @@ fn workday_intl_adjust( use_1904_dates~, ) { adjusted = adjusted + sign + if !formula_date_serial_is_supported(adjusted, use_1904_dates) { + return None + } while !is_workday_mask( weekend_mask, Double::from_int(adjusted), use_1904_dates~, ) { adjusted = adjusted + sign + if !formula_date_serial_is_supported(adjusted, use_1904_dates) { + return None + } } } } } } - adjusted + Some(adjusted) } ///| @@ -6314,9 +6339,22 @@ fn workday_intl_value( sign = -1 } let days_int = Double::to_int(trunc_double(days)) + if days_int == 0 { + return Number(Double::ceil(start_serial)) + } + let maximum_serial = maximum_supported_formula_date_serial(use_1904_dates) + // Bound the arithmetic before multiplying the week offset by seven. Any + // larger workday count must leave the supported 0000..9999 date domain, + // regardless of weekend or holiday configuration. + if days_int > maximum_serial || days_int < -maximum_serial { + return Error(formula_error_num) + } let offset = days_int / workdays let mut days_mod = days_int % workdays let mut end_date = Double::to_int(Double::ceil(start_serial)) + offset * 7 + if !formula_date_serial_is_supported(end_date, use_1904_dates) { + return Error(formula_error_num) + } if days_mod == 0 { while !is_workday_mask( weekend_mask, @@ -6324,10 +6362,16 @@ fn workday_intl_value( use_1904_dates~, ) { end_date = end_date - sign + if !formula_date_serial_is_supported(end_date, use_1904_dates) { + return Error(formula_error_num) + } } } else { while days_mod != 0 { end_date = end_date + sign + if !formula_date_serial_is_supported(end_date, use_1904_dates) { + return Error(formula_error_num) + } if is_workday_mask( weekend_mask, Double::from_int(end_date), @@ -6341,15 +6385,18 @@ fn workday_intl_value( } } } - let adjusted = workday_intl_adjust( - end_date, - sign, - holidays, - weekend_mask, - start_serial, - use_1904_dates~, - ) - Number(Double::from_int(adjusted)) + match + workday_intl_adjust( + end_date, + sign, + holidays, + weekend_mask, + start_serial, + use_1904_dates~, + ) { + Some(adjusted) => Number(Double::from_int(adjusted)) + None => Error(formula_error_num) + } } ///| From f88a97029fe81bd547911b609306ea90e17d352e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 14:00:45 +0800 Subject: [PATCH 058/150] fix(office): bound structural diagnostics --- office/format.mbt | 70 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/office/format.mbt b/office/format.mbt index 782d70a5..9e01d527 100644 --- a/office/format.mbt +++ b/office/format.mbt @@ -1256,6 +1256,54 @@ fn resolve_relationship_target(base : String, target : StringView) -> String? { @opc.resolve_part_target(base, target) } +///| +priv struct StructuralFindings { + mut first : String? + mut count : Int +} + +///| +fn StructuralFindings::new() -> StructuralFindings { + { first: None, count: 0 } +} + +///| +fn bounded_structural_finding(value : String, maximum? : Int = 512) -> String { + let output = StringBuilder::new() + let mut count = 0 + for character in value { + if count >= maximum { + output.write_char('…') |> ignore + break + } + output.write_char(character) |> ignore + count += 1 + } + output.to_string() +} + +///| +fn StructuralFindings::push( + self : StructuralFindings, + finding : String, +) -> Unit { + self.count += 1 + if self.first is None { + self.first = Some(bounded_structural_finding(finding)) + } +} + +///| +test "structural findings retain only one bounded diagnostic" { + let findings = StructuralFindings::new() + findings.push("x".repeat(1024)) + for _ in 0..<100_000 { + findings.push("discarded") + } + assert_eq(findings.count, 100_001) + assert_eq(findings.first.unwrap().length(), 513) +} + ///| fn validate_package_structure( archive : @zip.Archive, @@ -1263,8 +1311,8 @@ fn validate_package_structure( main_part : String, require_main_relationships : Bool, context : OfficeFormatContext, -) -> Array[String] raise OfficeError { - let findings : Array[String] = [] +) -> StructuralFindings raise OfficeError { + let findings = StructuralFindings::new() let parts : StableStringMap[Int] = SortedMap([]) let part_names : StableStringMap[String] = SortedMap([]) let package_parts : StableStringMap[Bool] = SortedMap([]) @@ -1944,7 +1992,7 @@ fn structural_findings( part_index : ArchivePartIndex, main_part : String, context : OfficeFormatContext, -) -> Array[String] raise OfficeError { +) -> StructuralFindings raise OfficeError { validate_package_structure( archive, part_index, @@ -2033,14 +2081,16 @@ pub fn detect_archive_format( let findings = structural_findings( actual, archive, part_index, main_part, context, ) - if findings.length() > 0 { - let first = findings[0] - let suffix = if findings.length() == 1 { - "" - } else { - " (and \{findings.length() - 1} more finding(s))" + match findings.first { + Some(first) => { + let suffix = if findings.count == 1 { + "" + } else { + " (and \{findings.count - 1} more finding(s))" + } + raise InvalidPackage(first + suffix) } - raise InvalidPackage(first + suffix) + None => () } context.checkpoint() actual From 582d3cb453a631c32e0ff9038866d9c970a77d1b Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 14:01:51 +0800 Subject: [PATCH 059/150] fix(ooxml): enforce XML name and BOM rules --- ooxml/start_tag_scanner.mbt | 86 +++++++++++++++++++++++++++++++------ xlsx/package_validation.mbt | 2 +- xlsx/read.mbt | 7 ++- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index d09dcb52..9f9d5ad6 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -88,19 +88,55 @@ fn check_xml_cancelled(cancelled : () -> Bool) -> Unit raise ParseXmlError { } ///| -fn is_xml_name_start(unit : UInt16) -> Bool { - (unit >= ('A' : UInt16) && unit <= ('Z' : UInt16)) || - (unit >= ('a' : UInt16) && unit <= ('z' : UInt16)) || - unit == ('_' : UInt16) || - unit >= 0x80 +fn xml_name_scalar_at(xml : StringView, index : Int, end : Int) -> (Int, Int)? { + if index >= end { + return None + } + let first = xml[index].to_int() + if first >= 0xd800 && first <= 0xdbff { + if index + 1 >= end { + return None + } + let second = xml[index + 1].to_int() + if second < 0xdc00 || second > 0xdfff { + return None + } + Some((0x10000 + ((first - 0xd800) << 10) + second - 0xdc00, index + 2)) + } else if first >= 0xdc00 && first <= 0xdfff { + None + } else { + Some((first, index + 1)) + } } ///| -fn is_xml_name_unit(unit : UInt16) -> Bool { - is_xml_name_start(unit) || - (unit >= ('0' : UInt16) && unit <= ('9' : UInt16)) || - unit == ('-' : UInt16) || - unit == ('.' : UInt16) +fn is_xml_name_start_scalar(code : Int) -> Bool { + (code >= 'A'.to_int() && code <= 'Z'.to_int()) || + (code >= 'a'.to_int() && code <= 'z'.to_int()) || + code == '_'.to_int() || + (code >= 0xc0 && code <= 0xd6) || + (code >= 0xd8 && code <= 0xf6) || + (code >= 0xf8 && code <= 0x2ff) || + (code >= 0x370 && code <= 0x37d) || + (code >= 0x37f && code <= 0x1fff) || + (code >= 0x200c && code <= 0x200d) || + (code >= 0x2070 && code <= 0x218f) || + (code >= 0x2c00 && code <= 0x2fef) || + (code >= 0x3001 && code <= 0xd7ff) || + (code >= 0xf900 && code <= 0xfdcf) || + (code >= 0xfdf0 && code <= 0xfffd) || + (code >= 0x10000 && code <= 0xeffff) +} + +///| +fn is_xml_name_scalar(code : Int) -> Bool { + is_xml_name_start_scalar(code) || + (code >= '0'.to_int() && code <= '9'.to_int()) || + code == '-'.to_int() || + code == '.'.to_int() || + code == 0xb7 || + (code >= 0x300 && code <= 0x36f) || + (code >= 0x203f && code <= 0x2040) } ///| @@ -116,13 +152,15 @@ fn validate_xml_qname( raise InvalidXml(msg="XML qualified name is too long") } let mut colon : Int? = None - for index in start.. false } } + guard xml_name_scalar_at(xml, index, end) is Some((code, next)) else { + raise InvalidXml(msg="XML qualified name is invalid") + } if local_start { - if !is_xml_name_start(unit) { + if !is_xml_name_start_scalar(code) { raise InvalidXml(msg="XML qualified name is invalid") } - } else if !is_xml_name_unit(unit) { + } else if !is_xml_name_scalar(code) { raise InvalidXml(msg="XML qualified name is invalid") } + index = next } } colon @@ -2056,6 +2098,24 @@ test "start tag scanner bounds qualified names" { } } +///| +test "start tag scanner enforces XML Unicode name ranges" { + for xml in ["<·bad/>", "<‿bad/>"] { + let scanner = XmlStartTagScanner::new(xml) + try scanner.next() catch { + InvalidXml(msg~) => inspect(msg, content="XML qualified name is invalid") + _ => fail("unexpected scanner cancellation") + } noraise { + _ => fail("expected invalid XML qualified name") + } + } + for xml in ["<é/>", "<𐀀/>", ""] { + let scanner = XmlStartTagScanner::new(xml) + assert_true(scanner.next()) + assert_false(scanner.next()) + } +} + ///| test "expanded-name canonicalization polls cancellation inside long XML" { let checks = [0] diff --git a/xlsx/package_validation.mbt b/xlsx/package_validation.mbt index 8e4ef34e..7fbe03f2 100644 --- a/xlsx/package_validation.mbt +++ b/xlsx/package_validation.mbt @@ -126,7 +126,7 @@ fn validate_ooxml_archive(archive : @zip.Archive) -> Array[String] { /// Decodes a package part as UTF-8, tolerating a leading BOM. Used only /// for the ASCII-structured package plumbing (content types, rels). fn decode_package_part(data : BytesView) -> String { - @encoding/utf8.decode(data) catch { + @encoding/utf8.decode(data, ignore_bom=true) catch { _ => "" } } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 4397fc6a..f9028735 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -91,11 +91,16 @@ fn decode_utf8( } None => () } - @encoding/utf8.decode(bytes) catch { + @encoding/utf8.decode(bytes, ignore_bom=true) catch { _ => raise InvalidXml(msg="invalid utf8") } } +///| +test "XLSX XML decoding strips a legal UTF-8 BOM" { + inspect(decode_utf8(b"\xEF\xBB\xBF", None), content="") +} + ///| /// OOXML numeric attributes must be finite before they enter the workbook /// model. MoonBit's parser accepts IEEE NaN and infinities, but those values From b5075d807b09e3923b9c721280836b7cccf342a3 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 14:13:58 +0800 Subject: [PATCH 060/150] fix(xlsx): honor extension namespace identity --- ooxml/start_tag_scanner.mbt | 196 ++++++++++++++++++++++------------ xlsx/data_validation_test.mbt | 17 +-- xlsx/read.mbt | 39 +++++-- xlsx/read_xml_namespaces.mbt | 44 +++++++- xlsx/style_test.mbt | 29 +++++ 5 files changed, 242 insertions(+), 83 deletions(-) diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 9f9d5ad6..21d39db6 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -53,12 +53,14 @@ pub struct XmlStartTagScanner { priv mut next_namespace_id : Int priv open_name_starts : Array[Int] priv open_name_ends : Array[Int] - priv open_name_rewrites : Array[Bool] + priv open_name_replacements : Array[String?] priv open_local_name_starts : Array[Int] priv open_local_name_ends : Array[Int] priv open_namespace_uris : Array[String] - priv rewrite_element_namespaces : Array[String] + priv rewrite_element_prefixes : Map[String, String] + priv reserved_element_prefixes : Map[String, String] priv rewrite_attribute_prefixes : Map[String, String] + priv reserved_attribute_prefixes : Map[String, String] priv rewrite_output : StringBuilder priv canonicalize_markup : Bool priv rewrite_output_limit : Int @@ -1046,27 +1048,24 @@ fn XmlStartTagScanner::validate_attributes( } else { canonical_prefix + ":" + local_name } - None => self.xml[attribute.name_start:attribute.name_end].to_owned() + None => + match self.reserved_attribute_prefixes.get(prefix.to_owned()) { + Some(expected_uri) if expected_uri != uri => + "_foreign_" + prefix.to_owned() + "_" + local_name + _ => self.xml[attribute.name_start:attribute.name_end].to_owned() + } } if canonical_names.contains(replacement) { raise InvalidXml(msg="XML canonical attribute name collision") } canonical_names.add(replacement) - match self.rewrite_attribute_prefixes.get(uri) { - Some(_) => - if !xml_span_equals( - self.xml, - attribute.name_start, - attribute.name_end, - replacement, - ) { - self.record_rewrite( - attribute.name_start, - attribute.name_end, - replacement, - ) - } - None => () + if !xml_span_equals( + self.xml, + attribute.name_start, + attribute.name_end, + replacement, + ) { + self.record_rewrite(attribute.name_start, attribute.name_end, replacement) } } } @@ -1157,23 +1156,35 @@ fn XmlStartTagScanner::escaped_character_data( } ///| -fn XmlStartTagScanner::should_unprefix_element( +fn XmlStartTagScanner::canonical_element_replacement( self : XmlStartTagScanner, namespace_uri : StringView, -) -> Bool { - for wanted in self.rewrite_element_namespaces { - if xml_string_equals_view(wanted, namespace_uri) { - return true - } + prefix : StringView, + local_name : StringView, +) -> String? { + match self.rewrite_element_prefixes.get(namespace_uri.to_owned()) { + Some(canonical_prefix) => + if canonical_prefix == "" { + Some(local_name.to_owned()) + } else { + Some(canonical_prefix + ":" + local_name.to_owned()) + } + None => + match self.reserved_element_prefixes.get(prefix.to_owned()) { + Some(expected_uri) if expected_uri != namespace_uri.to_owned() => + Some("_foreign_" + prefix.to_owned() + "_" + local_name.to_owned()) + _ => None + } } - false } ///| fn xml_start_tag_scanner_with_rewrites( xml : StringView, - element_namespaces : Array[String], + element_prefixes : Map[String, String], + reserved_element_prefixes : Map[String, String], attribute_prefixes : Map[String, String], + reserved_attribute_prefixes : Map[String, String], canonicalize_markup : Bool, rewrite_output_limit : Int, capture_element_namespaces? : Array[String] = [], @@ -1193,12 +1204,14 @@ fn xml_start_tag_scanner_with_rewrites( next_namespace_id: 2, open_name_starts: [], open_name_ends: [], - open_name_rewrites: [], + open_name_replacements: [], open_local_name_starts: [], open_local_name_ends: [], open_namespace_uris: [], - rewrite_element_namespaces: element_namespaces, + rewrite_element_prefixes: element_prefixes, + reserved_element_prefixes, rewrite_attribute_prefixes: attribute_prefixes, + reserved_attribute_prefixes, rewrite_output: StringBuilder::new(), canonicalize_markup, rewrite_output_limit, @@ -1228,7 +1241,16 @@ pub fn XmlStartTagScanner::new( xml : StringView, cancelled? : () -> Bool = () => false, ) -> XmlStartTagScanner { - xml_start_tag_scanner_with_rewrites(xml, [], Map([]), false, 0, cancelled~) + xml_start_tag_scanner_with_rewrites( + xml, + Map([]), + Map([]), + Map([]), + Map([]), + false, + 0, + cancelled~, + ) } ///| @@ -1247,7 +1269,7 @@ fn XmlStartTagScanner::consume_end_tag( self.xml[name_end] != ('>' : UInt16) { name_end = name_end + 1 } - let colon = validate_xml_qname(self.xml, name_start, name_end) + ignore(validate_xml_qname(self.xml, name_start, name_end)) let tag_end = find_xml_tag_end( self.xml, name_end, @@ -1269,18 +1291,11 @@ fn XmlStartTagScanner::consume_end_tag( if !xml_spans_equal(self.xml, name_start, name_end, open_start, open_end) { raise InvalidXml(msg="XML start and end tags do not match") } - let rewrite_name = self.open_name_rewrites[self.open_name_rewrites.length() - + let replacement = self.open_name_replacements[self.open_name_replacements.length() - 1] - if rewrite_name { - match colon { - Some(value) => - self.record_rewrite( - name_start, - name_end, - self.xml[value + 1:name_end].to_owned(), - ) - None => () - } + match replacement { + Some(value) => self.record_rewrite(name_start, name_end, value) + None => () } match self.capture_start { Some(capture_start) => @@ -1293,7 +1308,7 @@ fn XmlStartTagScanner::consume_end_tag( } ignore(self.open_name_starts.pop()) ignore(self.open_name_ends.pop()) - ignore(self.open_name_rewrites.pop()) + ignore(self.open_name_replacements.pop()) ignore(self.open_local_name_starts.pop()) ignore(self.open_local_name_ends.pop()) ignore(self.open_namespace_uris.pop()) @@ -1483,23 +1498,26 @@ pub fn XmlStartTagScanner::next( None => self.xml[name_start:name_start] } self.current_namespace_uri = self.resolve_prefix(prefix, false) - let rewrite_name = colon is Some(_) && - self.should_unprefix_element(self.current_namespace_uri) - if rewrite_name { - self.record_rewrite( - name_start, - name_end, - match colon { - Some(value) => self.xml[value + 1:name_end].to_owned() - None => self.xml[name_start:name_end].to_owned() - }, - ) - } - self.validate_attributes(name_end, tag_end) - self.current_name_start = match colon { + let local_name_start = match colon { Some(value) => value + 1 None => name_start } + let replacement = match + self.canonical_element_replacement( + self.current_namespace_uri, + prefix, + self.xml[local_name_start:name_end], + ) { + Some(value) if !xml_span_equals(self.xml, name_start, name_end, value) => + Some(value) + _ => None + } + match replacement { + Some(value) => self.record_rewrite(name_start, name_end, value) + None => () + } + self.validate_attributes(name_end, tag_end) + self.current_name_start = local_name_start self.current_name_end = name_end self.current_tag_end = tag_end self.current_depth = scope_depth @@ -1548,7 +1566,7 @@ pub fn XmlStartTagScanner::next( self.depth = scope_depth self.open_name_starts.push(name_start) self.open_name_ends.push(name_end) - self.open_name_rewrites.push(rewrite_name) + self.open_name_replacements.push(replacement) self.open_local_name_starts.push(self.current_name_start) self.open_local_name_ends.push(self.current_name_end) self.open_namespace_uris.push(self.current_namespace_uri) @@ -1634,7 +1652,9 @@ pub fn remove_xml_expanded_name_subtrees( namespaces.append(element_namespaces) let scanner = xml_start_tag_scanner_with_rewrites( xml, - [], + Map([]), + Map([]), + Map([]), Map([]), false, xml.length(), @@ -1671,7 +1691,9 @@ pub fn remove_xml_foreign_namespace_subtrees( namespaces.append(element_namespaces) let scanner = xml_start_tag_scanner_with_rewrites( xml, - [], + Map([]), + Map([]), + Map([]), Map([]), false, xml.length(), @@ -1705,33 +1727,53 @@ pub fn remove_xml_foreign_namespace_subtrees( ///| /// Validates `xml` and rewrites selected expanded names into lexical forms /// expected by a namespace-oblivious parser. Elements in `element_namespaces` -/// lose their prefix; attributes in `attribute_prefixes` receive the mapped -/// prefix. Comments and processing instructions are removed, while CDATA is -/// converted to escaped character data so namespace-oblivious parsers cannot -/// interpret markup-looking content as elements. Values and unselected names -/// are unchanged. +/// lose their prefix. Additional element namespaces in `element_prefixes` and +/// attributes in `attribute_prefixes` receive their mapped canonical prefix. +/// A source name that uses one of those reserved prefixes for another namespace +/// is defanged so a lexical parser cannot mistake it for a supported extension. +/// Comments and processing instructions are removed, while CDATA is converted +/// to escaped character data so namespace-oblivious parsers cannot interpret +/// markup-looking content as elements. Values and unselected names are +/// unchanged. pub fn canonicalize_xml_expanded_names( xml : StringView, element_namespaces : ArrayView[String], attribute_prefixes : ArrayView[(String, String)], max_output_chars? : Int = max_xml_start_tag_chars, cancelled? : () -> Bool = () => false, + element_prefixes? : ArrayView[(String, String)] = [], ) -> String raise ParseXmlError { check_xml_cancelled(cancelled) if max_output_chars < 0 { raise InvalidXml(msg="XML canonical output limit is invalid") } - let elements : Array[String] = [] - elements.append(element_namespaces) + let elements : Map[String, String] = Map([]) + for namespace_uri in element_namespaces { + elements[namespace_uri] = "" + } + let reserved_elements : Map[String, String] = Map([]) + for pair in element_prefixes { + let (namespace_uri, prefix) = pair + elements[namespace_uri] = prefix + if prefix != "" { + reserved_elements[prefix] = namespace_uri + } + } let attributes : Map[String, String] = Map([]) + let reserved_attributes : Map[String, String] = Map([]) for pair in attribute_prefixes { let (namespace_uri, prefix) = pair attributes[namespace_uri] = prefix + if prefix != "" { + reserved_attributes[prefix] = namespace_uri + } } let scanner = xml_start_tag_scanner_with_rewrites( xml, elements, + reserved_elements, attributes, + reserved_attributes, true, max_output_chars, cancelled~, @@ -2057,9 +2099,12 @@ test "expanded-name canonicalization neutralizes non-element markup" { ///| test "expanded-name canonicalization rejects lexical attribute collisions" { let xml = - #| + #| try - canonicalize_xml_expanded_names(xml, ["urn:sheet"], [("urn:rel", "r")]) + canonicalize_xml_expanded_names(xml, ["urn:sheet"], [ + ("urn:rel:one", "r"), + ("urn:rel:two", "r"), + ]) catch { InvalidXml(msg~) => inspect(msg, content="XML canonical attribute name collision") @@ -2069,6 +2114,21 @@ test "expanded-name canonicalization rejects lexical attribute collisions" { } } +///| +test "expanded-name canonicalization maps extensions and defangs reserved prefix aliases" { + let xml = + #|1 + inspect( + canonicalize_xml_expanded_names(xml, ["urn:sheet"], [("urn:rel", "r")], element_prefixes=[ + ("urn:extension", "x14"), + ("urn:math", "xm"), + ]), + content=( + #|1<_foreign_x14_item/> + ), + ) +} + ///| test "expanded-name canonicalization bounds escaped CDATA output" { try diff --git a/xlsx/data_validation_test.mbt b/xlsx/data_validation_test.mbt index 98d629fc..467a3afd 100644 --- a/xlsx/data_validation_test.mbt +++ b/xlsx/data_validation_test.mbt @@ -90,7 +90,7 @@ test "data validation workbook invalid sheet name" { } ///| -test "data validation read x14 extLst" { +test "data validation extension read follows namespace identity" { let workbook = @xlsx.Workbook::new() ignore(workbook.add_sheet("Sheet1")) let bytes = @xlsx.write(workbook) @@ -101,13 +101,14 @@ test "data validation read x14 extLst" { let xml = @encoding/utf8.decode(entry.data()) let ext = #| - #| - #| - #| - #| Sheet1!$B$1:$B$5 - #| A7:B8 - #| - #| + #| + #| Z99 + #| + #| + #| Sheet1!$B$1:$B$5 + #| A7:B8 + #| + #| #| #| let modified = xml.replace_all( diff --git a/xlsx/read.mbt b/xlsx/read.mbt index f9028735..ee904493 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3470,18 +3470,45 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path~) } - let styles_xml = decode(value) - let (styles, conditional_styles) = parse_styles(styles_xml) + let styles_source_xml = decode_source(value) + read_budget.charge_work(styles_source_xml.length()) + let styles_xml = canonicalize_xlsx_xml( + styles_source_xml, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(styles_xml.length()) + let styles_core_xml = match + xlsx_core_source_without_foreign( + styles_source_xml, + "styleSheet", + cancelled~, + ) { + Some(filtered) => { + read_budget.charge_work(filtered.length()) + let canonical = canonicalize_xlsx_xml( + filtered, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(canonical.length()) + canonical + } + None => styles_xml + } + let (styles, conditional_styles) = parse_styles(styles_core_xml) let (default_table_style, default_pivot_style) = parse_table_styles_defaults( - styles_xml, + styles_core_xml, ) - let indexed_colors = parse_indexed_colors(styles_xml) - let mru_colors_xml = parse_mru_colors_xml(styles_xml) + let indexed_colors = parse_indexed_colors(styles_core_xml) + let mru_colors_xml = parse_mru_colors_xml(styles_core_xml) let styles_ext_lst_xml = parse_styles_ext_lst_xml(styles_xml) ( styles, conditional_styles, - parse_default_font(styles_xml), + parse_default_font(styles_core_xml), default_table_style, default_pivot_style, indexed_colors, diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 229a1fd8..3469e6ac 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -1,3 +1,12 @@ +///| +let xlsx_extension_namespace_x14 = "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" + +///| +let xlsx_extension_namespace_x15 = "http://schemas.microsoft.com/office/spreadsheetml/2010/11/main" + +///| +let xlsx_extension_namespace_xm = "http://schemas.microsoft.com/office/excel/2006/main" + ///| /// Converts namespace-qualified OOXML element names into the lexical form used /// by the existing bounded feature parsers. The source is fully namespace- and @@ -7,15 +16,38 @@ fn canonicalize_xlsx_xml( max_output_chars? : Int = default_max_xml_part_bytes, cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { + let root_scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) + if !workbook_scanner_next(root_scanner) || root_scanner.depth() != 1 { + raise InvalidXml(msg="XLSX XML document element is missing") + } + let root_namespace = root_scanner.namespace_uri().to_owned() + let unprefixed_namespaces = [ + transitional_spreadsheet_namespace, strict_spreadsheet_namespace, + ] + let extension_prefixes : Array[(String, String)] = [] + for + namespace_and_prefix in [ + (xlsx_extension_namespace_x14, "x14"), + (xlsx_extension_namespace_x15, "x15"), + (xlsx_extension_namespace_xm, "xm"), + ] { + let (namespace_uri, prefix) = namespace_and_prefix + if namespace_uri == root_namespace { + unprefixed_namespaces.push(namespace_uri) + } else { + extension_prefixes.push((namespace_uri, prefix)) + } + } @ooxml.canonicalize_xml_expanded_names( xml, - [transitional_spreadsheet_namespace, strict_spreadsheet_namespace], + unprefixed_namespaces, [ (transitional_relationship_attribute_namespace, "r"), (strict_relationship_attribute_namespace, "r"), ], max_output_chars~, cancelled~, + element_prefixes=extension_prefixes, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled @@ -222,6 +254,16 @@ test "XLSX XML canonicalization handles Strict elements and relationship attribu ) } +///| +test "XLSX XML canonicalization follows extension namespace identity" { + let source = + #|A1 + let canonical = canonicalize_xlsx_xml(source) + assert_true(canonical.contains("")) + assert_true(canonical.contains("A1")) + assert_true(canonical.contains("<_foreign_x14_dataValidations/>")) +} + ///| test "XLSX XML canonicalization propagates cancellation from a long pass" { let checks = [0] diff --git a/xlsx/style_test.mbt b/xlsx/style_test.mbt index 746c4961..a79ae84b 100644 --- a/xlsx/style_test.mbt +++ b/xlsx/style_test.mbt @@ -181,6 +181,35 @@ test "default font roundtrip" { inspect(parsed.get_default_font(), content="Arial") } +///| +test "foreign style elements cannot spoof core fonts or cell formats" { + let workbook = @xlsx.Workbook::new() + workbook.set_default_font("Arial") + let sheet = workbook.add_sheet("Sheet1") + let style_id = workbook.add_style(@xlsx.Style::number_format("0.00")) + sheet.set_cell("A1", "1.5") + sheet.set_cell_style("A1", style_id) + let archive = @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/styles.xml") is Some(styles_bytes) else { + fail("missing styles.xml") + } + let styles_xml = @encoding/utf8.decode(styles_bytes) + let foreign = + #| + let modified = styles_xml.replace( + old=">\n Date: Sat, 18 Jul 2026 14:19:38 +0800 Subject: [PATCH 061/150] fix(office): make DOCX text scans cooperative --- office/cmd/office/docx_commands.mbt | 6 +- office/cmd/office/docx_read_model.mbt | 222 ++++++++++++++++++++++--- office/cmd/office/docx_read_output.mbt | 65 ++++++-- office/cmd/office/docx_read_wbtest.mbt | 77 ++++++++- office/cmd/office/query_work.mbt | 12 ++ 5 files changed, 343 insertions(+), 39 deletions(-) diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index 6e7fa0bc..0b3d9927 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -212,7 +212,11 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { let budget = DocxTextBudget::new_reported( "successful command output characters", available, max_output, docx_cli_output_framing_chars, ) - entry_text_with_budget(entry, budget) + entry_text_with_budget_cooperative( + entry, + budget, + cancelled=projection.cancelled, + ) } None => "" } diff --git a/office/cmd/office/docx_read_model.mbt b/office/cmd/office/docx_read_model.mbt index 5b3298f7..d7f90fbc 100644 --- a/office/cmd/office/docx_read_model.mbt +++ b/office/cmd/office/docx_read_model.mbt @@ -4,6 +4,9 @@ let docx_cli_max_element_text_chars : Int = 1024 * 1024 ///| let docx_cli_max_query_text_scan_chars : Int = 16 * 1024 * 1024 +///| +let docx_cli_text_yield_work_units : Int = 4096 + ///| struct DocxTextBudget { resource : String @@ -25,6 +28,8 @@ struct DocxTextCollector { mut scanned : Int mut pending_newlines : Int mut truncated : Bool + cancelled : () -> Bool + mut work_since_yield : Int } ///| @@ -56,6 +61,7 @@ fn DocxTextCollector::new_reported( reported_maximum : Int, reported_offset : Int, fail_on_limit : Bool, + cancelled? : () -> Bool = () => false, ) -> DocxTextCollector { { maximum, @@ -68,6 +74,29 @@ fn DocxTextCollector::new_reported( scanned: 0, pending_newlines: 0, truncated: false, + cancelled, + work_since_yield: 0, + } +} + +///| +fn DocxTextCollector::check_cancelled( + self : DocxTextCollector, +) -> Unit raise CliFailure { + check_office_read_cancelled(self.cancelled) +} + +///| +async fn DocxTextCollector::cooperate( + self : DocxTextCollector, + work? : Int = 1, +) -> Unit { + self.check_cancelled() + self.work_since_yield += work.max(0) + if self.work_since_yield >= docx_cli_text_yield_work_units { + self.work_since_yield = 0 + @async.pause() + self.check_cancelled() } } @@ -149,6 +178,70 @@ fn DocxTextCollector::write_literal_string( } } +///| +async fn DocxTextCollector::write_output_char_cooperative( + self : DocxTextCollector, + value : Char, +) -> Unit { + self.cooperate() + self.write_output_char(value) +} + +///| +async fn DocxTextCollector::flush_pending_newlines_cooperative( + self : DocxTextCollector, +) -> Unit { + let count = self.pending_newlines + self.pending_newlines = 0 + for _ in 0.. Unit { + self.cooperate() + if self.scanned >= self.maximum { + self.truncated = true + if self.fail_on_limit { + raise docx_resource_failure( + self.resource, + self.reported_maximum, + actual=self.reported_offset + self.scanned + 1, + ) + } + return + } + self.scanned += 1 + self.flush_pending_newlines_cooperative() + if self.truncated && !self.fail_on_limit { + return + } + self.write_output_char(value) +} + +///| +async fn DocxTextCollector::write_literal_string_cooperative( + self : DocxTextCollector, + value : String, +) -> Unit { + if self.truncated && !self.fail_on_limit { + return + } + for character in value { + self.write_literal_char_cooperative(character) + if self.truncated && !self.fail_on_limit { + break + } + } +} + ///| fn DocxTextCollector::queue_paragraph_separator( self : DocxTextCollector, @@ -206,42 +299,125 @@ fn DocxTextCollector::visit_all( } ///| -fn collect_docx_text_counted( +async fn DocxTextCollector::visit_cooperative( + self : DocxTextCollector, + element : @document.DocumentElement, +) -> Unit { + self.cooperate() + if self.truncated && !self.fail_on_limit { + return + } + match element { + Text(value) => self.write_literal_string_cooperative(value) + Tab => self.write_literal_char_cooperative('\t') + Paragraph(children~, ..) => { + self.visit_all_cooperative(children) + self.queue_paragraph_separator() + self.queue_paragraph_separator() + } + Document(children~, ..) + | Run(children~, ..) + | Hyperlink(children~, ..) + | Table(children~, ..) + | TableRow(children~, ..) + | TableCell(children~, ..) => self.visit_all_cooperative(children) + Checkbox(_) + | NoteReference(_) + | CommentReference(_) + | Image(_) + | Break(_) + | BookmarkStart(_) => () + } +} + +///| +async fn DocxTextCollector::visit_all_cooperative( + self : DocxTextCollector, + elements : Array[@document.DocumentElement], +) -> Unit { + for element in elements { + if self.truncated && !self.fail_on_limit { + break + } + self.visit_cooperative(element) + } +} + +///| +async fn trim_docx_text_end_cooperative( + value : String, + cancelled : () -> Bool, +) -> String { + let mut end = value.length() + let mut work = 0 + while end > 0 && value[end - 1] == ('\n' : UInt16) { + if work >= docx_cli_text_yield_work_units { + work = 0 + check_office_read_cancelled(cancelled) + @async.pause() + check_office_read_cancelled(cancelled) + } + end -= 1 + work += 1 + } + check_office_read_cancelled(cancelled) + if end == value.length() { + value + } else { + value[:end].to_owned() + } +} + +///| +async fn collect_docx_text_counted_cooperative( element : @document.DocumentElement, maximum : Int, resource : String, reported_maximum : Int, reported_offset : Int, fail_on_limit : Bool, -) -> DocxCollectedText raise CliFailure { + cancelled? : () -> Bool = () => false, +) -> DocxCollectedText { let collector = DocxTextCollector::new_reported( - resource, maximum, reported_maximum, reported_offset, fail_on_limit, + resource, + maximum, + reported_maximum, + reported_offset, + fail_on_limit, + cancelled~, ) - collector.visit(element) - { - text: collector.output.to_string().trim_end(chars="\n").to_owned(), - scanned: collector.scanned, - truncated: collector.truncated, - } + collector.visit_cooperative(element) + let text = trim_docx_text_end_cooperative( + collector.output.to_string(), + cancelled, + ) + { text, scanned: collector.scanned, truncated: collector.truncated } } ///| -fn collect_docx_text_prefix( +async fn collect_docx_text_prefix_cooperative( element : @document.DocumentElement, maximum : Int, + cancelled? : () -> Bool = () => false, ) -> (String, Bool) { - // Truncating collectors never raise. - let result = try! collect_docx_text_counted( - element, maximum, "text preview", maximum, 0, false, + let result = collect_docx_text_counted_cooperative( + element, + maximum, + "text preview", + maximum, + 0, + false, + cancelled~, ) (result.text, result.truncated) } ///| -fn entry_text_with_budget( +async fn entry_text_with_budget_cooperative( entry : DocxProjectionEntry, budget : DocxTextBudget, -) -> String raise CliFailure { + cancelled? : () -> Bool = () => false, +) -> String { guard entry.element is Some(element) else { return "" } let remaining = budget.maximum - budget.used if remaining < 0 { @@ -262,20 +438,28 @@ fn entry_text_with_budget( budget.reported_offset + budget.used, ) } - let result = collect_docx_text_counted( - element, element_limit, resource, reported_maximum, reported_offset, true, + let result = collect_docx_text_counted_cooperative( + element, + element_limit, + resource, + reported_maximum, + reported_offset, + true, + cancelled~, ) budget.used += result.scanned result.text } ///| -fn entry_text_prefix( +async fn entry_text_prefix_cooperative( entry : DocxProjectionEntry, maximum : Int, + cancelled? : () -> Bool = () => false, ) -> (String, Bool) { match entry.element { - Some(element) => collect_docx_text_prefix(element, maximum) + Some(element) => + collect_docx_text_prefix_cooperative(element, maximum, cancelled~) None => ("", false) } } diff --git a/office/cmd/office/docx_read_output.mbt b/office/cmd/office/docx_read_output.mbt index 6215375f..f3df5954 100644 --- a/office/cmd/office/docx_read_output.mbt +++ b/office/cmd/office/docx_read_output.mbt @@ -448,7 +448,11 @@ async fn get_docx_payload( bounded.budget.maximum, bounded.budget.used, ) - let text = entry_text_with_budget(entry, text_budget) + let text = entry_text_with_budget_cooperative( + entry, + text_budget, + cancelled=projection.cancelled, + ) bounded.budget.reserve_json_string_body(text) fields["text"] = Json::string(text) } @@ -519,7 +523,11 @@ async fn docx_text_payload( bounded.budget.maximum, bounded.budget.used, ) - let text = entry_text_with_budget(entry, text_budget) + let text = entry_text_with_budget_cooperative( + entry, + text_budget, + cancelled=projection.cancelled, + ) let json = text_record_json(entry, text) bounded.budget.reserve_array_item(json, !entries.is_empty()) entries.push(json) @@ -624,6 +632,7 @@ async fn entry_matches_query( query : PreparedDocxQuery, scan_budget : DocxTextBudget, work : OfficeQueryWorkBudget, + cancelled : () -> Bool, ) -> Bool { if !query_entry_is_in_scope(entry, query, work) { return false @@ -654,7 +663,11 @@ async fn entry_matches_query( } match query.text_pattern { Some(pattern) => { - let value = entry_text_with_budget(entry, scan_budget) + let value = entry_text_with_budget_cooperative( + entry, + scan_budget, + cancelled~, + ) pattern.is_in_cooperative(value, work) } None => true @@ -732,12 +745,20 @@ async fn docx_query_payload( projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { projection.cooperate() - if !entry_matches_query(entry, query, scan_budget, work) { + if !entry_matches_query( + entry, + query, + scan_budget, + work, + projection.cancelled, + ) { continue } if matched >= spec.offset && matches.length() < spec.limit { - let (preview, preview_truncated) = entry_text_prefix( - entry, docx_cli_query_preview_chars, + let (preview, preview_truncated) = entry_text_prefix_cooperative( + entry, + docx_cli_query_preview_chars, + cancelled=projection.cancelled, ) let record = query_record_json(entry, preview, preview_truncated) bounded.budget.reserve_array_item(record, !matches.is_empty()) @@ -951,8 +972,10 @@ async fn docx_outline_payload( Some(element) => { match docx_heading_level(element) { Some(level) => { - let (text, truncated) = entry_text_prefix( - entry, docx_cli_heading_preview_chars, + let (text, truncated) = entry_text_prefix_cooperative( + entry, + docx_cli_heading_preview_chars, + cancelled=projection.cancelled, ) let record = Json::object({ "path": Json::string(entry.path), @@ -1170,7 +1193,11 @@ async fn docx_text_human( output.maximum, output.used, ) - let text = entry_text_with_budget(entry, text_budget) + let text = entry_text_with_budget_cooperative( + entry, + text_budget, + cancelled=projection.cancelled, + ) output.write_terminal_safe(text, single_line=true) returned += 1 } @@ -1202,11 +1229,21 @@ async fn docx_query_human( projection.cooperate(work=docx_cli_projection_yield_elements) for entry in projection.entries { projection.cooperate() - if !entry_matches_query(entry, query, scan_budget, work) { + if !entry_matches_query( + entry, + query, + scan_budget, + work, + projection.cancelled, + ) { continue } if matched >= spec.offset && returned < spec.limit { - let (preview, _) = entry_text_prefix(entry, docx_cli_query_preview_chars) + let (preview, _) = entry_text_prefix_cooperative( + entry, + docx_cli_query_preview_chars, + cancelled=projection.cancelled, + ) output.begin_line() output.write_terminal_safe(entry.path) output.write_char('\t') @@ -1280,8 +1317,10 @@ async fn docx_outline_human( Some(element) => match docx_heading_level(element) { Some(level) => { - let (text, truncated) = entry_text_prefix( - entry, docx_cli_heading_preview_chars, + let (text, truncated) = entry_text_prefix_cooperative( + entry, + docx_cli_heading_preview_chars, + cancelled=projection.cancelled, ) output.begin_line() output.write_plain("heading \{level}\t") diff --git a/office/cmd/office/docx_read_wbtest.mbt b/office/cmd/office/docx_read_wbtest.mbt index 8f70ba0d..67d33fca 100644 --- a/office/cmd/office/docx_read_wbtest.mbt +++ b/office/cmd/office/docx_read_wbtest.mbt @@ -350,6 +350,7 @@ async test "DOCX exact queries cover the full supported XML token length" { PreparedDocxQuery::build(spec, None, work), DocxTextBudget::new("query text scan characters", 1), work, + () => false, ), ) try @@ -937,18 +938,24 @@ async test "DOCX warning retention and deduplication sets share one hard cap" { ///| async test "DOCX preview truncation excludes discarded paragraph separators" { let exact = @word.paragraph([@word.run([@word.text("x".repeat(159))])]) - let (exact_text, exact_truncated) = collect_docx_text_prefix(exact, 160) + let (exact_text, exact_truncated) = collect_docx_text_prefix_cooperative( + exact, 160, + ) assert_eq(exact_text, "x".repeat(159)) assert_false(exact_truncated) let over = @word.paragraph([@word.run([@word.text("x".repeat(161))])]) - let (over_text, over_truncated) = collect_docx_text_prefix(over, 160) + let (over_text, over_truncated) = collect_docx_text_prefix_cooperative( + over, 160, + ) assert_eq(over_text, "x".repeat(160)) assert_true(over_truncated) let internal = @word.document([ exact, @word.paragraph([@word.run([@word.text("y")])]), ]) - let (_, internal_truncated) = collect_docx_text_prefix(internal, 160) + let (_, internal_truncated) = collect_docx_text_prefix_cooperative( + internal, 160, + ) assert_true(internal_truncated) let collector = DocxTextCollector::new_reported("preview", 160, 160, 0, false) try! collector.write_literal_string("z".repeat(2 * 1024 * 1024)) @@ -956,16 +963,66 @@ async test "DOCX preview truncation excludes discarded paragraph separators" { assert_true(collector.truncated) } +///| +async test "DOCX text extraction yields within one large text node" { + let element = @word.paragraph([ + @word.run([@word.text("x".repeat(1024 * 1024))]), + ]) + @async.with_task_group() <| group => { + let task = group.spawn(() => { + collect_docx_text_prefix_cooperative(element, 1024 * 1024) + }) + @async.pause() + assert_true(task.try_wait() is None) + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + +///| +async test "DOCX cooperative text extraction forwards cancellation" { + let checks = [0] + let element = @word.paragraph([@word.run([@word.text("x".repeat(64 * 1024))])]) + try + collect_docx_text_prefix_cooperative(element, 64 * 1024, cancelled=() => { + checks[0] += 1 + checks[0] >= 128 + }) + catch { + CliFailure(error) => { + assert_eq(error.code, "office.cancelled") + assert_true(checks[0] >= 128) + } + _ => fail("unexpected cooperative text cancellation") + } noraise { + _ => fail("expected cooperative text cancellation") + } +} + ///| async test "literal newlines consume scan budgets even when presentation trims them" { let newline_only = @word.paragraph([@word.run([@word.text("\n".repeat(161))])]) - let (preview, truncated) = collect_docx_text_prefix(newline_only, 160) + let (preview, truncated) = collect_docx_text_prefix_cooperative( + newline_only, 160, + ) assert_eq(preview, "") assert_true(truncated) let projection = docx_read_test_projection() let entry = { ..projection.entries[0], element: Some(newline_only) } let aggregate = DocxTextBudget::new("query text scan characters", 16) - try entry_text_with_budget(entry, aggregate) catch { + try + entry_text_with_budget_cooperative( + entry, + aggregate, + cancelled=projection.cancelled, + ) + catch { CliFailure(error) => { assert_eq(error.code, "office.docx.resource_limit") guard error.details is Some(Object(details)) else { @@ -976,6 +1033,7 @@ async test "literal newlines consume scan budgets even when presentation trims t Some(Json::string("query text scan characters")), ) } + _ => fail("unexpected newline scan failure") } noraise { _ => fail("expected literal newlines to exhaust the scan budget") } @@ -996,7 +1054,13 @@ async test "DOCX text budgets retain an independent per-element ceiling" { "query text scan characters", docx_cli_max_element_text_chars * 2, ) - try entry_text_with_budget(oversized, aggregate) catch { + try + entry_text_with_budget_cooperative( + oversized, + aggregate, + cancelled=projection.cancelled, + ) + catch { CliFailure(error) => { assert_eq(error.code, "office.docx.resource_limit") guard error.details is Some(Object(details)) else { @@ -1011,6 +1075,7 @@ async test "DOCX text budgets retain an independent per-element ceiling" { Some(Json::number(docx_cli_max_element_text_chars.to_double())), ) } + _ => fail("unexpected per-element text failure") } noraise { _ => fail("expected the independent per-element ceiling") } diff --git a/office/cmd/office/query_work.mbt b/office/cmd/office/query_work.mbt index 6b5652c2..4a343a28 100644 --- a/office/cmd/office/query_work.mbt +++ b/office/cmd/office/query_work.mbt @@ -113,7 +113,12 @@ async fn OfficeLinearPattern::compile_cooperative( @async.pause() } while matched > 0 && characters[at] != characters[matched] { + if visited >= office_cli_query_yield_work_units { + visited = 0 + @async.pause() + } matched = failure[matched - 1] + visited += 1 } if characters[at] == characters[matched] { matched += 1 @@ -192,6 +197,7 @@ async fn OfficeLinearPattern::is_in_cooperative( } let mut matched = 0 let mut pending_work = 1 + let mut scheduling_work = 0 for raw_character in value { // Supplementary scalars occupy two UTF-16 units. This preserves the // conservative accounting while charging and yielding in small quanta. @@ -202,9 +208,15 @@ async fn OfficeLinearPattern::is_in_cooperative( @async.pause() } pending_work += scalar_work + scheduling_work += 1 let character = query_fold_character(raw_character, self.ignore_case) while matched > 0 && character != self.characters[matched] { + if scheduling_work >= office_cli_query_yield_work_units { + scheduling_work = 0 + @async.pause() + } matched = self.failure[matched - 1] + scheduling_work += 1 } if character == self.characters[matched] { matched += 1 From 9debb3a7b0d8339339cef176071f54e0c2906adf Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 14:29:17 +0800 Subject: [PATCH 062/150] fix(read): poll long validation passes --- docx2html/opc/validate.mbt | 18 +++- docx2html/opc/validate_test.mbt | 20 ++++ xlsx/read_worksheet_xml.mbt | 12 +-- xlsx/shared_formula_ranges.mbt | 184 +++++++++++++++++++++++++++++--- xlsx/worksheet.mbt | 37 ++++++- 5 files changed, 248 insertions(+), 23 deletions(-) diff --git a/docx2html/opc/validate.mbt b/docx2html/opc/validate.mbt index 7d7035e8..b1705b62 100644 --- a/docx2html/opc/validate.mbt +++ b/docx2html/opc/validate.mbt @@ -203,6 +203,7 @@ priv struct FindingCollector { max_zip_total_entry_name_chars : Int mut zip_entry_name_chars : Int xml_budget : @xml.XmlReadBudget + cancelled : () -> Bool mut truncated : Bool mut resource_limit : DocxValidationResourceLimit? mut main_document_part : String? @@ -262,6 +263,7 @@ fn FindingCollector::new_with_xml_limits( }, cancelled~, ), + cancelled, truncated: false, resource_limit: None, main_document_part: None, @@ -658,7 +660,21 @@ fn validate_docx_archive_into( seen.add(name) } let logical_name = check_entry_name(name, findings) - if entry.crc32() != @mbtzip.crc32(entry.data()) { + let actual_crc = @mbtzip.crc32_cancellable( + entry.data(), + cancelled=findings.cancelled, + ) catch { + ReadCancelled => { + findings.stop() + return + } + _ => { + findings.add("entry CRC-32 verification failed: \{findings.text(name)}") + findings.stop() + return + } + } + if entry.crc32() != actual_crc { findings.add( "entry data does not match its stored CRC-32: \{findings.text(name)}", ) diff --git a/docx2html/opc/validate_test.mbt b/docx2html/opc/validate_test.mbt index b9b05791..c49d1b6d 100644 --- a/docx2html/opc/validate_test.mbt +++ b/docx2html/opc/validate_test.mbt @@ -1246,6 +1246,26 @@ test "duplicate names and CRC corruption are findings" { ) } +///| +test "archive validation polls cancellation inside long CRC verification" { + let archive = @mbtzip.Archive::new() + archive.add("padding.bin", Bytes::make(16 * 1024, b'x')) + let checks = [0] + let report = @opc.validate_docx_archive_report_limited( + archive, + max_findings=8, + max_message_chars=256, + cancelled=() => { + checks[0] += 1 + checks[0] >= 4 + }, + ) + assert_eq(checks[0], 4) + assert_true(report.findings_truncated()) + assert_eq(report.findings(), []) + assert_eq(report.main_document_part(), None) +} + ///| test "an empty or slash-less Override PartName is a finding, not a crash" { let types = diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index ca3896ed..31938e6e 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -284,14 +284,10 @@ fn parse_worksheet( } // Shared formulas are a worksheet-wide construct: followers carry only an // index, so validate the complete group before exposing a partial model. - match budget { - Some(value) => { - value.checkpoint() - value.charge_work(cells.length()) - } - None => () - } - let shared_formula_masters_index = validated_shared_formula_masters(cells) + let shared_formula_masters_index = validated_shared_formula_masters( + cells, + budget?, + ) worksheet_parse_pass(xml, budget) let merged_cells = parse_merge_cells(xml) worksheet_parse_pass(xml, budget) diff --git a/xlsx/shared_formula_ranges.mbt b/xlsx/shared_formula_ranges.mbt index 4aa04a71..4806c403 100644 --- a/xlsx/shared_formula_ranges.mbt +++ b/xlsx/shared_formula_ranges.mbt @@ -6,6 +6,112 @@ priv struct SharedFormulaRangeEvent { delta : Int } +///| +fn compare_shared_formula_range_events( + left : SharedFormulaRangeEvent, + right : SharedFormulaRangeEvent, +) -> Int { + if left.row != right.row { + left.row.compare(right.row) + } else if left.delta != right.delta { + // Remove rectangles that ended on the preceding row before inserting + // rectangles that begin on this row. + left.delta.compare(right.delta) + } else if left.column_lo != right.column_lo { + left.column_lo.compare(right.column_lo) + } else { + left.column_hi.compare(right.column_hi) + } +} + +///| +/// Bottom-up merge sort used instead of `Array::sort_by` so hostile workloads +/// cannot hide an uninterruptible O(m log m) comparison pass inside the +/// runtime. Each pass is charged before it runs and polls at 4 Ki-event +/// boundaries while merging and copying back. +fn sort_shared_formula_range_events( + events : Array[SharedFormulaRangeEvent], + budget? : ReadBudget, +) -> Unit raise XlsxError { + if events.length() < 2 { + return + } + let scratch = events.copy() + let mut width = 1 + while width < events.length() { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(events.length()) + } + None => () + } + let mut run = 0 + let mut output = 0 + while run < events.length() { + let middle = if run > events.length() - width { + events.length() + } else { + run + width + } + let end = if middle > events.length() - width { + events.length() + } else { + middle + width + } + let mut left = run + let mut right = middle + while left < middle || right < end { + if (output & 4095) == 0 { + match budget { + Some(value) => value.checkpoint() + None => () + } + } + if right >= end || + ( + left < middle && + compare_shared_formula_range_events(events[left], events[right]) <= + 0 + ) { + scratch[output] = events[left] + left += 1 + } else { + scratch[output] = events[right] + right += 1 + } + output += 1 + } + run = end + } + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(events.length()) + } + None => () + } + for index in 0.. value.checkpoint() + None => () + } + } + events[index] = scratch[index] + } + width = if width > events.length() / 2 { + events.length() + } else { + width * 2 + } + } + match budget { + Some(value) => value.checkpoint() + None => () + } +} + ///| fn shared_formula_range_tree_push( maximums : Array[Int], @@ -126,12 +232,32 @@ fn shared_formula_range_tree_maximum( /// ranges remain valid. The column tree has a fixed 16,384-column footprint. fn validate_shared_formula_master_ranges( masters : Map[UInt, SharedFormulaMaster], + budget? : ReadBudget, ) -> Unit raise XlsxError { if masters.length() < 2 { + match budget { + Some(value) => value.checkpoint() + None => () + } return } let events : Array[SharedFormulaRangeEvent] = [] + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(masters.length()) + value.charge_work(masters.length()) + } + None => () + } + let mut master_index = 0 for _, master in masters { + if (master_index & 4095) == 0 { + match budget { + Some(value) => value.checkpoint() + None => () + } + } events.push({ row: master.row_lo, column_lo: master.column_lo, @@ -144,24 +270,31 @@ fn validate_shared_formula_master_ranges( column_hi: master.column_hi, delta: -1, }) + master_index += 1 } - events.sort_by((left, right) => { - if left.row != right.row { - left.row.compare(right.row) - } else if left.delta != right.delta { - // Remove rectangles that ended on the preceding row before inserting - // rectangles that begin on this row. - left.delta.compare(right.delta) - } else if left.column_lo != right.column_lo { - left.column_lo.compare(right.column_lo) - } else { - left.column_hi.compare(right.column_hi) - } - }) + sort_shared_formula_range_events(events, budget?) let tree_size = cell_ref_max_cols * 4 + 8 let maximums = Array::make(tree_size, 0) let pending_additions = Array::make(tree_size, 0) + match budget { + Some(value) => + // Each event performs at most one range maximum and one range update on + // a fixed-depth 16,384-column tree. Charging 32 event-width passes is a + // conservative bound for both traversals without risking multiplication + // overflow in attacker-controlled accounting. + for _ in 0..<32 { + value.charge_work(events.length()) + } + None => () + } + let mut event_index = 0 for event in events { + if (event_index & 4095) == 0 { + match budget { + Some(value) => value.checkpoint() + None => () + } + } if event.delta > 0 && shared_formula_range_tree_maximum( maximums, @@ -185,5 +318,30 @@ fn validate_shared_formula_master_ranges( event.column_hi, event.delta, ) + event_index += 1 + } + match budget { + Some(value) => value.checkpoint() + None => () + } +} + +///| +test "shared formula event sorting polls cancellation inside long passes" { + let events : Array[SharedFormulaRangeEvent] = [] + for index in 0..<(16 * 1024) { + events.push({ row: 16 * 1024 - index, column_lo: 1, column_hi: 1, delta: 1 }) + } + let checks = [0] + let budget = ReadBudget::new(ReadLimits::new(), cancelled=() => { + checks[0] += 1 + checks[0] >= 4 + }) + let result : Result[Unit, Error] = Ok( + sort_shared_formula_range_events(events, budget~), + ) catch { + error => Err(error) } + assert_eq(checks[0], 4) + assert_true(result is Err(ReadCancelled)) } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 53db4ee7..def80366 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -393,9 +393,24 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { /// are retained only once even for large shared ranges. fn validated_shared_formula_masters( cells : ArrayView[Cell], + budget? : ReadBudget, ) -> Map[UInt, SharedFormulaMaster] raise XlsxError { let masters : Map[UInt, SharedFormulaMaster] = Map([]) + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(cells.length()) + } + None => () + } + let mut cell_index = 0 for cell in cells { + if (cell_index & 4095) == 0 { + match budget { + Some(value) => value.checkpoint() + None => () + } + } match ( cell.formula_type, @@ -435,9 +450,24 @@ fn validated_shared_formula_masters( raise InvalidXml(msg="shared formula text missing") _ => () } + cell_index += 1 + } + validate_shared_formula_master_ranges(masters, budget?) + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(cells.length()) + } + None => () } - validate_shared_formula_master_ranges(masters) + cell_index = 0 for cell in cells { + if (cell_index & 4095) == 0 { + match budget { + Some(value) => value.checkpoint() + None => () + } + } match (cell.formula_type, cell.formula_shared_index, cell.formula_ref) { (Some(Shared), Some(shared_index), None) => match masters.get(shared_index) { @@ -451,6 +481,11 @@ fn validated_shared_formula_masters( } _ => () } + cell_index += 1 + } + match budget { + Some(value) => value.checkpoint() + None => () } masters } From 0b07bd07e35bd467e61b7ac8767bfb7701b56d2c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 14:29:46 +0800 Subject: [PATCH 063/150] chore: refresh generated interfaces --- ooxml/pkg.generated.mbti | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 749b870c..7adb4950 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -8,7 +8,7 @@ import { // Values pub fn attr_value(StringView, StringView) -> String? raise ParseXmlError -pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)], max_output_chars? : Int, cancelled? : () -> Bool) -> String raise ParseXmlError +pub fn canonicalize_xml_expanded_names(StringView, ArrayView[String], ArrayView[(String, String)], max_output_chars? : Int, cancelled? : () -> Bool, element_prefixes? : ArrayView[(String, String)]) -> String raise ParseXmlError pub fn decode_xml_attribute(StringView, cancelled? : () -> Bool) -> String raise ParseXmlError From a3d49b6261abd598ca0d14afb9835eec9a592d86 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:18:57 +0800 Subject: [PATCH 064/150] fix(xlsx): bound formatted text expansion --- office/cmd/office/xlsx_read_model.mbt | 8 ++ office/cmd/office/xlsx_read_wbtest.mbt | 27 +++++ xlsx/value_format.mbt | 138 ++++++++++++++++++++++--- xlsx/workbook.mbt | 4 +- 4 files changed, 163 insertions(+), 14 deletions(-) diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 9f61d1dd..ac1c3757 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -592,6 +592,13 @@ fn XlsxStringBudget::charge( self.used += count } +///| +/// Returns the largest formatted value that can still satisfy both the +/// per-value and aggregate retained-string ceilings. +fn XlsxStringBudget::remaining_for_value(self : XlsxStringBudget) -> Int { + self.per_value_maximum.min(self.maximum - self.used) +} + ///| fn XlsxFormatBudget::new( maximum? : Int = xlsx_cli_max_format_work_units, @@ -790,6 +797,7 @@ fn XlsxCellSnapshot::ensure_formatted( self.row, self.column, self.style_id, + max_output_chars=budget.strings.remaining_for_value(), ) }).unwrap_or(""), ) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 8f59a76c..7187a79b 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -791,6 +791,33 @@ async test "XLSX formatting is deferred and custom-format work is charged" { ) } +///| +async test "XLSX formatting rejects text amplification before materialization" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_str("Data", "A1", "x".repeat(64 * 1024)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("@".repeat(64 * 1024)), + ) + workbook.set_cell_style("Data", "A1", style_id) + let projection = make_xlsx_projection("amplified-format.xlsx", workbook, 10) + let regions = xlsx_scan_regions( + projection, + Some(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]")), + ) + let limited = xlsx_cli_error_async(() => { + ignore(collect_xlsx_cell_page(projection, regions, TextCells, 0, 1)) + }) + assert_eq(limited.code, "office.xlsx.resource_limit") + guard limited.details is Some(Object(details)) else { + fail("expected formatted-output limit details") + } + assert_eq( + details.get("resource"), + Some(Json::string("formatted_cell_value_chars")), + ) +} + ///| async test "XLSX outline and get use office schemas with canonical cell records" { let projection = xlsx_read_test_projection() diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index e5093f45..daa74c31 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -5,13 +5,29 @@ fn format_cell_value( style : Style?, options : Options, use_1904_format? : Bool = false, -) -> String { - match value_type { +) -> String raise XlsxError { + format_cell_value_limited(value_type, raw, style, options, use_1904_format~) +} + +///| +/// Formats a cell while optionally bounding the materialized UTF-16 output. +/// The text-pattern path checks each append before it reaches `StringBuilder`, +/// so repeated `@` placeholders cannot amplify a small cell into an unbounded +/// intermediate string. +fn format_cell_value_limited( + value_type : CellValueType, + raw : String, + style : Style?, + options : Options, + use_1904_format? : Bool = false, + max_output_chars? : Int, +) -> String raise XlsxError { + let output = match value_type { String => match style { Some(style) => match style.number_format { - Some(format) => format_text_value(raw, format) + Some(format) => format_text_value(raw, format, max_output_chars?) None => raw } None => raw @@ -29,28 +45,66 @@ fn format_cell_value( None => raw } } + check_formatted_output_length(output, max_output_chars) + output } ///| -fn format_text_value(raw : String, format : NumberFormat) -> String { +fn format_text_value( + raw : String, + format : NumberFormat, + max_output_chars? : Int, +) -> String raise XlsxError { match format { Builtin(_id) => raw Custom(code) => if code.to_lower().trim().to_owned() == "general" { raw } else { - format_text_pattern(raw, code) + format_text_pattern(raw, code, max_output_chars?) + } + } +} + +///| +fn checked_formatted_output_growth( + current : Int, + amount : Int, + maximum : Int?, +) -> Int raise XlsxError { + match maximum { + None => current + amount + Some(limit) => { + if limit < 0 { + raise InvalidOptions(msg="formatted output limit must be non-negative") + } + if amount < 0 || current < 0 || amount > limit - current { + raise ResourceLimitExceeded( + kind="formatted_cell_value_chars", + limit~, + actual=if limit < 0x7fffffff { limit + 1 } else { limit }, + ) } + current + amount + } } } +///| +fn check_formatted_output_length( + output : String, + maximum : Int?, +) -> Unit raise XlsxError { + ignore(checked_formatted_output_growth(0, output.length(), maximum)) +} + ///| fn format_number_with_format( raw : String, format : NumberFormat, options : Options, use_1904_format : Bool, -) -> String { +) -> String raise XlsxError { match format { Builtin(id) => format_number_builtin(raw, id, options, use_1904_format) Custom(code) => format_number_code(raw, code, options, use_1904_format~) @@ -63,7 +117,7 @@ fn format_number_builtin( id : Int, options : Options, use_1904_format : Bool, -) -> String { +) -> String raise XlsxError { let short_date = options.short_date_pattern let long_date = options.long_date_pattern let long_time = options.long_time_pattern @@ -130,7 +184,7 @@ fn format_number_code( code : String, options : Options, use_1904_format? : Bool = false, -) -> String { +) -> String raise XlsxError { let _ = options let sections = split_format_sections(code) if sections.length() == 0 { @@ -354,7 +408,11 @@ fn render_literal_segment(segment : String) -> String { } ///| -fn format_text_pattern(raw : String, code : String) -> String { +fn format_text_pattern( + raw : String, + code : String, + max_output_chars? : Int, +) -> String raise XlsxError { let sections = split_format_sections(code) if sections.length() == 0 { return raw @@ -362,12 +420,26 @@ fn format_text_pattern(raw : String, code : String) -> String { let section = if sections.length() >= 4 { sections[3] } else { sections[0] } let (pattern, _locale, currency_prefix) = parse_section_metadata(section) let sb = StringBuilder::new() + let mut output_chars = checked_formatted_output_growth( + 0, + currency_prefix.length(), + max_output_chars, + ) sb.write_view(currency_prefix) let mut in_quote = false let mut escape_next = false let mut skip_next = false for c in pattern { if escape_next { + output_chars = checked_formatted_output_growth( + output_chars, + if c.to_int() > 0xffff { + 2 + } else { + 1 + }, + max_output_chars, + ) sb.write_char(c) escape_next = false continue @@ -379,10 +451,50 @@ fn format_text_pattern(raw : String, code : String) -> String { match c { '\\' => escape_next = true '"' => in_quote = !in_quote - '_' => if !in_quote { skip_next = true } else { sb.write_char('_') } - '*' => if !in_quote { skip_next = true } else { sb.write_char('*') } - '@' => if !in_quote { sb.write_view(raw) } else { sb.write_char('@') } - _ => sb.write_char(c) + '_' => + if !in_quote { + skip_next = true + } else { + output_chars = checked_formatted_output_growth( + output_chars, 1, max_output_chars, + ) + sb.write_char('_') + } + '*' => + if !in_quote { + skip_next = true + } else { + output_chars = checked_formatted_output_growth( + output_chars, 1, max_output_chars, + ) + sb.write_char('*') + } + '@' => + if !in_quote { + output_chars = checked_formatted_output_growth( + output_chars, + raw.length(), + max_output_chars, + ) + sb.write_view(raw) + } else { + output_chars = checked_formatted_output_growth( + output_chars, 1, max_output_chars, + ) + sb.write_char('@') + } + _ => { + output_chars = checked_formatted_output_growth( + output_chars, + if c.to_int() > 0xffff { + 2 + } else { + 1 + }, + max_output_chars, + ) + sb.write_char(c) + } } } sb.to_string() diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index fa258137..cd578ff6 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1883,6 +1883,7 @@ pub fn Workbook::get_cell_value_styled_from_worksheet_rc( col : Int, style_id : Int, options? : Options, + max_output_chars? : Int, ) -> String? raise XlsxError { ignore(cell_ref_from(row, col)) self.check_style_id(style_id) @@ -1897,12 +1898,13 @@ pub fn Workbook::get_cell_value_styled_from_worksheet_rc( return Some(cell.value) } Some( - format_cell_value( + format_cell_value_limited( cell.value_type, cell.value, Some(self.styles[style_id]), resolved_options, use_1904_format=self.uses_1904_date_system(), + max_output_chars?, ), ) } From 575f563da53840b0a74f7b786559c3d5e9c49c01 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:24:54 +0800 Subject: [PATCH 065/150] fix(xlsx): bind feature readers to XML namespaces --- xlsx/read.mbt | 37 +++++++++-- xlsx/read_namespace_test.mbt | 27 ++++++++ xlsx/read_xml_namespaces.mbt | 123 +++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 7 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index ee904493..7f73cb64 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3757,6 +3757,13 @@ fn read_zip_archive_core_unchecked( if sheet_core_xml != sheet_xml { read_budget.charge_work(sheet_core_xml.length()) } + let sheet_feature_xml = canonicalize_xlsx_worksheet_features( + sheet_source_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(sheet_feature_xml.length()) let ( cells, shared_formula_masters_index, @@ -3783,7 +3790,7 @@ fn read_zip_archive_core_unchecked( let hyperlink_elements = parse_hyperlink_elements(sheet_core_xml) let table_part_ids = parse_table_part_ids(sheet_core_xml) let pivot_part_ids = parse_pivot_table_part_ids(sheet_core_xml) - let slicer_rel_ids = parse_sheet_slicer_rel_ids(sheet_xml) + let slicer_rel_ids = parse_sheet_slicer_rel_ids(sheet_feature_xml) let drawing_rel_id = parse_legacy_drawing_rel_id( sheet_core_xml, "drawing", ) @@ -3793,25 +3800,33 @@ fn read_zip_archive_core_unchecked( let legacy_drawing_hf_rel_id = parse_legacy_drawing_rel_id( sheet_core_xml, "legacyDrawingHF", ) - let sparkline_groups = parse_sparkline_groups(sheet_xml) + let sparkline_groups = parse_sparkline_groups(sheet_feature_xml) let data_validations = parse_data_validations( sheet_core_xml, budget=read_budget, ) - for dv in parse_data_validations_x14(sheet_xml, budget=read_budget) { + for + dv in parse_data_validations_x14(sheet_feature_xml, budget=read_budget) { data_validations.push(dv) } let conditional_formats = parse_conditional_formats( // Base data-bar rules carry their x14 correlation id in a nested // extLst, so this parser needs the preserved extension view. Cell and // row parsing above still uses the extension-free structural view. - sheet_xml, + sheet_feature_xml, budget=read_budget, ) - for cf in parse_conditional_formats_x14(sheet_xml, budget=read_budget) { + for + cf in parse_conditional_formats_x14( + sheet_feature_xml, + budget=read_budget, + ) { conditional_formats.push(cf) } - let x14_data_bars = parse_x14_data_bars(sheet_xml, budget=read_budget) + let x14_data_bars = parse_x14_data_bars( + sheet_feature_xml, + budget=read_budget, + ) let unknown_ext_blocks = parse_unknown_worksheet_ext_blocks(sheet_xml) let ignored_errors = parse_ignored_errors( sheet_core_xml, @@ -4288,7 +4303,15 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path=drawing_path) } - let drawing_xml = decode(drawing_bytes) + let drawing_source_xml = decode_source(drawing_bytes) + read_budget.charge_work(drawing_source_xml.length()) + let drawing_xml = canonicalize_xlsx_drawing_xml( + drawing_source_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(drawing_xml.length()) let drawing_metrics = drawing_metrics_for_sheet(sheet) if slicer_part_entries.length() > 0 { slicer_anchor_info = parse_drawing_slicer_anchors( diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index d265ce1f..23fb2c10 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -350,6 +350,33 @@ test "foreign core-looking elements cannot supply XLSX data" { let parsed = @xlsx.read(@zip.write(archive)) assert_eq(parsed.get_merge_cells("Qualified"), ["A1:B1"]) + let foreign_conditional = @zip.read( + namespace_qualified_xlsx_fixture(false, false), + ) + guard foreign_conditional.get("xl/worksheets/sheet.xml") + is Some(conditional_bytes) else { + fail("missing conditional-format fixture part") + } + let conditional_xml = @encoding/utf8.decode(conditional_bytes) catch { + _ => fail("conditional-format fixture is not UTF-8") + } + assert_true( + foreign_conditional.replace( + "xl/worksheets/sheet.xml", + @encoding/utf8.encode( + conditional_xml.replace( + old="", + new="1", + ), + ), + ), + ) + let conditional_parsed = @xlsx.read(@zip.write(foreign_conditional)) + guard conditional_parsed.sheet("Qualified") is Some(conditional_sheet) else { + fail("missing parsed conditional-format sheet") + } + assert_true(conditional_sheet.get_conditional_formats().is_empty()) + let foreign_strings = @zip.read( namespace_qualified_xlsx_fixture(false, false), ) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 3469e6ac..ff5a735a 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -7,6 +7,27 @@ let xlsx_extension_namespace_x15 = "http://schemas.microsoft.com/office/spreadsh ///| let xlsx_extension_namespace_xm = "http://schemas.microsoft.com/office/excel/2006/main" +///| +let transitional_spreadsheet_drawing_namespace = "http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" + +///| +let strict_spreadsheet_drawing_namespace = "http://purl.oclc.org/ooxml/drawingml/spreadsheetDrawing" + +///| +let transitional_drawing_main_namespace = "http://schemas.openxmlformats.org/drawingml/2006/main" + +///| +let strict_drawing_main_namespace = "http://purl.oclc.org/ooxml/drawingml/main" + +///| +let transitional_drawing_chart_namespace = "http://schemas.openxmlformats.org/drawingml/2006/chart" + +///| +let strict_drawing_chart_namespace = "http://purl.oclc.org/ooxml/drawingml/chart" + +///| +let drawing_slicer_namespace = "http://schemas.microsoft.com/office/drawing/2010/slicer" + ///| /// Converts namespace-qualified OOXML element names into the lexical form used /// by the existing bounded feature parsers. The source is fully namespace- and @@ -82,6 +103,88 @@ fn xlsx_core_source_without_foreign( } } +///| +/// Produces the extension-aware worksheet view used by known feature parsers. +/// Core SpreadsheetML plus the explicitly supported x14/x15/xm namespaces are +/// retained; every other namespace subtree is removed before lexical parsing, +/// including a foreign default namespace that reuses a core local name. +fn canonicalize_xlsx_worksheet_features( + source : StringView, + max_output_chars : Int, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(source, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "worksheet" || + ( + scanner.namespace_uri() != transitional_spreadsheet_namespace && + scanner.namespace_uri() != strict_spreadsheet_namespace + ) { + raise InvalidXml(msg="worksheet document element is invalid") + } + let dialect = scanner.namespace_uri().to_owned() + let filtered = @ooxml.remove_xml_foreign_namespace_subtrees( + source, + [ + dialect, xlsx_extension_namespace_x14, xlsx_extension_namespace_x15, xlsx_extension_namespace_xm, + ], + cancelled~, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + canonicalize_xlsx_xml( + filtered.unwrap_or(source.to_owned()), + max_output_chars~, + cancelled~, + ) +} + +///| +/// Canonicalizes DrawingML by expanded namespace identity. Prefix aliases and +/// default namespaces are normalized to the lexical names used by the existing +/// drawing feature readers, while a foreign namespace borrowing `xdr`, `a`, +/// `c`, or `sle` is defanged by the bounded XML canonicalizer. +fn canonicalize_xlsx_drawing_xml( + source : StringView, + max_output_chars : Int, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(source, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "wsDr" || + ( + scanner.namespace_uri() != transitional_spreadsheet_drawing_namespace && + scanner.namespace_uri() != strict_spreadsheet_drawing_namespace + ) { + raise InvalidXml(msg="drawing document element is invalid") + } + @ooxml.canonicalize_xml_expanded_names( + source, + [], + [ + (transitional_relationship_attribute_namespace, "r"), + (strict_relationship_attribute_namespace, "r"), + ], + max_output_chars~, + cancelled~, + element_prefixes=[ + (transitional_spreadsheet_drawing_namespace, "xdr"), + (strict_spreadsheet_drawing_namespace, "xdr"), + (transitional_drawing_main_namespace, "a"), + (strict_drawing_main_namespace, "a"), + (transitional_drawing_chart_namespace, "c"), + (strict_drawing_chart_namespace, "c"), + (drawing_slicer_namespace, "sle"), + ], + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } +} + ///| fn validate_xlsx_worksheet_core( xml : StringView, @@ -264,6 +367,26 @@ test "XLSX XML canonicalization follows extension namespace identity" { assert_true(canonical.contains("<_foreign_x14_dataValidations/>")) } +///| +test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" { + let valid = + #| + let canonical = canonicalize_xlsx_drawing_xml( + valid, default_max_xml_part_bytes, + ) + assert_true(canonical.contains("")) + assert_true(canonical.contains("")) + assert_true(canonical.contains("")) + let spoofed = + #| + assert_true( + canonicalize_xlsx_drawing_xml(spoofed, default_max_xml_part_bytes).contains( + "<_foreign_xdr_oneCellAnchor ", + ), + ) +} + ///| test "XLSX XML canonicalization propagates cancellation from a long pass" { let checks = [0] From 7442596fddaa9493dd6bc54c7a63a9f2fa78fbfb Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:29:16 +0800 Subject: [PATCH 066/150] fix(xlsx): avoid shared-formula bulk copies --- office/cmd/office/xlsx_read_model.mbt | 24 ++++++---------------- office/cmd/office/xlsx_read_wbtest.mbt | 3 --- xlsx/shared_formula_ranges.mbt | 28 ++++++++++++++++++++++++++ xlsx/worksheet.mbt | 13 ++++++++++++ 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index ac1c3757..283e4bb5 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -28,8 +28,6 @@ struct XlsxSheetTarget { index : Int path : String content : XlsxSheetContent - shared_formula_masters : Map[UInt, @xlsx.SharedFormulaMaster] - mut shared_formulas_indexed : Bool } ///| @@ -259,8 +257,6 @@ fn make_xlsx_projection( index, path: canonical_xlsx_sheet_path(name), content, - shared_formula_masters: Map([]), - shared_formulas_indexed: content is ChartSheet(_), } sheet_index[normalized_name] = index sheets.push(target) @@ -417,9 +413,8 @@ fn XlsxSheetTarget::worksheet( } ///| -/// Resolves shared-formula masters on demand. Parsed worksheets retain the -/// validated master index built under the XLSX read-work budget, so the first -/// follower copies only the usually-small master map and later lookups are O(1). +/// Resolves one shared-formula master through the worksheet's validated index +/// without cloning that index into command-owned storage. fn XlsxSheetTarget::shared_formula_master( self : XlsxSheetTarget, projection : XlsxProjection, @@ -427,17 +422,10 @@ fn XlsxSheetTarget::shared_formula_master( shared_index : UInt, ) -> @xlsx.SharedFormulaMaster raise CliFailure { projection.checkpoint() - if !self.shared_formulas_indexed { - let masters = xlsx_call(projection.file, () => { - worksheet.shared_formula_masters() - }) - for index, formula in masters { - projection.checkpoint() - self.shared_formula_masters[index] = formula - } - self.shared_formulas_indexed = true - } - match self.shared_formula_masters.get(shared_index) { + match + xlsx_call(projection.file, () => { + worksheet.shared_formula_master(shared_index) + }) { Some(master) => master None => raise xlsx_cli_failure( diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 7187a79b..d497b065 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -673,7 +673,6 @@ async test "XLSX snapshots preserve and resolve shared-formula followers" { opts=@xlsx.FormulaOpts::shared("A1:C1"), ) let projection = make_xlsx_projection("shared.xlsx", workbook, 10) - assert_false(projection.sheets[0].shared_formulas_indexed) let under = Some( resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]"), ) @@ -689,7 +688,6 @@ async test "XLSX snapshots preserve and resolve shared-formula followers" { } assert_eq(fields.get("matched_total"), Some(Json::number(3))) assert_eq(matches.length(), 3) - assert_true(projection.sheets[0].shared_formulas_indexed) assert_true(matches[0].stringify().contains("\"formula\":\"D1+1\"")) assert_true(matches[1].stringify().contains("\"formula\":\"E1+1\"")) assert_true(matches[2].stringify().contains("\"formula\":\"F1+1\"")) @@ -768,7 +766,6 @@ async test "XLSX formatting is deferred and custom-format work is charged" { ) assert_eq(page.matched_total, 2) assert_eq(page.cells.length(), 0) - assert_false(projection.sheets[0].shared_formulas_indexed) let limited = xlsx_cli_error_async(() => { ignore( collect_xlsx_cell_page( diff --git a/xlsx/shared_formula_ranges.mbt b/xlsx/shared_formula_ranges.mbt index 4806c403..4fbb6cf7 100644 --- a/xlsx/shared_formula_ranges.mbt +++ b/xlsx/shared_formula_ranges.mbt @@ -36,6 +36,15 @@ fn sort_shared_formula_range_events( if events.length() < 2 { return } + match budget { + Some(value) => { + // Account for and observe cancellation before allocating/copying the + // O(events) merge scratch buffer. + value.checkpoint() + value.charge_work(events.length()) + } + None => () + } let scratch = events.copy() let mut width = 1 while width < events.length() { @@ -345,3 +354,22 @@ test "shared formula event sorting polls cancellation inside long passes" { assert_eq(checks[0], 4) assert_true(result is Err(ReadCancelled)) } + +///| +test "shared formula sorting checks cancellation before scratch allocation" { + let events : Array[SharedFormulaRangeEvent] = [] + for index in 0..<(16 * 1024) { + events.push({ row: index, column_lo: 1, column_hi: 1, delta: 1 }) + } + let checks = [0] + let budget = ReadBudget::new(ReadLimits::new(), cancelled=() => { + checks[0] += 1 + true + }) + try sort_shared_formula_range_events(events, budget~) catch { + ReadCancelled => assert_eq(checks[0], 1) + _ => fail("expected cancellation before shared-formula scratch allocation") + } noraise { + _ => fail("expected shared-formula sort cancellation") + } +} diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index def80366..d0f49896 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -533,6 +533,18 @@ pub fn Worksheet::shared_formula_masters( clone_shared_formula_masters_index(self.shared_formula_masters_index) } +///| +/// Looks up one validated shared-formula master without cloning the complete +/// master index. Parsed worksheets already carry an eagerly validated index; +/// programmatically edited worksheets rebuild it lazily before this lookup. +pub fn Worksheet::shared_formula_master( + self : Worksheet, + shared_index : UInt, +) -> SharedFormulaMaster? raise XlsxError { + self.ensure_shared_formula_masters_index() + self.shared_formula_masters_index.get(shared_index) +} + ///| test "shared formula masters stay eagerly indexed after bounded read" { let source = Workbook::new() @@ -551,6 +563,7 @@ test "shared formula masters stay eagerly indexed after bounded read" { let exported = sheet.shared_formula_masters() exported.clear() assert_eq(sheet.shared_formula_masters_index.length(), 1) + assert_true(sheet.shared_formula_master(0) is Some(_)) } ///| From 4a12a5fb19c2eee5fcbb928f38c297c9932d868d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:38:33 +0800 Subject: [PATCH 067/150] fix(xlsx): bound read preprocessing --- office/cmd/office/xlsx_read_model.mbt | 21 +++-- office/cmd/office/xlsx_read_output.mbt | 19 ++--- office/cmd/office/xlsx_read_wbtest.mbt | 17 ++++ xlsx/read.mbt | 17 +++- xlsx/read_drawing_xml.mbt | 107 +++++++++++++++++++++---- xlsx/worksheet.mbt | 40 +++++++++ 6 files changed, 188 insertions(+), 33 deletions(-) diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 283e4bb5..eca75463 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -328,9 +328,20 @@ fn xlsx_rect_from_coordinate(coordinate : @lib.SelectorCoordinate) -> XlsxRect { } ///| -fn xlsx_used_rect(worksheet : @xlsx.Worksheet) -> XlsxRect? { - let max_col = worksheet.max_col() - let max_row = worksheet.max_row() +fn xlsx_used_rect( + projection : XlsxProjection, + worksheet : @xlsx.Worksheet, +) -> XlsxRect? raise CliFailure { + projection.checkpoint() + let (max_row, max_col) = worksheet.used_bounds_limited( + maximum_stored_cells=projection.max_scan_cells, + cancelled=projection.cancelled, + ) catch { + ReadCancelled => + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + error => raise xlsx_read_failure(error, projection.file) + } + projection.checkpoint() if max_col < 1 || max_row < 1 { None } else { @@ -514,7 +525,7 @@ fn xlsx_scan_regions( Some(Sheet(sheet)) => match sheet.content { Worksheet(worksheet) => - match xlsx_used_rect(worksheet) { + match xlsx_used_rect(projection, worksheet) { Some(rect) => regions.push({ sheet, rect }) None => () } @@ -525,7 +536,7 @@ fn xlsx_scan_regions( projection.checkpoint() match sheet.content { Worksheet(worksheet) => - match xlsx_used_rect(worksheet) { + match xlsx_used_rect(projection, worksheet) { Some(rect) => regions.push({ sheet, rect }) None => () } diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 348a6874..b3d1a55a 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -118,12 +118,10 @@ fn xlsx_sheet_summary_json( Worksheet(worksheet) => { fields["kind"] = Json::string("worksheet") fields["state"] = Json::string(xlsx_sheet_state_name(worksheet.state())) - let max_row = worksheet.max_row() - let max_col = worksheet.max_col() - fields["max_row"] = Json::number(max_row.to_double()) - fields["max_column"] = Json::number(max_col.to_double()) - match xlsx_used_rect(worksheet) { + match xlsx_used_rect(projection, worksheet) { Some(rect) => { + fields["max_row"] = Json::number(rect.row_hi.to_double()) + fields["max_column"] = Json::number(rect.col_hi.to_double()) let reference = rect.range_reference(projection.file) let path = canonical_xlsx_range_path(sheet.name, reference) metadata.charge_item([reference, path]) @@ -133,7 +131,10 @@ fn xlsx_sheet_summary_json( "cell_count": Json::number(rect.cell_count().to_double()), }) } - None => () + None => { + fields["max_row"] = Json::number(0) + fields["max_column"] = Json::number(0) + } } let merges = worksheet.merged_cells() projection.checkpoint() @@ -179,7 +180,7 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) } let defined_names : Array[Json] = [] - for defined in projection.workbook.get_defined_names() { + for defined in projection.workbook.defined_names() { projection.checkpoint() metadata.charge_item([ defined.name, @@ -533,7 +534,7 @@ fn retain_xlsx_outline_records( budget.reserve_array_item(record, !sheets.is_empty()) sheets.push(record) } - for defined in projection.workbook.get_defined_names() { + for defined in projection.workbook.defined_names() { projection.checkpoint() metadata.charge_item([ defined.name, @@ -903,7 +904,7 @@ fn xlsx_outline_human( ChartSheet(_) => output.write_plain("\tchart-sheet") Worksheet(worksheet) => { output.write_plain("\tworksheet") - match xlsx_used_rect(worksheet) { + match xlsx_used_rect(projection, worksheet) { Some(rect) => { output.write_char('\t') output.write_terminal_safe(rect.range_reference(projection.file)) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index d497b065..aed3f38a 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -430,6 +430,23 @@ test "XLSX selector failures distinguish format, absence, and scan limits" { assert_eq(fields.get("actual"), Some(Json::number(110))) } +///| +test "XLSX used-range preprocessing rejects excess stored cells before scanning" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_str("Data", "A1", "first") + workbook.set_cell_str("Data", "B1", "second") + let projection = make_xlsx_projection("stored-cells.xlsx", workbook, 1) + let limited = xlsx_cli_error(() => ignore(xlsx_scan_regions(projection, None))) + assert_eq(limited.code, "office.xlsx.resource_limit") + guard limited.details is Some(Object(details)) else { + fail("expected stored-cell limit details") + } + assert_eq(details.get("resource"), Some(Json::string("stored_cells"))) + assert_eq(details.get("limit"), Some(Json::number(1))) + assert_eq(details.get("actual"), Some(Json::number(2))) +} + ///| test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" { let projection = xlsx_read_test_projection() diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 7f73cb64..018f0e85 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4313,9 +4313,15 @@ fn read_zip_archive_core_unchecked( read_budget.checkpoint() read_budget.charge_work(drawing_xml.length()) let drawing_metrics = drawing_metrics_for_sheet(sheet) + let drawing_anchors = parse_drawing_anchors( + drawing_xml, + budget=read_budget, + ) if slicer_part_entries.length() > 0 { slicer_anchor_info = parse_drawing_slicer_anchors( - drawing_xml, drawing_metrics, + drawing_anchors, + drawing_metrics, + budget=read_budget, ) } let drawing_rels_xml = match @@ -4324,7 +4330,7 @@ fn read_zip_archive_core_unchecked( None => "" } let drawing_images = parse_drawing_images( - drawing_xml, + drawing_anchors, drawing_rels_xml, drawing_path, part_names, @@ -4335,7 +4341,7 @@ fn read_zip_archive_core_unchecked( ) sheet.images.append(drawing_images) let drawing_charts = parse_drawing_charts( - drawing_xml, + drawing_anchors, drawing_rels_xml, drawing_path, part_names, @@ -4346,7 +4352,10 @@ fn read_zip_archive_core_unchecked( cancelled~, ) sheet.charts.append(drawing_charts) - let drawing_shapes = parse_drawing_shapes(drawing_xml) + let drawing_shapes = parse_drawing_shapes( + drawing_anchors, + budget=read_budget, + ) sheet.shapes.append(drawing_shapes) } None => () diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index c1af42bb..89490c27 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -108,15 +108,30 @@ fn drawing_axis_span_emu( ///| fn parse_drawing_anchors( drawing_xml : StringView, + budget? : ReadBudget, ) -> Array[(String, PicturePositioning)] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(drawing_xml.length()) + } + None => () + } let xml_str = drawing_xml.to_owned() let anchors : Array[(String, PicturePositioning)] = [] + let mut parsed = 0 let mut first_one = true for chunk in xml_str.split(" value.checkpoint() + None => () + } + } let text = chunk.to_owned() let close_tag = "" let end = match text.find(close_tag) { @@ -125,6 +140,7 @@ fn parse_drawing_anchors( } let slice = text[:end] anchors.push((" value.checkpoint() + None => () + } + } let text = chunk.to_owned() let close_tag = "" let end = match text.find(close_tag) { @@ -154,13 +176,32 @@ fn parse_drawing_anchors( None => TwoCell } anchors.push((full, positioning)) + parsed += 1 + } + match budget { + Some(value) => value.checkpoint() + None => () } anchors } +///| +fn charge_drawing_anchor_pass( + budget : ReadBudget?, + anchor_xml : String, +) -> Unit raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(anchor_xml.length()) + } + None => () + } +} + ///| fn parse_drawing_images( - drawing_xml : StringView, + anchors : ArrayView[(String, PicturePositioning)], drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], @@ -190,9 +231,9 @@ fn parse_drawing_images( cancelled~, ) } - let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { let (anchor_xml, positioning) = entry + charge_drawing_anchor_pass(budget, anchor_xml) let pic_body = match extract_tag_body_from(anchor_xml, "xdr:pic") { Some(value) => value None => continue @@ -377,7 +418,7 @@ fn parse_drawing_images( ///| fn parse_drawing_charts( - drawing_xml : StringView, + anchors : ArrayView[(String, PicturePositioning)], drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], @@ -398,9 +439,9 @@ fn parse_drawing_charts( cancelled~, ) } - let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { let (anchor_xml, positioning) = entry + charge_drawing_anchor_pass(budget, anchor_xml) let chart_tag = match tag_attributes_in(anchor_xml, "c:chart") { Some(value) => value None => continue @@ -529,12 +570,13 @@ fn parse_drawing_charts( ///| fn parse_drawing_shapes( - drawing_xml : StringView, + anchors : ArrayView[(String, PicturePositioning)], + budget? : ReadBudget, ) -> Array[Shape] raise XlsxError { let shapes : Array[Shape] = [] - let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { let (anchor_xml, positioning) = entry + charge_drawing_anchor_pass(budget, anchor_xml) if anchor_xml.contains(" Map[String, SlicerAnchorInfo] raise XlsxError { let out : Map[String, SlicerAnchorInfo] = Map([]) - let anchors = parse_drawing_anchors(drawing_xml) for entry in anchors { let (anchor_xml, positioning) = entry + charge_drawing_anchor_pass(budget, anchor_xml) let slicer_tag = match tag_attributes_in(anchor_xml, "sle:slicer") { Some(value) => value None => continue @@ -1087,6 +1130,40 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { debug_inspect(anchors[1].1, content="TwoCell") } +///| +test "read_drawing_xml wb: anchor parsing checks limits before copying source" { + let drawing_xml = "" + let limited = ReadBudget::new( + ReadLimits::with_values(max_parser_work_units=1), + ) + let limit_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + parse_drawing_anchors(drawing_xml, budget=limited), + ) catch { + error => Err(error) + } + match limit_result { + Err(ResourceLimitExceeded(kind~, limit~, actual~)) => { + inspect(kind, content="parser_work_units") + inspect(limit, content="1") + inspect(actual, content="2") + } + _ => fail("expected drawing work limit") + } + + let mut checks = 0 + let cancelled = ReadBudget::new(ReadLimits::new(), cancelled=() => { + checks += 1 + true + }) + let cancel_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + parse_drawing_anchors(drawing_xml, budget=cancelled), + ) catch { + error => Err(error) + } + assert_true(cancel_result is Err(ReadCancelled)) + inspect(checks, content="1") +} + ///| test "read_drawing_xml wb: parse_drawing_images empty rel map branch" { let drawing_xml = wb_one_cell_anchor( @@ -1097,7 +1174,7 @@ test "read_drawing_xml wb: parse_drawing_images empty rel map branch" { let metrics = drawing_metrics_for_sheet(Worksheet::new("Sheet1")) let result : Result[Array[Image], Error] = Ok( parse_drawing_images( - drawing_xml, + parse_drawing_anchors(drawing_xml), "", "xl/drawings/drawing1.xml", Map([]), @@ -1125,7 +1202,7 @@ test "read_drawing_xml wb: signed marker offsets are retained" { ) let metrics = drawing_metrics_for_sheet(Worksheet::new("Sheet1")) let images = parse_drawing_images( - drawing_xml, + parse_drawing_anchors(drawing_xml), rels_xml, "xl/drawings/drawing1.xml", Map([]), @@ -1151,7 +1228,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { ) let missing_from_result : Result[Array[Image], Error] = Ok( parse_drawing_images( - missing_from, + parse_drawing_anchors(missing_from), rels_xml, "xl/drawings/drawing1.xml", Map([]), @@ -1170,7 +1247,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { let missing_ext = wb_one_cell_anchor(wb_from_xml(), "") let missing_ext_result : Result[Array[Image], Error] = Ok( parse_drawing_images( - missing_ext, + parse_drawing_anchors(missing_ext), rels_xml, "xl/drawings/drawing1.xml", Map([]), @@ -1192,7 +1269,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { ) let cx_invalid_result : Result[Array[Image], Error] = Ok( parse_drawing_images( - cx_invalid, + parse_drawing_anchors(cx_invalid), rels_xml, "xl/drawings/drawing1.xml", Map([]), @@ -1211,7 +1288,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { let cy_missing = wb_one_cell_anchor(wb_from_xml(), "") let cy_missing_result : Result[Array[Image], Error] = Ok( parse_drawing_images( - cy_missing, + parse_drawing_anchors(cy_missing), rels_xml, "xl/drawings/drawing1.xml", Map([]), @@ -1233,7 +1310,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { ) let invalid_to_result : Result[Array[Image], Error] = Ok( parse_drawing_images( - invalid_to, + parse_drawing_anchors(invalid_to), rels_xml, "xl/drawings/drawing1.xml", Map([]), diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index d0f49896..54e5b139 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2770,6 +2770,46 @@ pub fn Worksheet::max_col(self : Worksheet) -> Int { max_col } +///| +/// Returns the used `(max_row, max_column)` bounds after rejecting a worksheet +/// whose stored-cell representation exceeds `maximum_stored_cells`. The O(1) +/// length preflight happens before the scan, so duplicate coordinates cannot +/// hide unbounded preprocessing behind a tiny logical used range. +pub fn Worksheet::used_bounds_limited( + self : Worksheet, + maximum_stored_cells~ : Int, + cancelled? : () -> Bool = () => false, +) -> (Int, Int) raise XlsxError { + if maximum_stored_cells < 1 { + raise InvalidOptions(msg="maximum stored cells must be positive") + } + let stored = self.cells.length() + if stored > maximum_stored_cells { + raise ResourceLimitExceeded( + kind="stored_cells", + limit=maximum_stored_cells, + actual=stored, + ) + } + let mut max_row = 0 + let mut max_col = 0 + for index, cell in self.cells { + if (index & 4095) == 0 && cancelled() { + raise ReadCancelled + } + if cell.row > max_row { + max_row = cell.row + } + if cell.col > max_col { + max_col = cell.col + } + } + if cancelled() { + raise ReadCancelled + } + (max_row, max_col) +} + ///| /// Writes `values` across row `row` (1-based) starting at column A, one raw /// string per cell. Cells beyond `values` are left unchanged. For a single From e485dc6d964e1cc2f99a037ab9e89a509462f394 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:41:24 +0800 Subject: [PATCH 068/150] fix(docx): poll relationship validation --- docx2html/opc/validate.mbt | 179 +++++++++++++++++++----------- docx2html/opc/validate_wbtest.mbt | 42 +++++++ 2 files changed, 156 insertions(+), 65 deletions(-) create mode 100644 docx2html/opc/validate_wbtest.mbt diff --git a/docx2html/opc/validate.mbt b/docx2html/opc/validate.mbt index b1705b62..d0ea3257 100644 --- a/docx2html/opc/validate.mbt +++ b/docx2html/opc/validate.mbt @@ -429,6 +429,20 @@ fn FindingCollector::stop(self : FindingCollector) -> Unit { self.truncated = true } +///| +/// Polls the caller's cooperative cancellation hook. Cancellation is recorded +/// as a stopped/truncated validation rather than a finding so command callers +/// can translate their own cancellation state without attacker-controlled +/// diagnostics obscuring it. +fn FindingCollector::checkpoint(self : FindingCollector) -> Bool { + if (self.cancelled)() { + self.stop() + false + } else { + true + } +} + ///| fn default_findings(result : (Array[String], Bool)) -> Array[String] { let (findings, truncated) = result @@ -1379,54 +1393,56 @@ fn check_relationships( match parse_relationships_part(root_rels_name, bytes, findings) { Some(root) => { let main_parts : Array[(String, DocumentDialect)] = [] - for_each_relationship(ROOT_RELS_PART, root, findings, ( - _id, - rel_type, - target, - external, - ) => { - let dialect = office_document_dialect(rel_type) - if external { - match dialect { - Some(_) => - findings.add( - "\{ROOT_RELS_PART}: officeDocument relationship is not internal", - ) - None => () + if !for_each_relationship(ROOT_RELS_PART, root, findings, ( + _id, + rel_type, + target, + external, + ) => { + let dialect = office_document_dialect(rel_type) + if external { + match dialect { + Some(_) => + findings.add( + "\{ROOT_RELS_PART}: officeDocument relationship is not internal", + ) + None => () + } + return } - return - } - match resolve_part_target("", target) { - Some(resolved) => - if is_relationship_part_name(resolved) { - findings.add( - "relationship target must not be a Relationships part: \{ROOT_RELS_PART} -> \{findings.text(target)}", - ) - } else { - if !contains_part(canonical_names, resolved) { + match resolve_part_target("", target) { + Some(resolved) => + if is_relationship_part_name(resolved) { findings.add( - "relationship target missing: \{ROOT_RELS_PART} -> \{findings.text(target)} (resolved \{findings.text(resolved)})", + "relationship target must not be a Relationships part: \{ROOT_RELS_PART} -> \{findings.text(target)}", ) + } else { + if !contains_part(canonical_names, resolved) { + findings.add( + "relationship target missing: \{ROOT_RELS_PART} -> \{findings.text(target)} (resolved \{findings.text(resolved)})", + ) + } + match dialect { + Some(value) => main_parts.push((resolved, value)) + None => () + } } + None => { + findings.add( + "relationship target invalid: \{ROOT_RELS_PART} -> \{findings.text(target)}", + ) match dialect { - Some(value) => main_parts.push((resolved, value)) + Some(_) => + findings.add( + "\{ROOT_RELS_PART}: officeDocument relationship has an invalid target: \{findings.text(target)}", + ) None => () } } - None => { - findings.add( - "relationship target invalid: \{ROOT_RELS_PART} -> \{findings.text(target)}", - ) - match dialect { - Some(_) => - findings.add( - "\{ROOT_RELS_PART}: officeDocument relationship has an invalid target: \{findings.text(target)}", - ) - None => () - } } - } - }) + }) { + return + } match main_parts { [(part, dialect)] => match find_part(parts, canonical_names, part) { @@ -1487,31 +1503,33 @@ fn check_relationships( match parse_relationships_part(name, bytes, findings) { Some(root) => { let base = rels_base_dir(logical_name) - for_each_relationship(name, root, findings, ( - _id, - _rel_type, - target, - external, - ) => { - if !external { - match resolve_part_target(base, target) { - Some(resolved) => - if is_relationship_part_name(resolved) { - findings.add( - "relationship target must not be a Relationships part: \{findings.text(name)} -> \{findings.text(target)}", - ) - } else if !contains_part(canonical_names, resolved) { + if !for_each_relationship(name, root, findings, ( + _id, + _rel_type, + target, + external, + ) => { + if !external { + match resolve_part_target(base, target) { + Some(resolved) => + if is_relationship_part_name(resolved) { + findings.add( + "relationship target must not be a Relationships part: \{findings.text(name)} -> \{findings.text(target)}", + ) + } else if !contains_part(canonical_names, resolved) { + findings.add( + "relationship target missing: \{findings.text(name)} -> \{findings.text(target)} (resolved \{findings.text(resolved)})", + ) + } + None => findings.add( - "relationship target missing: \{findings.text(name)} -> \{findings.text(target)} (resolved \{findings.text(resolved)})", + "relationship target invalid: \{findings.text(name)} -> \{findings.text(target)}", ) - } - None => - findings.add( - "relationship target invalid: \{findings.text(name)} -> \{findings.text(target)}", - ) + } } - } - }) + }) { + return + } } None => () } @@ -1596,17 +1614,27 @@ fn for_each_relationship( root : @xml.XmlElement, findings : FindingCollector, visit : (String, String, String, Bool) -> Unit, -) -> Unit { +) -> Bool { + if !findings.checkpoint() { + return false + } let relationships_name = "{\{PACKAGE_RELATIONSHIPS_NAMESPACE}}Relationships" let relationship_name = "{\{PACKAGE_RELATIONSHIPS_NAMESPACE}}Relationship" if root.name != relationships_name { findings.add( "\{findings.text(part_name)}: root element is not in the OPC package relationships namespace", ) - return + return true } let mut valid_part = true for attribute, _ in root.attributes { + if !findings.checkpoint() { + return false + } + if findings.full() { + findings.stop() + return false + } findings.add( "\{findings.text(part_name)}: Relationships has undeclared attribute: \{findings.text(attribute)}", ) @@ -1615,10 +1643,12 @@ fn for_each_relationship( let ids : StableStringSet = SortedSet([]) let relationships : Array[(String, String, String, Bool)] = [] for node in root.children { + if !findings.checkpoint() { + return false + } if findings.full() { findings.stop() - valid_part = false - break + return false } guard node is XmlElement(element) else { guard node is XmlText(text) else { continue } @@ -1645,6 +1675,9 @@ fn for_each_relationship( } let mut valid = true for attribute, _ in element.attributes { + if !findings.checkpoint() { + return false + } if attribute != "Id" && attribute != "Type" && attribute != "Target" && @@ -1656,6 +1689,9 @@ fn for_each_relationship( } } for child in element.children { + if !findings.checkpoint() { + return false + } match child { XmlElement(_) => { findings.add( @@ -1669,6 +1705,9 @@ fn for_each_relationship( XmlText(_) => () } } + if !findings.checkpoint() { + return false + } let id = @xml.collapse_xml_schema_whitespace( element.attributes.get("Id").unwrap_or(""), ) @@ -1690,6 +1729,9 @@ fn for_each_relationship( } else { ids.add(id) } + if !findings.checkpoint() { + return false + } let rel_type = @xml.collapse_xml_schema_whitespace( element.attributes.get("Type").unwrap_or(""), ) @@ -1704,6 +1746,9 @@ fn for_each_relationship( ) valid = false } + if !findings.checkpoint() { + return false + } let target = @xml.collapse_xml_schema_whitespace( element.attributes.get("Target").unwrap_or(""), ) @@ -1749,10 +1794,14 @@ fn for_each_relationship( } if valid_part { for relationship in relationships { + if !findings.checkpoint() { + return false + } let (id, rel_type, target, external) = relationship visit(id, rel_type, target, external) } } + findings.checkpoint() } ///| diff --git a/docx2html/opc/validate_wbtest.mbt b/docx2html/opc/validate_wbtest.mbt new file mode 100644 index 00000000..182374f5 --- /dev/null +++ b/docx2html/opc/validate_wbtest.mbt @@ -0,0 +1,42 @@ +///| +test "relationship semantic validation polls cancellation within elements" { + let source = + #| + #| + #| + let root = read_opc_relationships_xml_limited( + @utf8.encode(source), + @xml.xml_read_budget( + max_source_units=4096, + max_tokens=64, + max_materialized_chars=4096, + max_token_chars=4096, + ), + ) + let mut checks = 0 + let findings = FindingCollector::new_with_xml_limits( + 8, + 256, + 256, + 1024, + 4096, + 64, + 4096, + 4096, + () => { + checks += 1 + checks >= 4 + }, + ) + let mut visits = 0 + let completed = for_each_relationship("_rels/.rels", root, findings, ( + _id, + _kind, + _target, + _external, + ) => visits += 1) + assert_false(completed) + assert_eq(checks, 4) + assert_eq(visits, 0) + assert_true(findings.report().findings_truncated()) +} From 96f8592954f6bec42fa6688ab226d28f64506665 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:43:34 +0800 Subject: [PATCH 069/150] fix(zip): poll while initializing owned copies --- zip/reader.mbt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/zip/reader.mbt b/zip/reader.mbt index 86e8f38d..c106b8d9 100644 --- a/zip/reader.mbt +++ b/zip/reader.mbt @@ -123,14 +123,15 @@ fn copy_zip_bytes_cancellable( cancelled : () -> Bool, ) -> Bytes raise ZipError { check_zip_cancelled(cancelled) - let output = FixedArray::make(bytes.length(), b'\x00') - let mut offset = 0 - while offset < bytes.length() { - let end = (offset + 64 * 1024).min(bytes.length()) - output.blit_from_bytesview(offset, bytes[offset:end]) - offset = end - check_zip_cancelled(cancelled) - } + // Initialize directly from the source. `FixedArray::make` would zero-fill + // the complete destination before the first chunk checkpoint, defeating the + // cancellation guarantee for a permitted 32-MiB stored entry. + let output = FixedArray::makei(bytes.length(), index => { + if (index & 0xffff) == 0 { + check_zip_cancelled(cancelled) + } + bytes[index] + }) check_zip_cancelled(cancelled) output.unsafe_reinterpret_as_bytes() } From 52f40ab7eff58047164bdda2a8d6db6e80f656e1 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:48:13 +0800 Subject: [PATCH 070/150] fix(xlsx): reject derivable OPC part names --- ooxml/moon.pkg | 1 + ooxml/opc_part_name.mbt | 126 ++++++++++++++++++++++++++++++++++++++ xlsx/read.mbt | 34 ++++++++++ xlsx/read_limits_test.mbt | 24 ++++++++ 4 files changed, 185 insertions(+) diff --git a/ooxml/moon.pkg b/ooxml/moon.pkg index 45cfbe27..c4e4f9b7 100644 --- a/ooxml/moon.pkg +++ b/ooxml/moon.pkg @@ -2,4 +2,5 @@ import { "moonbitlang/core/debug", "moonbitlang/core/string", "moonbitlang/core/encoding/utf8", + "moonbitlang/core/sorted_map", } diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt index f480dfd9..a4377f67 100644 --- a/ooxml/opc_part_name.mbt +++ b/ooxml/opc_part_name.mbt @@ -224,6 +224,105 @@ pub fn logical_opc_part_name_from_zip_item_name(name : StringView) -> String? { Some(logical) } +///| +type OpcPartNameChildren = @sorted_map.SortedMap[String, Int] + +///| +priv struct OpcPartNameTrieNode { + mut terminal : String? + mut first_part : String? + children : OpcPartNameChildren +} + +///| +fn new_opc_part_name_trie_node() -> OpcPartNameTrieNode { + { terminal: None, first_part: None, children: SortedMap([]) } +} + +///| +/// The collision reported while registering a logical OPC PartName. +pub(all) enum OpcPartNameConflict { + Equivalent(String) + Derivable(String) +} + +///| +/// A segment trie for OPC PartName identity and non-derivability checks. +/// Work and storage are linear in aggregate PartName length, while ordered +/// child indexes keep hostile names independent of string-hash behavior. +pub struct OpcPartNameRegistry { + priv nodes : Array[OpcPartNameTrieNode] +} + +///| +/// Creates an empty OPC PartName registry. +pub fn OpcPartNameRegistry::new() -> OpcPartNameRegistry { + { nodes: [new_opc_part_name_trie_node()] } +} + +///| +/// Registers `name`, using `display` in any returned conflict. Identity is +/// ASCII-case-insensitive and a name conflicts when either side is derivable +/// from the other by appending one or more path segments. +pub fn OpcPartNameRegistry::register( + self : OpcPartNameRegistry, + name : StringView, + display : String, +) -> OpcPartNameConflict? { + let key = package_part_name_key(name) + let segments = key.split("/").collect() + let mut node_index = 0 + let mut complete_path = true + for segment_view in segments { + match self.nodes[node_index].terminal { + Some(existing) => return Some(Derivable(existing)) + None => () + } + let segment = segment_view.to_owned() + match self.nodes[node_index].children.get(segment) { + Some(child) => node_index = child + None => { + complete_path = false + break + } + } + } + if complete_path { + match self.nodes[node_index].terminal { + Some(existing) => return Some(Equivalent(existing)) + None => () + } + match self.nodes[node_index].first_part { + Some(existing) => return Some(Derivable(existing)) + None => () + } + } + // The preflight is mutation-free, so a rejected name cannot leave a partial + // trie path that changes a later conflict result. + node_index = 0 + if self.nodes[node_index].first_part is None { + self.nodes[node_index].first_part = Some(display) + } + for segment_view in segments { + let segment = segment_view.to_owned() + let child = match self.nodes[node_index].children.get(segment) { + Some(existing) => existing + None => { + let fresh = self.nodes.length() + self.nodes.push(new_opc_part_name_trie_node()) + self.nodes[node_index].children[segment] = fresh + fresh + } + } + node_index = child + if self.nodes[node_index].first_part is None { + self.nodes[node_index].first_part = Some(display) + } + } + self.nodes[node_index].terminal = Some(display) + None +} + ///| test "OPC physical names restore Unicode logical identity" { debug_inspect( @@ -235,3 +334,30 @@ test "OPC physical names restore Unicode logical identity" { content="None", ) } + +///| +test "OPC PartName registry rejects equivalent and derivable identities" { + let registry = OpcPartNameRegistry::new() + assert_true(registry.register("custom/a.xml", "custom/a.xml") is None) + match registry.register("CUSTOM/A.XML", "CUSTOM/A.XML") { + Some(Equivalent(existing)) => inspect(existing, content="custom/a.xml") + _ => fail("expected equivalent OPC PartName") + } + match registry.register("custom/a.xml/child.xml", "custom/a.xml/child.xml") { + Some(Derivable(existing)) => inspect(existing, content="custom/a.xml") + _ => fail("expected descendant OPC PartName conflict") + } + + let descendant_first = OpcPartNameRegistry::new() + assert_true( + descendant_first.register( + "custom/a.xml/child.xml", "custom/a.xml/child.xml", + ) + is None, + ) + match descendant_first.register("custom/a.xml", "custom/a.xml") { + Some(Derivable(existing)) => + inspect(existing, content="custom/a.xml/child.xml") + _ => fail("expected ancestor OPC PartName conflict") + } +} diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 018f0e85..19241953 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3124,6 +3124,7 @@ fn archive_part_name_index( archive : @zip.Archive, ) -> Map[String, String] raise XlsxError { let names : Map[String, String] = Map([]) + let registry = @ooxml.OpcPartNameRegistry::new() for entry in archive.entries() { let physical_name = entry.name() if physical_name.has_suffix("/") { @@ -3143,6 +3144,17 @@ fn archive_part_name_index( } } let key = @ooxml.package_part_name_key(logical_name) + if logical_name != content_types_part_path { + match registry.register(logical_name, physical_name) { + Some(Equivalent(_)) => + raise InvalidPackage(msg="duplicate or equivalent OPC part name") + Some(Derivable(existing)) => + raise InvalidPackage( + msg="OPC part names are derivable: \{existing} and \{physical_name}", + ) + None => () + } + } match names.get(key) { Some(_) => raise InvalidPackage(msg="duplicate or equivalent OPC part name") @@ -3211,6 +3223,28 @@ test "read archive index rejects ambiguous and misspelled reserved entries" { } } +///| +test "read archive index rejects OPC-derivable parts in either order" { + for + entries in [ + ["custom/a.xml", "custom/a.xml/child.xml"], + ["custom/a.xml/child.xml", "custom/a.xml"], + ] { + let archive = @zip.Archive::new() + archive.add(content_types_part_path, b"manifest") + for name in entries { + archive.add(name, b"part") + } + try archive_part_name_index(archive) catch { + InvalidPackage(msg~) => + assert_true(msg.contains("OPC part names are derivable")) + _ => fail("unexpected derivable OPC part error") + } noraise { + _ => fail("expected derivable OPC part rejection") + } + } +} + ///| fn actual_archive_part_path( part_names : Map[String, String], diff --git a/xlsx/read_limits_test.mbt b/xlsx/read_limits_test.mbt index 93ec8774..892d8a2d 100644 --- a/xlsx/read_limits_test.mbt +++ b/xlsx/read_limits_test.mbt @@ -85,6 +85,30 @@ test "bounded archive and byte reads produce equivalent workbooks" { inspect(@xlsx.write(from_archive) == @xlsx.write(from_bytes), content="true") } +///| +test "direct XLSX reads reject OPC-derivable part names" { + for + entries in [ + ["custom/a.xml", "custom/a.xml/child.xml"], + ["custom/a.xml/child.xml", "custom/a.xml"], + ] { + let archive = @zip.read(bounded_workbook_fixture()) + for name in entries { + archive.add(name, b"part") + } + let result : Result[@xlsx.Workbook, Error] = Ok( + @xlsx.read(@zip.write(archive)), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.InvalidPackage(msg~)) => + assert_true(msg.contains("OPC part names are derivable")) + _ => fail("expected direct XLSX derivable-part rejection") + } + } +} + ///| test "archive read rejects compatibility and constructed archives" { let bytes = bounded_workbook_fixture() From b9b2c519f423511bf75ecf7ca9adc83ede97d519 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:48:58 +0800 Subject: [PATCH 071/150] docs(office): document compact JSON output --- docs/agent-json-schemas.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/agent-json-schemas.md b/docs/agent-json-schemas.md index 16d68bf8..8b171232 100644 --- a/docs/agent-json-schemas.md +++ b/docs/agent-json-schemas.md @@ -37,8 +37,11 @@ disagree, the tests are the source of truth and this document has a bug. values; unified `office.xlsx.*` payloads use 1-based sheet `index` values. Rows and columns are never sent as bare numbers; cell and range references are A1-style strings. -- Output is pretty-printed (2-space indent) UTF-8 with a trailing newline, - deterministic for a given workbook. +- Output is deterministic UTF-8 with a trailing newline. Standalone xlsx and + docx inspection commands use a 2-space indent; the unified `office` + `outline`, `get`, `text`, and `query` commands emit compact JSON so their + output ceiling can be accounted exactly. JSON Lines output is also compact, + one value per line. Consumers must not depend on insignificant whitespace. ## `xlsx.outline/1` — workbook structure map (`xlsx outline `) From 10bbcf6c9278e983784af19e7d97d2aeda341810 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 15:49:27 +0800 Subject: [PATCH 072/150] chore: refresh generated interfaces --- ooxml/pkg.generated.mbti | 11 +++++++++++ xlsx/pkg.generated.mbti | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 7adb4950..16e6d187 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -49,6 +49,17 @@ pub suberror ParseXmlError { pub fn ParseXmlError::to_repr(Self) -> @debug.Repr // Types and methods +pub(all) enum OpcPartNameConflict { + Equivalent(String) + Derivable(String) +} + +pub struct OpcPartNameRegistry { + // private fields +} +pub fn OpcPartNameRegistry::new() -> Self +pub fn OpcPartNameRegistry::register(Self, StringView, String) -> OpcPartNameConflict? + type WorkbookManifest pub fn WorkbookManifest::add_content_default(Self, String, String) -> Unit pub fn WorkbookManifest::add_content_override(Self, String, String) -> Unit diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 2b964a1c..77875d78 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1852,7 +1852,7 @@ pub fn Workbook::get_cell_value_raw(Self, StringView, StringView) -> CellValue? pub fn Workbook::get_cell_value_raw_rc(Self, StringView, Int, Int) -> CellValue? raise XlsxError pub fn Workbook::get_cell_value_rc(Self, StringView, Int, Int, raw? : Bool, options? : Options) -> String? raise XlsxError pub fn Workbook::get_cell_value_styled(Self, StringView, StringView, Int, options? : Options) -> String? raise XlsxError -pub fn Workbook::get_cell_value_styled_from_worksheet_rc(Self, Worksheet, Int, Int, Int, options? : Options) -> String? raise XlsxError +pub fn Workbook::get_cell_value_styled_from_worksheet_rc(Self, Worksheet, Int, Int, Int, options? : Options, max_output_chars? : Int) -> String? raise XlsxError pub fn Workbook::get_col(Self, StringView, Int) -> Array[String] raise XlsxError pub fn Workbook::get_col_outline_level(Self, StringView, Int) -> Int raise XlsxError pub fn Workbook::get_col_style(Self, StringView, Int) -> Int? raise XlsxError @@ -2183,6 +2183,7 @@ pub fn Worksheet::set_row_style(Self, Int, Int) -> Unit raise XlsxError pub fn Worksheet::set_row_visible(Self, Int, Bool) -> Unit raise XlsxError pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxError pub fn Worksheet::shapes(Self) -> ArrayView[Shape] +pub fn Worksheet::shared_formula_master(Self, UInt) -> SharedFormulaMaster? raise XlsxError pub fn Worksheet::shared_formula_masters(Self) -> Map[UInt, SharedFormulaMaster] raise XlsxError pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? @@ -2193,6 +2194,7 @@ pub fn Worksheet::tables(Self) -> ArrayView[Table] pub fn Worksheet::unmerge_cells(Self, String) -> Unit raise XlsxError pub fn Worksheet::unprotect_sheet(Self, password? : String) -> Unit raise XlsxError pub fn Worksheet::unset_conditional_format(Self, String) -> Unit raise XlsxError +pub fn Worksheet::used_bounds_limited(Self, maximum_stored_cells~ : Int, cancelled? : () -> Bool) -> (Int, Int) raise XlsxError pub fn Worksheet::vml_drawing_hf_xml(Self) -> String? pub fn Worksheet::vml_drawing_xml(Self) -> String? From 198e3d6199ec99f6a296056c6734912cd06d7969 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 16:38:32 +0800 Subject: [PATCH 073/150] fix(zip): avoid hidden initialization in cancellable copies --- zip/reader.mbt | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/zip/reader.mbt b/zip/reader.mbt index c106b8d9..f6923e24 100644 --- a/zip/reader.mbt +++ b/zip/reader.mbt @@ -123,19 +123,31 @@ fn copy_zip_bytes_cancellable( cancelled : () -> Bool, ) -> Bytes raise ZipError { check_zip_cancelled(cancelled) - // Initialize directly from the source. `FixedArray::make` would zero-fill - // the complete destination before the first chunk checkpoint, defeating the - // cancellation guarantee for a permitted 32-MiB stored entry. - let output = FixedArray::makei(bytes.length(), index => { + // An initialized FixedArray constructor performs a complete zero-fill before + // user code can poll cancellation. Allocate uninitialized primitive storage + // instead, then initialize every byte under explicit checkpoints. + let output : UninitializedArray[Byte] = UninitializedArray::make( + bytes.length(), + ) + for index in 0.. Bytes = "%identity" + ///| fn u32_to_int(value : UInt) -> Int raise ZipError { let max_u32_int : UInt = 0x7FFFFFFF From 2cc0ba582d2cf8da0ce7a7fcff70b66c59fc8ee5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 16:40:22 +0800 Subject: [PATCH 074/150] fix(xlsx): canonicalize DrawingML compatibility prefixes --- xlsx/read_xml_namespaces.mbt | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index ff5a735a..748a6752 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -28,6 +28,9 @@ let strict_drawing_chart_namespace = "http://purl.oclc.org/ooxml/drawingml/chart ///| let drawing_slicer_namespace = "http://schemas.microsoft.com/office/drawing/2010/slicer" +///| +let markup_compatibility_namespace = "http://schemas.openxmlformats.org/markup-compatibility/2006" + ///| /// Converts namespace-qualified OOXML element names into the lexical form used /// by the existing bounded feature parsers. The source is fully namespace- and @@ -145,7 +148,7 @@ fn canonicalize_xlsx_worksheet_features( /// Canonicalizes DrawingML by expanded namespace identity. Prefix aliases and /// default namespaces are normalized to the lexical names used by the existing /// drawing feature readers, while a foreign namespace borrowing `xdr`, `a`, -/// `c`, or `sle` is defanged by the bounded XML canonicalizer. +/// `c`, `sle`, or `mc` is defanged by the bounded XML canonicalizer. fn canonicalize_xlsx_drawing_xml( source : StringView, max_output_chars : Int, @@ -178,6 +181,7 @@ fn canonicalize_xlsx_drawing_xml( (transitional_drawing_chart_namespace, "c"), (strict_drawing_chart_namespace, "c"), (drawing_slicer_namespace, "sle"), + (markup_compatibility_namespace, "mc"), ], ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) @@ -370,7 +374,7 @@ test "XLSX XML canonicalization follows extension namespace identity" { ///| test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" { let valid = - #| + #| let canonical = canonicalize_xlsx_drawing_xml( valid, default_max_xml_part_bytes, ) @@ -378,13 +382,16 @@ test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" assert_true(canonical.contains("")) assert_true(canonical.contains("")) assert_true(canonical.contains("")) + assert_true(canonical.contains("")) + assert_true(canonical.contains("")) let spoofed = - #| - assert_true( - canonicalize_xlsx_drawing_xml(spoofed, default_max_xml_part_bytes).contains( - "<_foreign_xdr_oneCellAnchor ", - ), + #| + let spoofed_canonical = canonicalize_xlsx_drawing_xml( + spoofed, default_max_xml_part_bytes, ) + assert_true(spoofed_canonical.contains("<_foreign_xdr_oneCellAnchor ")) + assert_true(spoofed_canonical.contains("<_foreign_mc_AlternateContent/>")) + assert_false(spoofed_canonical.contains(" Date: Sat, 18 Jul 2026 16:40:33 +0800 Subject: [PATCH 075/150] docs(office): distinguish success and failure JSON layout --- docs/agent-json-schemas.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/agent-json-schemas.md b/docs/agent-json-schemas.md index 8b171232..f12a0f5d 100644 --- a/docs/agent-json-schemas.md +++ b/docs/agent-json-schemas.md @@ -38,10 +38,11 @@ disagree, the tests are the source of truth and this document has a bug. Rows and columns are never sent as bare numbers; cell and range references are A1-style strings. - Output is deterministic UTF-8 with a trailing newline. Standalone xlsx and - docx inspection commands use a 2-space indent; the unified `office` - `outline`, `get`, `text`, and `query` commands emit compact JSON so their - output ceiling can be accounted exactly. JSON Lines output is also compact, - one value per line. Consumers must not depend on insignificant whitespace. + docx inspection commands use a 2-space indent. Successful unified `office` + `outline`, `get`, `text`, and `query` envelopes are compact so their output + ceiling can be accounted exactly; unified `--json` failure envelopes use a + 2-space indent. JSON Lines output is compact, one value per line. Consumers + must not depend on insignificant whitespace. ## `xlsx.outline/1` — workbook structure map (`xlsx outline `) From fd62f597dbd71e357ea508ad84360e13394c7ddc Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 16:50:34 +0800 Subject: [PATCH 076/150] fix(office): count XLSX outline metadata without copies --- office/cmd/office/xlsx_read_output.mbt | 24 ++--- xlsx/pkg.generated.mbti | 2 + xlsx/worksheet.mbt | 121 +++++++++++++++++++++++++ xlsx/worksheet_types.mbt | 6 ++ 4 files changed, 139 insertions(+), 14 deletions(-) diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index b3d1a55a..c2b4e3f2 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -79,18 +79,14 @@ fn xlsx_conditional_format_range_count( worksheet : @xlsx.Worksheet, ) -> Int raise CliFailure { projection.checkpoint() - let formats = xlsx_call(projection.file, () => { - worksheet.get_conditional_formats() - }) - let mut count = 0 - for ranges, _ in formats { - projection.checkpoint() - for part in ranges.split(" ") { - projection.checkpoint() - if part.length() > 0 { - count += 1 - } - } + let count = worksheet.conditional_format_range_count_limited( + office_read_max_parser_items, + office_read_max_xml_part_bytes, + cancelled=projection.cancelled, + ) catch { + ReadCancelled => + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + error => raise xlsx_read_failure(error, projection.file) } projection.checkpoint() count @@ -142,7 +138,7 @@ fn xlsx_sheet_summary_json( projection.checkpoint() let comments = worksheet.comments() projection.checkpoint() - let hyperlinks = worksheet.get_hyperlink_cells() + let hyperlink_count = worksheet.hyperlink_count() projection.checkpoint() let validations = worksheet.data_validations() projection.checkpoint() @@ -157,7 +153,7 @@ fn xlsx_sheet_summary_json( "images": Json::number(worksheet.images().length().to_double()), "pivot_tables": Json::number(pivots.length().to_double()), "comments": Json::number(comments.length().to_double()), - "hyperlinks": Json::number(hyperlinks.length().to_double()), + "hyperlinks": Json::number(hyperlink_count.to_double()), "data_validations": Json::number(validations.length().to_double()), "conditional_format_ranges": Json::number( xlsx_conditional_format_range_count(projection, worksheet).to_double(), diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 77875d78..a8dc0dc8 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -2109,6 +2109,7 @@ pub fn Worksheet::col_outline_level(Self, Int) -> Int raise XlsxError pub fn Worksheet::col_visible(Self, Int) -> Bool raise XlsxError pub fn Worksheet::cols(Self) -> Cols pub fn Worksheet::comments(Self) -> ArrayView[Comment] +pub fn Worksheet::conditional_format_range_count_limited(Self, Int, Int, cancelled? : () -> Bool) -> Int raise XlsxError pub fn Worksheet::conditional_formats(Self) -> ArrayView[String] pub fn Worksheet::data_validations(Self) -> ArrayView[String] pub fn Worksheet::delete_chart(Self, StringView) -> Unit raise XlsxError @@ -2140,6 +2141,7 @@ pub fn Worksheet::get_row(Self, Int) -> Array[String] raise XlsxError pub fn Worksheet::get_row_height(Self, Int) -> Double? raise XlsxError pub fn Worksheet::get_rows(Self) -> Array[Array[String]] pub fn Worksheet::get_sheet_protection(Self) -> SheetProtectionOptions +pub fn Worksheet::hyperlink_count(Self) -> Int pub fn Worksheet::ignored_errors(Self) -> ArrayView[IgnoredError] pub fn Worksheet::images(Self) -> ArrayView[Image] pub fn Worksheet::max_col(Self) -> Int diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 54e5b139..cd643a87 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2495,6 +2495,127 @@ pub fn Worksheet::get_conditional_formats( formats } +///| +/// Counts normalized `sqref` ranges directly from retained conditional-format +/// fragments. Unlike `get_conditional_formats`, this does not materialize rule +/// objects, maps, or copied range strings. Both retained-source work and the +/// result are bounded, and long scans poll `cancelled`. +pub fn Worksheet::conditional_format_range_count_limited( + self : Worksheet, + maximum_ranges : Int, + maximum_work_units : Int, + cancelled? : () -> Bool = () => false, +) -> Int raise XlsxError { + if maximum_ranges < 0 || maximum_work_units < 0 { + raise InvalidOptions( + msg="conditional-format count limits must be non-negative", + ) + } + let mut count = 0 + let mut work = 0 + for xml in self.conditional_formats { + check_read_cancelled(cancelled) + let (next_work, work_exceeded) = read_budget_actual( + work, + xml.length(), + maximum_work_units, + ) + if work_exceeded { + raise ResourceLimitExceeded( + kind="conditional_format_count_work_units", + limit=maximum_work_units, + actual=next_work, + ) + } + work = next_work + let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "conditionalFormatting" { + raise InvalidXml(msg="conditionalFormatting tag missing") + } + let sqref = scanner.attribute_view("", "sqref") catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + guard sqref is Some(sqref) else { + raise InvalidXml(msg="conditional formatting sqref missing") + } + let mut in_token = false + for index in 0.. + ), + ) + sheet.conditional_formats.push( + ( + #| + ), + ) + inspect(sheet.conditional_format_range_count_limited(3, 4096), content="3") + try sheet.conditional_format_range_count_limited(2, 4096) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="conditional_format_ranges") + assert_eq(limit, 2) + assert_eq(actual, 3) + } + _ => fail("expected conditional-format range limit") + } noraise { + _ => fail("expected conditional-format range limit") + } +} + +///| +test "conditional-format range count polls cancellation" { + let sheet = Worksheet::new("Sheet1") + sheet.conditional_formats.push( + "", + ) + let checks = [0] + try + sheet.conditional_format_range_count_limited(5000, 64 * 1024, cancelled=() => { + checks[0] += 1 + checks[0] >= 3 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 3) + _ => fail("expected conditional-format count cancellation") + } noraise { + _ => fail("expected conditional-format count cancellation") + } +} + ///| pub fn Worksheet::unset_conditional_format( self : Worksheet, diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index ad701dae..89fe13b8 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -277,6 +277,12 @@ pub fn Worksheet::comments(self : Worksheet) -> ArrayView[Comment] { self.comments } +///| +/// Returns the retained hyperlink count without copying cell references. +pub fn Worksheet::hyperlink_count(self : Worksheet) -> Int { + self.hyperlinks.length() +} + ///| pub fn Worksheet::auto_filter(self : Worksheet) -> AutoFilter? { match self.auto_filter { From b54bea48b68c7be6770f30b6d4307575f8ba58d2 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 17:03:30 +0800 Subject: [PATCH 077/150] fix(xlsx): bound OPC part-name indexing --- ooxml/opc_part_name.mbt | 263 +++++++++++++++++++++++++++------------ ooxml/pkg.generated.mbti | 5 + ooxml/read_parse.mbt | 19 +++ xlsx/read.mbt | 106 +++++++++++++++- 4 files changed, 309 insertions(+), 84 deletions(-) diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt index a4377f67..3a035756 100644 --- a/ooxml/opc_part_name.mbt +++ b/ooxml/opc_part_name.mbt @@ -1,32 +1,32 @@ ///| -fn opc_ascii_hex_digit(character : Char) -> Bool { +fn opc_ascii_hex_digit(character : UInt16) -> Bool { (character >= '0' && character <= '9') || (character >= 'A' && character <= 'F') || (character >= 'a' && character <= 'f') } ///| -fn opc_ascii_hex_value(character : Char) -> Int { +fn opc_ascii_hex_value(character : UInt16) -> Int { if character >= '0' && character <= '9' { - character.to_int() - '0'.to_int() + character.to_int() - ('0' : UInt16).to_int() } else if character >= 'A' && character <= 'F' { - character.to_int() - 'A'.to_int() + 10 + character.to_int() - ('A' : UInt16).to_int() + 10 } else { - character.to_int() - 'a'.to_int() + 10 + character.to_int() - ('a' : UInt16).to_int() + 10 } } ///| -fn opc_percent_byte(characters : ArrayView[Char], index : Int) -> Int? { - if index + 2 >= characters.length() || - characters[index] != '%' || - !opc_ascii_hex_digit(characters[index + 1]) || - !opc_ascii_hex_digit(characters[index + 2]) { +fn opc_percent_byte(text : StringView, index : Int) -> Int? { + if index + 2 >= text.length() || + text[index] != ('%' : UInt16) || + !opc_ascii_hex_digit(text[index + 1]) || + !opc_ascii_hex_digit(text[index + 2]) { None } else { Some( - opc_ascii_hex_value(characters[index + 1]) * 16 + - opc_ascii_hex_value(characters[index + 2]), + opc_ascii_hex_value(text[index + 1]) * 16 + + opc_ascii_hex_value(text[index + 2]), ) } } @@ -104,16 +104,16 @@ fn opc_utf8_percent_length(first : Int) -> Int? { ///| fn opc_decode_percent_scalar( - characters : ArrayView[Char], + text : StringView, index : Int, ) -> (String, Char, Int)? { - guard opc_percent_byte(characters, index) is Some(first) && first >= 0x80 else { + guard opc_percent_byte(text, index) is Some(first) && first >= 0x80 else { return None } guard opc_utf8_percent_length(first) is Some(length) else { return None } let bytes : Array[Byte] = [] for ordinal in 0.. return None } - let decoded_characters = decoded.to_array() - guard decoded_characters.length() == 1 else { return None } - Some((decoded, decoded_characters[0], index + length * 3)) + guard decoded.get_char(0) is Some(decoded_character) else { return None } + let decoded_width = if decoded_character.to_int() > 0xffff { 2 } else { 1 } + guard decoded_width == decoded.length() else { return None } + Some((decoded, decoded_character, index + length * 3)) } ///| -/// Reports whether one logical OPC PartName segment is canonical under the -/// RFC 3987 mapping required by ECMA-376 Part 2. -pub fn is_valid_opc_part_segment(segment : StringView) -> Bool { +fn is_valid_opc_part_segment_cancellable_impl( + segment : StringView, + cancelled : () -> Bool, +) -> Bool raise ParseXmlError { if segment == "" || segment.has_suffix(".") { return false } - let characters = segment.to_array() let mut index = 0 - while index < characters.length() { - let character = characters[index] + let mut next_checkpoint = 0 + while index < segment.length() { + if index >= next_checkpoint { + if cancelled() { + raise ReadCancelled + } + next_checkpoint = index + 4096 + } + guard segment.get_char(index) is Some(character) else { return false } if character == '%' { - guard opc_percent_byte(characters, index) is Some(decoded) else { + guard opc_percent_byte(segment, index) is Some(decoded) else { return false } if decoded == 0x2f || decoded == 0x5c || opc_uri_unreserved_byte(decoded) { return false } if decoded >= 0x80 { - match opc_decode_percent_scalar(characters, index) { + match opc_decode_percent_scalar(segment, index) { Some((_, scalar, next)) => { if opc_iri_ucschar(scalar) { return false @@ -165,49 +173,74 @@ pub fn is_valid_opc_part_segment(segment : StringView) -> Bool { character != '@' { return false } - index = index + 1 + index = index + (if character.to_int() > 0xffff { 2 } else { 1 }) } } true } +///| +/// Reports whether one logical OPC PartName segment is canonical under the +/// RFC 3987 mapping required by ECMA-376 Part 2. +pub fn is_valid_opc_part_segment(segment : StringView) -> Bool { + try! is_valid_opc_part_segment_cancellable_impl(segment, () => false) +} + ///| /// Maps an ASCII physical ZIP item name to its logical OPC PartName. Literal /// non-ASCII IRI scalars are restored from their canonical UTF-8 `%HH` /// projection; opaque legal percent escapes remain unchanged. pub fn logical_opc_part_name_from_zip_item_name(name : StringView) -> String? { + try! logical_opc_part_name_from_zip_item_name_cancellable(name) +} + +///| +/// Cancellation-aware form of `logical_opc_part_name_from_zip_item_name`. +/// It scans directly over the input view and does not copy the whole physical +/// name into an intermediate character array. +pub fn logical_opc_part_name_from_zip_item_name_cancellable( + name : StringView, + cancelled? : () -> Bool = () => false, +) -> String? raise ParseXmlError { if name == "" || name.has_prefix("/") || name.has_suffix("/") { return None } - let characters = name.to_array() + if cancelled() { + raise ReadCancelled + } let output = StringBuilder::new(size_hint=name.length()) let mut index = 0 - while index < characters.length() { - let character = characters[index] - if !character.is_ascii() { + let mut next_checkpoint = 0 + while index < name.length() { + if index >= next_checkpoint { + if cancelled() { + raise ReadCancelled + } + next_checkpoint = index + 4096 + } + let unit = name[index] + guard unit.to_char() is Some(character) && character.is_ascii() else { return None } if character == '%' { - guard opc_percent_byte(characters, index) is Some(decoded) else { - return None - } + guard opc_percent_byte(name, index) is Some(decoded) else { return None } if decoded >= 0x80 { - match opc_decode_percent_scalar(characters, index) { + match opc_decode_percent_scalar(name, index) { Some((scalar_text, scalar, next)) if opc_iri_ucschar(scalar) => { output.write_string(scalar_text) index = next } _ => { - output.write_char(characters[index]) - output.write_char(characters[index + 1]) - output.write_char(characters[index + 2]) + output.write_char(character) + output.write_char(name[index + 1].to_char().unwrap()) + output.write_char(name[index + 2].to_char().unwrap()) index = index + 3 } } } else { - output.write_char(characters[index]) - output.write_char(characters[index + 1]) - output.write_char(characters[index + 2]) + output.write_char(character) + output.write_char(name[index + 1].to_char().unwrap()) + output.write_char(name[index + 2].to_char().unwrap()) index = index + 3 } } else { @@ -217,7 +250,7 @@ pub fn logical_opc_part_name_from_zip_item_name(name : StringView) -> String? { } let logical = output.to_string() for segment in logical.split("/") { - if !is_valid_opc_part_segment(segment) { + if !is_valid_opc_part_segment_cancellable_impl(segment, cancelled) { return None } } @@ -225,7 +258,16 @@ pub fn logical_opc_part_name_from_zip_item_name(name : StringView) -> String? { } ///| -type OpcPartNameChildren = @sorted_map.SortedMap[String, Int] +fn opc_part_name_identity_unit(unit : UInt16) -> UInt16 { + if unit >= ('A' : UInt16) && unit <= ('Z' : UInt16) { + (unit.to_int() + 32).to_uint16() + } else { + unit + } +} + +///| +type OpcPartNameChildren = @sorted_map.SortedMap[UInt16, Int] ///| priv struct OpcPartNameTrieNode { @@ -247,9 +289,10 @@ pub(all) enum OpcPartNameConflict { } ///| -/// A segment trie for OPC PartName identity and non-derivability checks. +/// A UTF-16 code-unit trie for OPC PartName identity and non-derivability checks. /// Work and storage are linear in aggregate PartName length, while ordered -/// child indexes keep hostile names independent of string-hash behavior. +/// scalar child indexes avoid repeated comparisons of attacker-controlled +/// string prefixes and keep hostile names independent of string-hash behavior. pub struct OpcPartNameRegistry { priv nodes : Array[OpcPartNameTrieNode] } @@ -269,57 +312,98 @@ pub fn OpcPartNameRegistry::register( name : StringView, display : String, ) -> OpcPartNameConflict? { - let key = package_part_name_key(name) - let segments = key.split("/").collect() + try! self.register_cancellable(name, display) +} + +///| +/// Cancellation-aware form of `register`. Rejected or cancelled registrations +/// never link a partial path into the registry. +pub fn OpcPartNameRegistry::register_cancellable( + self : OpcPartNameRegistry, + name : StringView, + display : String, + cancelled? : () -> Bool = () => false, +) -> OpcPartNameConflict? raise ParseXmlError { + if cancelled() { + raise ReadCancelled + } let mut node_index = 0 - let mut complete_path = true - for segment_view in segments { - match self.nodes[node_index].terminal { - Some(existing) => return Some(Derivable(existing)) - None => () + let mut index = 0 + let mut missing_index : Int? = None + while index < name.length() { + if index % 4096 == 0 && cancelled() { + raise ReadCancelled + } + let unit = opc_part_name_identity_unit(name[index]) + if unit == ('/' : UInt16) { + match self.nodes[node_index].terminal { + Some(existing) => return Some(Derivable(existing)) + None => () + } } - let segment = segment_view.to_owned() - match self.nodes[node_index].children.get(segment) { + match self.nodes[node_index].children.get(unit) { Some(child) => node_index = child None => { - complete_path = false + missing_index = Some(index) break } } + index = index + 1 } - if complete_path { + if missing_index is None { match self.nodes[node_index].terminal { Some(existing) => return Some(Equivalent(existing)) None => () } - match self.nodes[node_index].first_part { - Some(existing) => return Some(Derivable(existing)) - None => () + match + self.nodes[node_index].children + .get('/') + .map(child => self.nodes[child].first_part) { + Some(Some(existing)) => return Some(Derivable(existing)) + _ => () } } - // The preflight is mutation-free, so a rejected name cannot leave a partial - // trie path that changes a later conflict result. - node_index = 0 - if self.nodes[node_index].first_part is None { - self.nodes[node_index].first_part = Some(display) - } - for segment_view in segments { - let segment = segment_view.to_owned() - let child = match self.nodes[node_index].children.get(segment) { - Some(existing) => existing - None => { - let fresh = self.nodes.length() - self.nodes.push(new_opc_part_name_trie_node()) - self.nodes[node_index].children[segment] = fresh - fresh + + match missing_index { + None => { + if cancelled() { + raise ReadCancelled } + self.nodes[node_index].terminal = Some(display) } - node_index = child - if self.nodes[node_index].first_part is None { - self.nodes[node_index].first_part = Some(display) + Some(start) => { + // Construct the missing suffix off-registry. Only after every allocation + // and cancellation point succeeds do we append and link it once. + let base = self.nodes.length() + let detached : Array[OpcPartNameTrieNode] = [] + for offset in start.. fail("expected ancestor OPC PartName conflict") } + + let sibling = OpcPartNameRegistry::new() + assert_true(sibling.register("custom/a.xmlx", "custom/a.xmlx") is None) + assert_true(sibling.register("custom/a.xml", "custom/a.xml") is None) +} + +///| +test "cancelled OPC PartName registration does not link a partial path" { + let registry = OpcPartNameRegistry::new() + let checks = [0] + let long_name = "custom/a.xml/" + "child/".repeat(2000) + "part.xml" + try + registry.register_cancellable(long_name, long_name, cancelled=() => { + checks[0] = checks[0] + 1 + checks[0] >= 3 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 3) + _ => fail("unexpected OPC PartName registration error") + } noraise { + _ => fail("expected OPC PartName registration cancellation") + } + assert_true(registry.register("custom/a.xml", "custom/a.xml") is None) } diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 16e6d187..c7a4529c 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -24,8 +24,12 @@ pub fn is_valid_opc_part_segment(StringView) -> Bool pub fn logical_opc_part_name_from_zip_item_name(StringView) -> String? +pub fn logical_opc_part_name_from_zip_item_name_cancellable(StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError + pub fn package_part_name_key(StringView) -> String +pub fn package_part_name_key_cancellable(StringView, cancelled? : () -> Bool) -> String raise ParseXmlError + pub fn parse_external_relationship_targets(StringView, StringView, cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError pub fn parse_external_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError @@ -59,6 +63,7 @@ pub struct OpcPartNameRegistry { } pub fn OpcPartNameRegistry::new() -> Self pub fn OpcPartNameRegistry::register(Self, StringView, String) -> OpcPartNameConflict? +pub fn OpcPartNameRegistry::register_cancellable(Self, StringView, String, cancelled? : () -> Bool) -> OpcPartNameConflict? raise ParseXmlError type WorkbookManifest pub fn WorkbookManifest::add_content_default(Self, String, String) -> Unit diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 69b700c4..fae3bcf0 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -315,9 +315,28 @@ priv struct PackageContentTypes { /// Returns the ASCII-case-insensitive identity key used for OPC part names. /// The leading slash, when present, remains part of the key. pub fn package_part_name_key(name : StringView) -> String { + try! package_part_name_key_cancellable(name) +} + +///| +/// Cancellation-aware form of `package_part_name_key`. It performs the same +/// ASCII-only case fold while polling at least once per 4,096 UTF-16 code +/// units. +pub fn package_part_name_key_cancellable( + name : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise ParseXmlError { + if cancelled() { + raise ReadCancelled + } let output = StringBuilder::new(size_hint=name.length()) + let mut offset = 0 for character in name { + if offset % 4096 == 0 && cancelled() { + raise ReadCancelled + } output.write_char(character.to_ascii_lowercase()) + offset = offset + (if character.to_int() > 0xffff { 2 } else { 1 }) } output.to_string() } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 19241953..cbd284e6 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3119,33 +3119,89 @@ fn first_existing_workbook_rel_target_path( None } +///| +fn budgeted_package_part_name_key( + name : StringView, + budget : ReadBudget, +) -> String raise XlsxError { + budget.checkpoint() + budget.charge_work(name.length()) + @ooxml.package_part_name_key_cancellable(name, cancelled=budget.cancelled) catch { + InvalidXml(msg~) => raise InvalidPackage(msg~) + ReadCancelled => raise ReadCancelled + } +} + +///| +fn budgeted_logical_opc_part_name( + name : StringView, + budget : ReadBudget, +) -> String? raise XlsxError { + budget.checkpoint() + budget.charge_work(name.length()) + @ooxml.logical_opc_part_name_from_zip_item_name_cancellable( + name, + cancelled=budget.cancelled, + ) catch { + InvalidXml(msg~) => raise InvalidPackage(msg~) + ReadCancelled => raise ReadCancelled + } +} + +///| +fn budgeted_register_opc_part_name( + registry : @ooxml.OpcPartNameRegistry, + name : StringView, + display : String, + budget : ReadBudget, +) -> @ooxml.OpcPartNameConflict? raise XlsxError { + budget.checkpoint() + // Registration performs a mutation-free lookup pass and, for a new branch, + // constructs a detached suffix before linking it. Charge both linear passes + // before either can run. + budget.charge_work(name.length()) + budget.charge_work(name.length()) + registry.register_cancellable(name, display, cancelled=budget.cancelled) catch { + InvalidXml(msg~) => raise InvalidPackage(msg~) + ReadCancelled => raise ReadCancelled + } +} + ///| fn archive_part_name_index( archive : @zip.Archive, + budget? : ReadBudget, ) -> Map[String, String] raise XlsxError { + let budget = budget.unwrap_or(ReadBudget::new(ReadLimits::new())) let names : Map[String, String] = Map([]) let registry = @ooxml.OpcPartNameRegistry::new() + let content_types_key = @ooxml.package_part_name_key(content_types_part_path) for entry in archive.entries() { + budget.checkpoint() let physical_name = entry.name() if physical_name.has_suffix("/") { continue } + budget.charge_work(physical_name.length()) let logical_name = if physical_name == content_types_part_path { physical_name - } else if @ooxml.package_part_name_key(physical_name) == - @ooxml.package_part_name_key(content_types_part_path) { + } else if budgeted_package_part_name_key(physical_name, budget) == + content_types_key { raise InvalidPackage( msg="reserved content-types part name has invalid spelling", ) } else { - match @ooxml.logical_opc_part_name_from_zip_item_name(physical_name) { + match budgeted_logical_opc_part_name(physical_name, budget) { Some(value) => value None => raise InvalidPackage(msg="invalid OPC ZIP item name") } } - let key = @ooxml.package_part_name_key(logical_name) + let key = budgeted_package_part_name_key(logical_name, budget) if logical_name != content_types_part_path { - match registry.register(logical_name, physical_name) { + match + budgeted_register_opc_part_name( + registry, logical_name, physical_name, budget, + ) { Some(Equivalent(_)) => raise InvalidPackage(msg="duplicate or equivalent OPC part name") Some(Derivable(existing)) => @@ -3155,11 +3211,13 @@ fn archive_part_name_index( None => () } } + budget.charge_work(key.length()) match names.get(key) { Some(_) => raise InvalidPackage(msg="duplicate or equivalent OPC part name") None => () } + budget.charge_work(key.length()) names[key] = physical_name } names @@ -3245,6 +3303,42 @@ test "read archive index rejects OPC-derivable parts in either order" { } } +///| +test "read archive index charges physical OPC name work before decoding" { + let archive = @zip.Archive::new() + archive.add("custom/" + "a".repeat(64) + ".xml", b"part") + let budget = ReadBudget::new( + ReadLimits::with_values(max_parser_work_units=32), + ) + try archive_part_name_index(archive, budget~) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="parser_work_units") + assert_eq(limit, 32) + assert_true(actual > limit) + } + _ => fail("unexpected archive name budget error") + } noraise { + _ => fail("expected archive name work limit") + } +} + +///| +test "read archive index polls cancellation inside long OPC names" { + let archive = @zip.Archive::new() + archive.add("custom/" + "a".repeat(9000) + ".xml", b"part") + let checks = [0] + let budget = ReadBudget::new(ReadLimits::new(), cancelled=() => { + checks[0] = checks[0] + 1 + checks[0] >= 4 + }) + try archive_part_name_index(archive, budget~) catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("unexpected archive name cancellation error") + } noraise { + _ => fail("expected archive name cancellation") + } +} + ///| fn actual_archive_part_path( part_names : Map[String, String], @@ -3350,8 +3444,8 @@ fn read_zip_archive_core_unchecked( cancelled : () -> Bool, transcoder? : (String, Bytes) -> String raise XlsxError, ) -> Workbook raise XlsxError { - let part_names = archive_part_name_index(archive) let read_budget = ReadBudget::new(limits, cancelled~) + let part_names = archive_part_name_index(archive, budget=read_budget) let decode_source = (value : BytesView) => { if value.length() > limits.max_xml_part_bytes { raise ResourceLimitExceeded( From 6e446f9f2d59c8350a763d60aa32dbe748640db3 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 17:17:50 +0800 Subject: [PATCH 078/150] fix(office): bound shared formula translation --- office/cmd/office/pkg.generated.mbti | 2 + office/cmd/office/read_package.mbt | 4 + office/cmd/office/xlsx_read_model.mbt | 62 ++- office/cmd/office/xlsx_read_wbtest.mbt | 58 +++ xlsx/pkg.generated.mbti | 1 + xlsx/shared_formula_translate.mbt | 651 ++++++++++++++++++------ xlsx/shared_formula_validation_test.mbt | 27 + 7 files changed, 637 insertions(+), 168 deletions(-) diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 53ce4539..381bf495 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -69,6 +69,8 @@ type XlsxCellSnapshot type XlsxFormatBudget +type XlsxFormulaBudget + type XlsxJsonRetention type XlsxMetadataBudget diff --git a/office/cmd/office/read_package.mbt b/office/cmd/office/read_package.mbt index f58cbece..fb5a920a 100644 --- a/office/cmd/office/read_package.mbt +++ b/office/cmd/office/read_package.mbt @@ -213,6 +213,10 @@ fn xlsx_read_failure(error : @xlsx.XlsxError, file : String) -> CliFailure { invalid_package("shared string index is out of range: \{index}") InvalidStyleId(index~) => invalid_package("cell style index is out of range: \{index}") + ReadCancelled => + CliFailure( + @lib.protocol_error("office.cancelled", "XLSX read was cancelled"), + ) _ => CliFailure( @lib.protocol_error( diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index eca75463..e33ac283 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -10,6 +10,9 @@ let xlsx_cli_max_scanned_string_chars : Int = 16 * 1024 * 1024 ///| let xlsx_cli_max_format_work_units : Int = 16 * 1024 * 1024 +///| +let xlsx_cli_max_formula_work_units : Int = 64 * 1024 * 1024 + ///| let xlsx_cli_max_metadata_items : Int = 10_000 @@ -88,6 +91,7 @@ struct XlsxScanBudget { maximum : Int strings : XlsxStringBudget formatting : XlsxFormatBudget + formulas : XlsxFormulaBudget mut scanned : Int cancelled : () -> Bool } @@ -98,6 +102,12 @@ struct XlsxFormatBudget { mut used : Int } +///| +struct XlsxFormulaBudget { + maximum : Int + mut used : Int +} + ///| struct XlsxMetadataBudget { maximum_items : Int @@ -633,16 +643,50 @@ fn XlsxFormatBudget::charge_style( projection.checkpoint() } +///| +fn XlsxFormulaBudget::new( + maximum? : Int = xlsx_cli_max_formula_work_units, +) -> XlsxFormulaBudget { + { maximum, used: 0 } +} + +///| +fn XlsxFormulaBudget::remaining(self : XlsxFormulaBudget) -> Int { + self.maximum - self.used +} + +///| +fn XlsxFormulaBudget::charge( + self : XlsxFormulaBudget, + amount : Int, +) -> Unit raise CliFailure { + if amount < 0 || amount > self.maximum - self.used { + let actual = if amount < 0 || amount > 0x7fffffff - self.used { + 0x7fffffff + } else { + self.used + amount + } + raise xlsx_resource_failure( + "shared formula translation work units", + self.maximum, + actual~, + ) + } + self.used += amount +} + ///| fn XlsxScanBudget::new( maximum : Int, format_work_maximum? : Int = xlsx_cli_max_format_work_units, + formula_work_maximum? : Int = xlsx_cli_max_formula_work_units, cancelled? : () -> Bool = () => false, ) -> XlsxScanBudget { { maximum, strings: XlsxStringBudget::new(), formatting: XlsxFormatBudget::new(maximum=format_work_maximum), + formulas: XlsxFormulaBudget::new(maximum=formula_work_maximum), scanned: 0, cancelled, } @@ -686,6 +730,7 @@ fn xlsx_cell_formula( worksheet : @xlsx.Worksheet, row : Int, column : Int, + budget : XlsxScanBudget, ) -> (String?, Bool) raise CliFailure { projection.checkpoint() let result = match @@ -702,9 +747,18 @@ fn xlsx_cell_formula( let master = sheet.shared_formula_master( projection, worksheet, shared_index, ) - Some( - xlsx_call(projection.file, () => master.translate_to(row, column)), - ) + let (translated, work) = xlsx_call(projection.file, () => { + master.translate_to_limited( + row, + column, + maximum_input_chars=xlsx_cli_max_cell_string_chars, + maximum_output_chars=budget.strings.remaining_for_value(), + maximum_work_units=budget.formulas.remaining(), + cancelled=budget.cancelled, + ) + }) + budget.formulas.charge(work) + Some(translated) } (Some(Shared), None) => raise xlsx_cli_failure( @@ -741,7 +795,7 @@ fn xlsx_cell_snapshot( worksheet.get_cell_value_raw(reference) }) let (formula, cached_value_present) = xlsx_cell_formula( - projection, sheet, worksheet, row, column, + projection, sheet, worksheet, row, column, budget, ) let normalized_raw = match raw { Some(String("")) if formula is Some(_) && cached_value_present => raw diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index aed3f38a..cb290090 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -507,6 +507,10 @@ test "XLSX string and metadata accounting fail before unbounded retention" { aggregate.charge("cell value", "abc") let too_many = xlsx_cli_error(() => aggregate.charge("cell formula", "def")) assert_eq(too_many.code, "office.xlsx.resource_limit") + let formulas = XlsxFormulaBudget::new(maximum=3) + formulas.charge(2) + let formula_work = xlsx_cli_error(() => formulas.charge(2)) + assert_eq(formula_work.code, "office.xlsx.resource_limit") let metadata : XlsxMetadataBudget = { maximum_items: 1, maximum_string_chars: 4, @@ -721,6 +725,60 @@ async test "XLSX snapshots preserve and resolve shared-formula followers" { assert_eq(master_fields.get("matched_total"), Some(Json::number(1))) } +///| +test "XLSX followers reject oversized shared masters before translation" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "D1+1", + opts=@xlsx.FormulaOpts::shared("A1:B1"), + ) + let archive = @zip.read(@xlsx.write(workbook)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => @utf8.decode(value, ignore_bom=true) + None => fail("missing shared-formula worksheet") + } + let oversized = "A1+" + "1+".repeat(512 * 1024) + "1" + assert_true(oversized.length() > xlsx_cli_max_cell_string_chars) + let updated = xml.replace(old="D1+1", new=oversized) + assert_true(updated != xml) + assert_true(archive.replace(path, @utf8.encode(updated))) + let source = xlsx_bounded_test_source( + "oversized-shared-formula.xlsx", + @zip.write(archive), + ) + let projection = open_xlsx_projection(source, 10) + guard resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]/cell[B1]") + is Cell(target, rect, ..) else { + fail("expected shared-formula follower selector") + } + let error = xlsx_cli_error(() => { + ignore( + xlsx_cell_snapshot( + projection, + target, + rect.row_lo, + rect.col_lo, + XlsxScanBudget::new(1), + ), + ) + }) + assert_eq(error.code, "office.xlsx.resource_limit") + guard error.details is Some(Object(details)) else { + fail("expected shared-formula limit details") + } + assert_eq( + details.get("resource"), + Some(Json::string("shared_formula_input_chars")), + ) + assert_eq( + details.get("limit"), + Some(Json::number(xlsx_cli_max_cell_string_chars.to_double())), + ) +} + ///| test "XLSX structured reads reject shared-formula followers without a master" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index a8dc0dc8..c471983e 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1359,6 +1359,7 @@ pub struct SharedFormulaMaster { // private fields } pub fn SharedFormulaMaster::translate_to(Self, Int, Int) -> String raise XlsxError +pub fn SharedFormulaMaster::translate_to_limited(Self, Int, Int, maximum_input_chars~ : Int, maximum_output_chars~ : Int, maximum_work_units~ : Int, cancelled? : () -> Bool) -> (String, Int) raise XlsxError pub struct SheetBackground { data : Bytes diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index bf684fa3..a80029a1 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -14,7 +14,6 @@ pub struct SharedFormulaMaster { ///| priv struct FormulaCellToken { - start : Int end : Int column_start : Int column_end : Int @@ -43,16 +42,141 @@ priv struct FormulaRowToken { } ///| -fn formula_chars_text( +priv struct FormulaTranslationBudget { + maximum_work_units : Int + cancelled : () -> Bool + mut work_units : Int + mut next_checkpoint : Int +} + +///| +fn FormulaTranslationBudget::new( + maximum_work_units : Int, + cancelled : () -> Bool, +) -> FormulaTranslationBudget { + { maximum_work_units, cancelled, work_units: 0, next_checkpoint: 0 } +} + +///| +fn FormulaTranslationBudget::checkpoint( + self : FormulaTranslationBudget, +) -> Unit raise XlsxError { + if (self.cancelled)() { + raise ReadCancelled + } +} + +///| +fn FormulaTranslationBudget::charge_work( + self : FormulaTranslationBudget, + amount : Int, +) -> Unit raise XlsxError { + if amount <= 0 { + return + } + let (actual, exceeded) = read_budget_actual( + self.work_units, + amount, + self.maximum_work_units, + ) + if exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_work_units", + limit=self.maximum_work_units, + actual~, + ) + } + self.work_units = actual + if self.work_units >= self.next_checkpoint { + self.checkpoint() + self.next_checkpoint = if self.work_units > 0x7fffffff - 4096 { + 0x7fffffff + } else { + self.work_units + 4096 + } + } +} + +///| +priv struct FormulaTranslationOutput { + builder : StringBuilder + budget : FormulaTranslationBudget + maximum_chars : Int + mut chars : Int +} + +///| +fn FormulaTranslationOutput::new( + size_hint : Int, + maximum_chars : Int, + budget : FormulaTranslationBudget, +) -> FormulaTranslationOutput { + { builder: StringBuilder::new(size_hint~), budget, maximum_chars, chars: 0 } +} + +///| +fn FormulaTranslationOutput::charge_chars( + self : FormulaTranslationOutput, + amount : Int, +) -> Unit raise XlsxError { + let (actual, exceeded) = read_budget_actual( + self.chars, + amount, + self.maximum_chars, + ) + if exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_output_chars", + limit=self.maximum_chars, + actual~, + ) + } + self.chars = actual +} + +///| +fn FormulaTranslationOutput::write_char( + self : FormulaTranslationOutput, + character : Char, +) -> Unit raise XlsxError { + let width = if character.to_int() > 0xffff { 2 } else { 1 } + self.budget.charge_work(width) + self.charge_chars(width) + self.builder.write_char(character) +} + +///| +fn FormulaTranslationOutput::write_string( + self : FormulaTranslationOutput, + value : String, +) -> Unit raise XlsxError { + let width = value.length() + self.budget.charge_work(width) + self.charge_chars(width) + self.builder.write_string(value) +} + +///| +fn formula_character( + chars : ArrayView[Char], + index : Int, + budget : FormulaTranslationBudget, +) -> Char raise XlsxError { + let character = chars[index] + budget.charge_work(if character.to_int() > 0xffff { 2 } else { 1 }) + character +} + +///| +fn FormulaTranslationOutput::write_formula_chars( + self : FormulaTranslationOutput, chars : ArrayView[Char], start : Int, end : Int, -) -> String { - let out = StringBuilder::new() +) -> Unit raise XlsxError { for index in start.. Bool { fn formula_reference_start_boundary( chars : ArrayView[Char], start : Int, -) -> Bool { - start == 0 || !formula_reference_word_char(chars[start - 1]) + budget : FormulaTranslationBudget, +) -> Bool raise XlsxError { + start == 0 || + !formula_reference_word_char(formula_character(chars, start - 1, budget)) } ///| -fn formula_reference_end_boundary(chars : ArrayView[Char], end : Int) -> Bool { +fn formula_reference_end_boundary( + chars : ArrayView[Char], + end : Int, + budget : FormulaTranslationBudget, +) -> Bool raise XlsxError { if end >= chars.length() { return true } - let character = chars[end] + let character = formula_character(chars, end, budget) !formula_reference_word_char(character) && character != '(' && character != '[' && @@ -92,10 +222,11 @@ fn formula_column_number( chars : ArrayView[Char], start : Int, end : Int, -) -> Int? { + budget : FormulaTranslationBudget, +) -> Int? raise XlsxError { let mut column = 0 for index in start.. Int? { + budget : FormulaTranslationBudget, +) -> Int? raise XlsxError { if start >= end { return None } let mut value = 0 for index in start.. FormulaCellToken? { + budget : FormulaTranslationBudget, +) -> FormulaCellToken? raise XlsxError { if start < 0 || start >= chars.length() { return None } let mut index = start - let absolute_column = chars[index] == '$' + let absolute_column = formula_character(chars, index, budget) == '$' if absolute_column { index += 1 } let column_start = index - while index < chars.length() && chars[index].is_ascii_alphabetic() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_alphabetic() { + break + } index += 1 } let column_end = index if column_end == column_start || column_end - column_start > 3 { return None } - let absolute_row = index < chars.length() && chars[index] == '$' + let absolute_row = if index < chars.length() { + formula_character(chars, index, budget) == '$' + } else { + false + } if absolute_row { index += 1 } let row_start = index - while index < chars.length() && chars[index].is_ascii_digit() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_digit() { + break + } index += 1 } let row_end = index - guard formula_column_number(chars, column_start, column_end) is Some(column) && - formula_bounded_decimal(chars, row_start, row_end, excel_max_rows) + guard formula_column_number(chars, column_start, column_end, budget) + is Some(column) && + formula_bounded_decimal(chars, row_start, row_end, excel_max_rows, budget) is Some(row) else { return None } Some({ - start, end: index, column_start, column_end, @@ -193,24 +336,29 @@ fn parse_formula_cell_token( fn parse_formula_column_token( chars : ArrayView[Char], start : Int, -) -> FormulaColumnToken? { + budget : FormulaTranslationBudget, +) -> FormulaColumnToken? raise XlsxError { if start < 0 || start >= chars.length() { return None } let mut index = start - let absolute = chars[index] == '$' + let absolute = formula_character(chars, index, budget) == '$' if absolute { index += 1 } let letters_start = index - while index < chars.length() && chars[index].is_ascii_alphabetic() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_alphabetic() { + break + } index += 1 } let letters_end = index if letters_end == letters_start || letters_end - letters_start > 3 { return None } - guard formula_column_number(chars, letters_start, letters_end) is Some(column) else { + guard formula_column_number(chars, letters_start, letters_end, budget) + is Some(column) else { return None } Some({ start, end: index, absolute, column }) @@ -220,21 +368,27 @@ fn parse_formula_column_token( fn parse_formula_row_token( chars : ArrayView[Char], start : Int, -) -> FormulaRowToken? { + budget : FormulaTranslationBudget, +) -> FormulaRowToken? raise XlsxError { if start < 0 || start >= chars.length() { return None } let mut index = start - let absolute = chars[index] == '$' + let absolute = formula_character(chars, index, budget) == '$' if absolute { index += 1 } let digits_start = index - while index < chars.length() && chars[index].is_ascii_digit() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_digit() { + break + } index += 1 } let digits_end = index - guard formula_bounded_decimal(chars, digits_start, digits_end, excel_max_rows) + guard formula_bounded_decimal( + chars, digits_start, digits_end, excel_max_rows, budget, + ) is Some(row) else { return None } @@ -262,73 +416,93 @@ fn formula_shifted_row(row : Int, delta : Int) -> String? { } ///| -fn shifted_formula_cell_text( +fn write_shifted_formula_cell( chars : ArrayView[Char], token : FormulaCellToken, column_delta : Int, row_delta : Int, -) -> String? { - let out = StringBuilder::new() + out : FormulaTranslationOutput, +) -> Unit raise XlsxError { + let shifted_column = if token.absolute_column { + None + } else { + formula_shifted_column(token.column, column_delta) + } + let shifted_row = if token.absolute_row { + None + } else { + formula_shifted_row(token.row, row_delta) + } + if (!token.absolute_column && shifted_column is None) || + (!token.absolute_row && shifted_row is None) { + out.write_string("#REF!") + return + } if token.absolute_column { out.write_char('$') - for index in token.column_start.. String? { + out : FormulaTranslationOutput, +) -> Unit raise XlsxError { if token.absolute { - return Some(formula_chars_text(chars, token.start, token.end)) + out.write_formula_chars(chars, token.start, token.end) + } else { + out.write_string( + formula_shifted_column(token.column, delta).unwrap_or("#REF!"), + ) } - formula_shifted_column(token.column, delta) } ///| -fn shifted_formula_row_text( +fn write_shifted_formula_row( chars : ArrayView[Char], token : FormulaRowToken, delta : Int, -) -> String? { + out : FormulaTranslationOutput, +) -> Unit raise XlsxError { if token.absolute { - return Some(formula_chars_text(chars, token.start, token.end)) + out.write_formula_chars(chars, token.start, token.end) + } else { + out.write_string(formula_shifted_row(token.row, delta).unwrap_or("#REF!")) } - formula_shifted_row(token.row, delta) } ///| -fn formula_range_separator_end(chars : ArrayView[Char], start : Int) -> Int? { +fn formula_range_separator_end( + chars : ArrayView[Char], + start : Int, + budget : FormulaTranslationBudget, +) -> Int? raise XlsxError { let mut index = start - while index < chars.length() && chars[index].is_ascii_whitespace() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_whitespace() { + break + } index += 1 } - if index >= chars.length() || chars[index] != ':' { + if index >= chars.length() || formula_character(chars, index, budget) != ':' { return None } index += 1 - while index < chars.length() && chars[index].is_ascii_whitespace() { + while index < chars.length() { + if !formula_character(chars, index, budget).is_ascii_whitespace() { + break + } index += 1 } Some(index) @@ -340,45 +514,37 @@ fn translate_formula_cell_reference_at( start : Int, column_delta : Int, row_delta : Int, -) -> (String, Int)? { - guard formula_reference_start_boundary(chars, start) && - parse_formula_cell_token(chars, start) is Some(first) else { + out : FormulaTranslationOutput, +) -> Int? raise XlsxError { + let budget = out.budget + guard formula_reference_start_boundary(chars, start, budget) && + parse_formula_cell_token(chars, start, budget) is Some(first) else { return None } - match formula_range_separator_end(chars, first.end) { + match formula_range_separator_end(chars, first.end, budget) { Some(second_start) => - match parse_formula_cell_token(chars, second_start) { - Some(second) if formula_reference_end_boundary(chars, second.end) => { - let first_text = shifted_formula_cell_text( - chars, first, column_delta, row_delta, - ) - let second_text = shifted_formula_cell_text( - chars, second, column_delta, row_delta, - ) - return Some( - ( - first_text.unwrap_or("#REF!") + - formula_chars_text(chars, first.end, second.start) + - second_text.unwrap_or("#REF!"), - second.end, - ), + match parse_formula_cell_token(chars, second_start, budget) { + Some(second) if formula_reference_end_boundary( + chars, + second.end, + budget, + ) => { + write_shifted_formula_cell(chars, first, column_delta, row_delta, out) + out.write_formula_chars(chars, first.end, second_start) + write_shifted_formula_cell( + chars, second, column_delta, row_delta, out, ) + return Some(second.end) } _ => () } None => () } - if !formula_reference_end_boundary(chars, first.end) { + if !formula_reference_end_boundary(chars, first.end, budget) { return None } - Some( - ( - shifted_formula_cell_text(chars, first, column_delta, row_delta).unwrap_or( - "#REF!", - ), - first.end, - ), - ) + write_shifted_formula_cell(chars, first, column_delta, row_delta, out) + Some(first.end) } ///| @@ -386,24 +552,20 @@ fn translate_formula_column_range_at( chars : ArrayView[Char], start : Int, column_delta : Int, -) -> (String, Int)? { - guard formula_reference_start_boundary(chars, start) && - parse_formula_column_token(chars, start) is Some(first) && - formula_range_separator_end(chars, first.end) is Some(second_start) && - parse_formula_column_token(chars, second_start) is Some(second) && - formula_reference_end_boundary(chars, second.end) else { + out : FormulaTranslationOutput, +) -> Int? raise XlsxError { + let budget = out.budget + guard formula_reference_start_boundary(chars, start, budget) && + parse_formula_column_token(chars, start, budget) is Some(first) && + formula_range_separator_end(chars, first.end, budget) is Some(second_start) && + parse_formula_column_token(chars, second_start, budget) is Some(second) && + formula_reference_end_boundary(chars, second.end, budget) else { return None } - let first_text = shifted_formula_column_text(chars, first, column_delta) - let second_text = shifted_formula_column_text(chars, second, column_delta) - Some( - ( - first_text.unwrap_or("#REF!") + - formula_chars_text(chars, first.end, second.start) + - second_text.unwrap_or("#REF!"), - second.end, - ), - ) + write_shifted_formula_column(chars, first, column_delta, out) + out.write_formula_chars(chars, first.end, second.start) + write_shifted_formula_column(chars, second, column_delta, out) + Some(second.end) } ///| @@ -411,24 +573,20 @@ fn translate_formula_row_range_at( chars : ArrayView[Char], start : Int, row_delta : Int, -) -> (String, Int)? { - guard formula_reference_start_boundary(chars, start) && - parse_formula_row_token(chars, start) is Some(first) && - formula_range_separator_end(chars, first.end) is Some(second_start) && - parse_formula_row_token(chars, second_start) is Some(second) && - formula_reference_end_boundary(chars, second.end) else { + out : FormulaTranslationOutput, +) -> Int? raise XlsxError { + let budget = out.budget + guard formula_reference_start_boundary(chars, start, budget) && + parse_formula_row_token(chars, start, budget) is Some(first) && + formula_range_separator_end(chars, first.end, budget) is Some(second_start) && + parse_formula_row_token(chars, second_start, budget) is Some(second) && + formula_reference_end_boundary(chars, second.end, budget) else { return None } - let first_text = shifted_formula_row_text(chars, first, row_delta) - let second_text = shifted_formula_row_text(chars, second, row_delta) - Some( - ( - first_text.unwrap_or("#REF!") + - formula_chars_text(chars, first.end, second.start) + - second_text.unwrap_or("#REF!"), - second.end, - ), - ) + write_shifted_formula_row(chars, first, row_delta, out) + out.write_formula_chars(chars, first.end, second.start) + write_shifted_formula_row(chars, second, row_delta, out) + Some(second.end) } ///| @@ -436,18 +594,19 @@ fn copy_formula_quoted_token( chars : ArrayView[Char], start : Int, quote : Char, - out : StringBuilder, -) -> Int { + out : FormulaTranslationOutput, +) -> Int raise XlsxError { let mut index = start - out.write_char(chars[index]) + out.write_char(formula_character(chars, index, out.budget)) index += 1 while index < chars.length() { - let character = chars[index] + let character = formula_character(chars, index, out.budget) out.write_char(character) index += 1 if character == quote { - if index < chars.length() && chars[index] == quote { - out.write_char(chars[index]) + if index < chars.length() && + formula_character(chars, index, out.budget) == quote { + out.write_char(formula_character(chars, index, out.budget)) index += 1 } else { break @@ -461,12 +620,12 @@ fn copy_formula_quoted_token( fn copy_formula_bracket_token( chars : ArrayView[Char], start : Int, - out : StringBuilder, -) -> Int { + out : FormulaTranslationOutput, +) -> Int raise XlsxError { let mut depth = 0 let mut index = start while index < chars.length() { - let character = chars[index] + let character = formula_character(chars, index, out.budget) out.write_char(character) index += 1 if character == '[' { @@ -499,50 +658,110 @@ fn formula_unquoted_sheet_name_char(character : Char) -> Bool { fn formula_unquoted_3d_qualifier_end( chars : ArrayView[Char], start : Int, -) -> Int? { + budget : FormulaTranslationBudget, +) -> Int? raise XlsxError { if start < 0 || start >= chars.length() || - !formula_reference_start_boundary(chars, start) { + !formula_reference_start_boundary(chars, start, budget) { return None } let mut index = start - while index < chars.length() && formula_unquoted_sheet_name_char(chars[index]) { + while index < chars.length() { + if !formula_unquoted_sheet_name_char( + formula_character(chars, index, budget), + ) { + break + } index += 1 } - if index == start || index >= chars.length() || chars[index] != ':' { + if index == start || + index >= chars.length() || + formula_character(chars, index, budget) != ':' { return None } index += 1 let second_start = index - while index < chars.length() && formula_unquoted_sheet_name_char(chars[index]) { + while index < chars.length() { + if !formula_unquoted_sheet_name_char( + formula_character(chars, index, budget), + ) { + break + } index += 1 } - if index == second_start || index >= chars.length() || chars[index] != '!' { + if index == second_start || + index >= chars.length() || + formula_character(chars, index, budget) != '!' { return None } Some(index + 1) } +///| +fn validate_shared_formula_limits( + maximum_input_chars : Int, + maximum_output_chars : Int, + maximum_work_units : Int, +) -> Unit raise XlsxError { + if maximum_input_chars < 0 || + maximum_output_chars < 0 || + maximum_work_units < 0 { + raise InvalidOptions( + msg="shared formula translation limits cannot be negative", + ) + } +} + ///| /// Applies Excel copy semantics to A1 references while preserving formula -/// text outside reference tokens. Double-quoted literals, quoted sheet names, -/// unquoted 3-D sheet ranges, structured-reference brackets, and -/// external-workbook brackets are copied verbatim. Cell/range, whole-column, -/// and whole-row references honor `$` anchors; translations outside the XLSX -/// grid become `#REF!`. -fn translate_shared_formula( +/// text outside reference tokens. The input ceiling is checked before the +/// bounded character copy is allocated; every scan and write charges work, +/// every write charges output, and long tokens poll cancellation. +fn translate_shared_formula_limited( formula : String, column_delta : Int, row_delta : Int, -) -> String { + maximum_input_chars : Int, + maximum_output_chars : Int, + maximum_work_units : Int, + cancelled : () -> Bool, +) -> (String, Int) raise XlsxError { + validate_shared_formula_limits( + maximum_input_chars, maximum_output_chars, maximum_work_units, + ) + let input_chars = formula.length() + if input_chars > maximum_input_chars { + raise ResourceLimitExceeded( + kind="shared_formula_input_chars", + limit=maximum_input_chars, + actual=input_chars, + ) + } + let budget = FormulaTranslationBudget::new(maximum_work_units, cancelled) + budget.checkpoint() + // Charge the scalar materialization before allocating it. Subsequent reads + // and writes are charged separately so adversarial rescans remain bounded. + budget.charge_work(input_chars) if formula == "" || (column_delta == 0 && row_delta == 0) { - return formula + if input_chars > maximum_output_chars { + raise ResourceLimitExceeded( + kind="shared_formula_output_chars", + limit=maximum_output_chars, + actual=input_chars, + ) + } + return (formula, budget.work_units) } let chars = formula.to_array() - let out = StringBuilder::new() + budget.checkpoint() + let out = FormulaTranslationOutput::new( + input_chars.min(maximum_output_chars), + maximum_output_chars, + budget, + ) let mut index = 0 while index < chars.length() { - let character = chars[index] + let character = formula_character(chars, index, budget) if character == '"' || character == '\'' { index = copy_formula_quoted_token(chars, index, character, out) continue @@ -551,37 +770,53 @@ fn translate_shared_formula( index = copy_formula_bracket_token(chars, index, out) continue } - match formula_unquoted_3d_qualifier_end(chars, index) { + match formula_unquoted_3d_qualifier_end(chars, index, budget) { Some(end) => { - for position in index.. () } let translated = match - translate_formula_cell_reference_at(chars, index, column_delta, row_delta) { - Some(value) => Some(value) + translate_formula_cell_reference_at( + chars, index, column_delta, row_delta, out, + ) { + Some(next) => Some(next) None => - match translate_formula_column_range_at(chars, index, column_delta) { - Some(value) => Some(value) - None => translate_formula_row_range_at(chars, index, row_delta) + match + translate_formula_column_range_at(chars, index, column_delta, out) { + Some(next) => Some(next) + None => translate_formula_row_range_at(chars, index, row_delta, out) } } match translated { - Some((text, next)) => { - out.write_string(text) - index = next - } + Some(next) => index = next None => { out.write_char(character) index += 1 } } } - out.to_string() + (out.builder.to_string(), budget.work_units) +} + +///| +fn translate_shared_formula( + formula : String, + column_delta : Int, + row_delta : Int, +) -> String { + let (translated, _) = try! translate_shared_formula_limited( + formula, + column_delta, + row_delta, + 0x7fffffff, + 0x7fffffff, + 0x7fffffff, + () => false, + ) + translated } ///| @@ -596,6 +831,34 @@ fn SharedFormulaMaster::contains_coordinate( column <= self.column_hi } +///| +/// Resolves formula text at a follower coordinate with explicit input, +/// output, work, and cancellation limits. The returned work count lets a +/// caller maintain one aggregate translation budget across many cells. +pub fn SharedFormulaMaster::translate_to_limited( + self : SharedFormulaMaster, + row : Int, + column : Int, + maximum_input_chars~ : Int, + maximum_output_chars~ : Int, + maximum_work_units~ : Int, + cancelled? : () -> Bool = () => false, +) -> (String, Int) raise XlsxError { + ignore(cell_ref_from(row, column)) + if !self.contains_coordinate(row, column) { + raise InvalidXml(msg="shared formula follower is outside the master range") + } + translate_shared_formula_limited( + self.formula, + column - self.column, + row - self.row, + maximum_input_chars, + maximum_output_chars, + maximum_work_units, + cancelled, + ) +} + ///| /// Resolves the formula text represented by this shared master at a follower /// coordinate. Coordinates outside the master's declared shared range are @@ -605,11 +868,14 @@ pub fn SharedFormulaMaster::translate_to( row : Int, column : Int, ) -> String raise XlsxError { - ignore(cell_ref_from(row, column)) - if !self.contains_coordinate(row, column) { - raise InvalidXml(msg="shared formula follower is outside the master range") - } - translate_shared_formula(self.formula, column - self.column, row - self.row) + let (translated, _) = self.translate_to_limited( + row, + column, + maximum_input_chars=0x7fffffff, + maximum_output_chars=0x7fffffff, + maximum_work_units=0x7fffffff, + ) + translated } ///| @@ -645,3 +911,60 @@ test "shared formulas preserve unquoted 3-D sheet qualifiers" { ), ) } + +///| +test "shared formula limits reject input output and work independently" { + try + translate_shared_formula_limited("A1", 1, 0, 1, 10, 100, () => false) + catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="shared_formula_input_chars") + assert_eq(limit, 1) + assert_eq(actual, 2) + } + _ => fail("unexpected shared formula input limit error") + } noraise { + _ => fail("expected shared formula input limit") + } + try + translate_shared_formula_limited("A1", 1, 0, 10, 1, 100, () => false) + catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="shared_formula_output_chars") + assert_eq(limit, 1) + assert_true(actual > limit) + } + _ => fail("unexpected shared formula output limit error") + } noraise { + _ => fail("expected shared formula output limit") + } + try + translate_shared_formula_limited("A1", 1, 0, 10, 10, 2, () => false) + catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="shared_formula_work_units") + assert_eq(limit, 2) + assert_true(actual > limit) + } + _ => fail("unexpected shared formula work limit error") + } noraise { + _ => fail("expected shared formula work limit") + } +} + +///| +test "shared formula translation polls inside long quoted tokens" { + let checks = [0] + let formula = "\"" + "a".repeat(9000) + "\"+A1" + try + translate_shared_formula_limited(formula, 1, 0, 10000, 10000, 100000, () => { + checks[0] = checks[0] + 1 + checks[0] >= 4 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("unexpected shared formula cancellation error") + } noraise { + _ => fail("expected shared formula cancellation") + } +} diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt index 3a7d987e..eb9c290f 100644 --- a/xlsx/shared_formula_validation_test.mbt +++ b/xlsx/shared_formula_validation_test.mbt @@ -88,6 +88,33 @@ test "XLSX read ignores expressions stored on shared-formula followers" { fail("missing shared-formula master") } assert_eq(master.translate_to(1, 2), "E1+1") + let (bounded, work) = master.translate_to_limited( + 1, + 2, + maximum_input_chars=4, + maximum_output_chars=4, + maximum_work_units=256, + ) + assert_eq(bounded, "E1+1") + assert_true(work > 0) + try + master.translate_to_limited( + 1, + 2, + maximum_input_chars=3, + maximum_output_chars=4, + maximum_work_units=256, + ) + catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "shared_formula_input_chars") + assert_eq(limit, 3) + assert_eq(actual, 4) + } + _ => fail("unexpected bounded shared-formula error") + } noraise { + _ => fail("expected bounded shared-formula rejection") + } } ///| From 5e687e3ab57fac6105a3b79f8020ce422232470b Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:04:24 +0800 Subject: [PATCH 079/150] fix(xlsx): preserve shared formulas across consumers --- xlsx/formula_builtins_stats.mbt | 12 +++ xlsx/shared_formula_translate.mbt | 20 ++++- xlsx/shared_formula_validation_test.mbt | 106 ++++++++++++++++++++++++ xlsx/worksheet.mbt | 70 ++++++++++++++-- 4 files changed, 202 insertions(+), 6 deletions(-) diff --git a/xlsx/formula_builtins_stats.mbt b/xlsx/formula_builtins_stats.mbt index bb8a5803..52a35268 100644 --- a/xlsx/formula_builtins_stats.mbt +++ b/xlsx/formula_builtins_stats.mbt @@ -1922,6 +1922,18 @@ fn calc_cell_value_internal( if cell.row == row && cell.col == col { match cell.formula { Some(formula) => { + let formula = match + (formula, cell.formula_type, cell.formula_shared_index) { + ("", Some(Shared), Some(shared_index)) => + match sheet.shared_formula_master(shared_index) { + Some(master) => master.translate_to(row, col) + None => + raise InvalidXml(msg="shared formula follower has no master") + } + ("", Some(Shared), None) => + raise InvalidXml(msg="shared formula index missing") + _ => formula + } let expr = parse_formula_expr(formula) let prev_sheet = ctx.current_sheet let prev_ref = ctx.current_ref diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index a80029a1..7e025029 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -628,7 +628,13 @@ fn copy_formula_bracket_token( let character = formula_character(chars, index, out.budget) out.write_char(character) index += 1 - if character == '[' { + if character == '\'' && index < chars.length() { + // Structured-reference column names use apostrophe to escape syntax + // characters (notably `[` and `]`). The escaped character is data, so it + // must not affect bracket nesting or become eligible for A1 translation. + out.write_char(formula_character(chars, index, out.budget)) + index += 1 + } else if character == '[' { depth += 1 } else if character == ']' { depth -= 1 @@ -890,6 +896,18 @@ test "shared formulas translate relative A1 references without touching literals ) } +///| +test "shared formulas preserve apostrophe-escaped structured-reference brackets" { + inspect( + translate_shared_formula("Table1[']A1]+B1", 1, 0), + content="Table1[']A1]+C1", + ) + inspect( + translate_shared_formula("Table1['[A1]+B1", 1, 0), + content="Table1['[A1]+C1", + ) +} + ///| test "shared formulas emit REF for translated coordinates outside the grid" { inspect( diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt index eb9c290f..978aa7b9 100644 --- a/xlsx/shared_formula_validation_test.mbt +++ b/xlsx/shared_formula_validation_test.mbt @@ -117,6 +117,112 @@ test "XLSX read ignores expressions stored on shared-formula followers" { } } +///| +test "XLSX calculation resolves shared-formula followers through their master" { + let source = @xlsx.Workbook::new() + let sheet = source.add_sheet("Data") + sheet.set_cell_value("A1", Numeric(1.0)) + sheet.set_cell_value("B1", Numeric(2.0)) + sheet.set_cell_value("A2", Numeric(3.0)) + sheet.set_cell_value("B2", Numeric(4.0)) + sheet.set_cell_formula_opts( + "C1", + "A1+B1", + opts=@xlsx.FormulaOpts::shared("C1:C2"), + ) + let parsed = @xlsx.read(@xlsx.write(source)) + inspect(parsed.calc_cell_value("Data", "C1"), content="3") + inspect(parsed.calc_cell_value("Data", "C2"), content="7") +} + +///| +test "XLSX read accepts a singleton shared-formula ST_Ref" { + let source = @xlsx.Workbook::new() + let sheet = source.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "1+1", + opts=@xlsx.FormulaOpts::shared("A1:A1"), + ) + let archive = @zip.read(@xlsx.write(source)) + let path = "xl/worksheets/sheet1.xml" + let xml = match archive.get(path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("shared-formula worksheet is not UTF-8") + } + None => fail("missing shared-formula worksheet") + } + let updated = xml.replace(old="ref=\"A1:A1\"", new="ref=\"A1\"") + assert_true(updated != xml) + assert_true(archive.replace(path, @encoding/utf8.encode(updated))) + let parsed = @xlsx.read(@zip.write(archive)) + guard parsed.sheet("Data") is Some(parsed_sheet) else { + fail("missing singleton shared-formula worksheet") + } + guard parsed_sheet.shared_formula_masters().get(0) is Some(master) else { + fail("missing singleton shared-formula master") + } + assert_eq(master.translate_to(1, 1), "1+1") +} + +///| +fn shared_formula_edit_fixture() -> @xlsx.Workbook raise { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "C1", + "A1+B1", + opts=@xlsx.FormulaOpts::shared("C1:C3"), + ) + workbook +} + +///| +fn assert_structural_edit_deshared( + workbook : @xlsx.Workbook, + references : ArrayView[String], +) -> Unit raise { + guard workbook.sheet("Data") is Some(sheet) else { + fail("missing structurally edited worksheet") + } + assert_eq(sheet.shared_formula_masters().length(), 0) + for reference in references { + assert_true(sheet.get_cell_formula(reference) is Some(_)) + } + let roundtripped = @xlsx.read(@xlsx.write(workbook)) + guard roundtripped.sheet("Data") is Some(roundtripped_sheet) else { + fail("missing roundtripped structurally edited worksheet") + } + assert_eq(roundtripped_sheet.shared_formula_masters().length(), 0) + for reference in references { + assert_true(roundtripped_sheet.get_cell_formula(reference) is Some(_)) + } +} + +///| +test "row and column edits materialize shared formulas before moving cells" { + let insert_rows = shared_formula_edit_fixture() + insert_rows.insert_rows("Data", 1, 1) + assert_structural_edit_deshared(insert_rows, ["C2", "C3", "C4"]) + + let remove_rows = shared_formula_edit_fixture() + remove_rows.remove_rows("Data", 2, 1) + assert_structural_edit_deshared(remove_rows, ["C1", "C2"]) + + let insert_cols = shared_formula_edit_fixture() + insert_cols.insert_cols("Data", 3, 1) + assert_structural_edit_deshared(insert_cols, ["D1", "D2", "D3"]) + + let remove_cols = shared_formula_edit_fixture() + remove_cols.remove_cols("Data", 2, 1) + assert_structural_edit_deshared(remove_cols, ["B1", "B2", "B3"]) + + let duplicate_row = shared_formula_edit_fixture() + duplicate_row.duplicate_row_to("Data", 1, 4) + assert_structural_edit_deshared(duplicate_row, ["C1", "C2", "C3", "C4"]) +} + ///| test "XLSX read rejects overlapping shared-formula master ranges" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index cd643a87..65ae3a29 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -23,6 +23,10 @@ fn parse_range_ref( for part in range_ref.split(":") { parts.push(part.to_owned()) } + if parts.length() == 1 { + let (row, col) = cell_ref_to_rc(parts[0]) + return (row, col, row, col) + } if parts.length() != 2 { raise InvalidCellRef(value=range_ref.to_owned()) } @@ -501,6 +505,42 @@ fn clone_shared_formula_masters_index( cloned } +///| +/// Converts one shared-formula cell to an independent normal formula before a +/// structural edit moves or duplicates it. Shared ranges are metadata over +/// fixed coordinates; retaining that metadata while coordinates change makes +/// the group internally inconsistent. Materializing each cell preserves the +/// formula represented at its original coordinate and makes subsequent cell +/// movement follow the same rules as every other normal formula. +fn normal_formula_cell_for_structural_edit( + cell : Cell, + masters : Map[UInt, SharedFormulaMaster], +) -> Cell raise XlsxError { + match (cell.formula_type, cell.formula_shared_index) { + (Some(Shared), Some(shared_index)) => { + let master = match masters.get(shared_index) { + Some(value) => value + None => raise InvalidXml(msg="shared formula cell has no master") + } + { + reference: cell.reference, + row: cell.row, + col: cell.col, + value: cell.value, + value_type: cell.value_type, + rich_text: cell.rich_text, + formula: Some(master.translate_to(cell.row, cell.col)), + formula_type: None, + formula_ref: None, + formula_shared_index: None, + formula_value_present: cell.formula_value_present, + style_id: cell.style_id, + } + } + _ => cell + } +} + ///| fn Worksheet::invalidate_shared_formula_masters_index(self : Worksheet) -> Unit { self.shared_formula_masters_index.clear() @@ -3441,6 +3481,7 @@ fn Worksheet::insert_rows( msg="row count \{count} exceeds the sheet's \{cell_ref_max_rows} rows", ) } + let shared_formula_masters = validated_shared_formula_masters(self.cells) // Transactional: stage every shifted collection into locals first — cell, // merge, hyperlink, filter, table, sparkline, image, and chart references // shift through `cell_ref_from`/range adjusters that raise once a coordinate @@ -3449,7 +3490,10 @@ fn Worksheet::insert_rows( // `self` is mutated until every stage succeeds, so an overflowing insert // raises and leaves the worksheet unchanged instead of half-shifted. let updated_cells : Array[Cell] = [] - for cell in self.cells { + for stored_cell in self.cells { + let cell = normal_formula_cell_for_structural_edit( + stored_cell, shared_formula_masters, + ) if cell.row >= row { let new_row = cell.row + count let reference = cell_ref_from(new_row, cell.col) @@ -3596,9 +3640,13 @@ fn Worksheet::remove_rows( msg="row count \{count} exceeds the sheet's \{cell_ref_max_rows} rows", ) } + let shared_formula_masters = validated_shared_formula_masters(self.cells) let end_row = row + count - 1 let updated_cells : Array[Cell] = [] - for cell in self.cells { + for stored_cell in self.cells { + let cell = normal_formula_cell_for_structural_edit( + stored_cell, shared_formula_masters, + ) if cell.row < row { updated_cells.push(cell) continue @@ -3904,8 +3952,12 @@ fn Worksheet::duplicate_row_to( if target_row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{target_row}") } + let shared_formula_masters = validated_shared_formula_masters(self.cells) let row_cells : Array[Cell] = [] - for cell in self.cells { + for stored_cell in self.cells { + let cell = normal_formula_cell_for_structural_edit( + stored_cell, shared_formula_masters, + ) if cell.row == row { row_cells.push(cell) } @@ -3958,12 +4010,16 @@ fn Worksheet::insert_cols( msg="column count \{count} exceeds the sheet's \{cell_ref_max_cols} columns", ) } + let shared_formula_masters = validated_shared_formula_masters(self.cells) // Transactional (see insert_rows): stage everything into locals — reference // shifts raise once a coordinate leaves the grid, column dimensions and page // breaks are bound-checked explicitly — and commit only once every stage has // succeeded, so an overflowing insert leaves the worksheet unchanged. let updated_cells : Array[Cell] = [] - for cell in self.cells { + for stored_cell in self.cells { + let cell = normal_formula_cell_for_structural_edit( + stored_cell, shared_formula_masters, + ) if cell.col >= col { let new_col = cell.col + count let reference = cell_ref_from(cell.row, new_col) @@ -4106,9 +4162,13 @@ fn Worksheet::remove_cols( msg="column count \{count} exceeds the sheet's \{cell_ref_max_cols} columns", ) } + let shared_formula_masters = validated_shared_formula_masters(self.cells) let end_col = col + count - 1 let updated_cells : Array[Cell] = [] - for cell in self.cells { + for stored_cell in self.cells { + let cell = normal_formula_cell_for_structural_edit( + stored_cell, shared_formula_masters, + ) if cell.col < col { updated_cells.push(cell) continue From 245e6333afcfcdca8d83b59e8418e729fb50f402 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:15:01 +0800 Subject: [PATCH 080/150] fix(xlsx): reject ambiguous package markup --- ooxml/start_tag_scanner.mbt | 18 ++++++++++ xlsx/ooxml_rels.mbt | 70 ++++++++++++++++++++++++++++++++++-- xlsx/read.mbt | 8 ++++- xlsx/read_shared_strings.mbt | 51 ++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 4 deletions(-) diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 21d39db6..951ce27e 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -1445,6 +1445,7 @@ pub fn XmlStartTagScanner::next( Some(value) => value None => raise InvalidXml(msg="XML CDATA section is not closed") } + validate_xml_character_span(self.xml, start + 9, end, self.cancelled) if self.canonicalize_markup { self.record_rewrite( start, @@ -2147,6 +2148,23 @@ test "expanded-name canonicalization bounds escaped CDATA output" { } } +///| +test "strict start-tag scanning rejects XML-invalid controls in CDATA" { + let xml = "" + let scanner = XmlStartTagScanner::new(xml) + try { + while scanner.next() { + + } + } catch { + InvalidXml(msg~) => + assert_eq(msg, "XML markup contains an invalid character") + _ => fail("unexpected CDATA validation error") + } noraise { + _ => fail("XML-invalid CDATA control was accepted") + } +} + ///| test "start tag scanner bounds qualified names" { let scanner = XmlStartTagScanner::new("<" + "a".repeat(1025) + "/>") diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 57b5943f..5b5a448c 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -167,21 +167,66 @@ fn drop_first_path_segment(path : StringView) -> String { } } +///| +fn relationship_target_has_ambiguous_first_segment_colon( + target : StringView, +) -> Bool { + if target.has_prefix("/") { + return false + } + for character in target { + if character == '/' { + return false + } + if character == ':' { + return true + } + } + false +} + +///| +fn validate_relationship_target_uri( + target : StringView, +) -> Unit raise XlsxError { + if target == "" || + target.has_prefix("//") || + target.contains("\\") || + target.contains("?") || + target.contains("#") || + relationship_target_has_ambiguous_first_segment_colon(target) { + raise InvalidXml(msg="relationship target invalid") + } +} + ///| fn normalize_rel_part_path(path : StringView) -> String raise XlsxError { let segments : Array[String] = [] - for chunk in path.split("/") { + let chunks = path.split("/").collect() + for index, chunk in chunks { let segment = chunk.to_owned() - if segment == "" || segment == "." { + if segment == "" { + // Empty URI path segments are significant. Silently removing them lets + // malformed targets such as `xl//workbook.xml` alias valid package parts. + raise InvalidXml(msg="relationship target invalid") + } + if segment == "." { + // A terminal dot segment resolves to a directory URI, not an OPC part. + if index + 1 == chunks.length() { + raise InvalidXml(msg="relationship target invalid") + } continue } if segment == ".." { - if segments.length() == 0 { + if segments.length() == 0 || index + 1 == chunks.length() { raise InvalidXml(msg="relationship target invalid") } ignore(segments.pop()) continue } + if !@ooxml.is_valid_opc_part_segment(segment) { + raise InvalidXml(msg="relationship target invalid") + } segments.push(segment) } if segments.length() == 0 { @@ -200,6 +245,7 @@ fn resolve_part_rel_target( if source == "" || source.has_prefix("/") || target_str == "" { raise InvalidXml(msg="relationship target invalid") } + validate_relationship_target_uri(target_str) let normalized_source = normalize_rel_part_path(source) if normalized_source != source { raise InvalidXml(msg="relationship target invalid") @@ -221,6 +267,7 @@ fn resolve_rel_target( base_folder : StringView, ) -> String raise XlsxError { let target_str = target.to_owned() + validate_relationship_target_uri(target_str) let raw = if target_str.has_prefix("/") { drop_first_path_segment(target_str) } else if target_str.has_prefix("../") { @@ -460,3 +507,20 @@ test "ooxml_rels: normalize and resolve invalid-path guards" { _ => fail("expected InvalidXml for non-xl absolute resolve_rel_target") } } + +///| +test "ooxml_rels: invalid empty and terminal path segments never alias parts" { + for + target in [ + "worksheets//sheet1.xml", "worksheets/sheet1.xml/.", "worksheets/sheet1.xml/", + "https://example.invalid/sheet1.xml", "worksheets\\sheet1.xml", "worksheets/sheet1.xml?query", + "worksheets/sheet1.xml#fragment", + ] { + try resolve_workbook_rel_target(target) catch { + InvalidXml(msg~) => assert_eq(msg, "relationship target invalid") + _ => fail("unexpected relationship target error") + } noraise { + _ => fail("malformed relationship target was accepted: \{target}") + } + } +} diff --git a/xlsx/read.mbt b/xlsx/read.mbt index cbd284e6..c1848695 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3657,7 +3657,13 @@ fn read_zip_archive_core_unchecked( ) } let theme_path = workbook_related_part_path( - workbook_rels, rel_theme, workbook_xml_part_path, theme_part_path, part_names, + workbook_rels, + rel_theme, + workbook_xml_part_path, + theme_part_path, + part_names, + budget=read_budget, + cancelled~, ) let theme_xml = match theme_path { Some(path) => diff --git a/xlsx/read_shared_strings.mbt b/xlsx/read_shared_strings.mbt index 99c475b4..d34138f5 100644 --- a/xlsx/read_shared_strings.mbt +++ b/xlsx/read_shared_strings.mbt @@ -368,6 +368,38 @@ fn parse_shared_strings( xml : StringView, budget? : ReadBudget, ) -> Array[SharedStringEntry] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => () + } + let scanner = @ooxml.XmlStartTagScanner::new( + xml, + cancelled=match budget { + Some(value) => value.cancelled + None => () => false + }, + ) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "sst" || + ( + scanner.namespace_uri() != "" && + scanner.namespace_uri() != transitional_spreadsheet_namespace && + scanner.namespace_uri() != strict_spreadsheet_namespace + ) { + raise InvalidXml(msg="shared strings document element is invalid") + } + let shared_strings_namespace = scanner.namespace_uri().to_owned() + while workbook_scanner_next(scanner) { + if scanner.namespace_uri() == shared_strings_namespace && + scanner.local_name() == "si" && + scanner.depth() != 2 { + raise InvalidXml(msg="shared string item is not a direct child") + } + } match budget { Some(value) => { value.checkpoint() @@ -407,3 +439,22 @@ fn parse_shared_strings( } values } + +///| +test "shared strings reject nested same-namespace index decoys" { + let valid = + #|real + let values = parse_shared_strings(valid) + assert_eq(values.length(), 1) + assert_eq(values[0].text, "real") + + let nested = + #|decoyreal + try parse_shared_strings(nested) catch { + InvalidXml(msg~) => + assert_eq(msg, "shared string item is not a direct child") + _ => fail("unexpected nested shared-string error") + } noraise { + _ => fail("nested shared-string decoy was accepted") + } +} From a7c89e9487f56e25f3a9fafeb425b951bf84c8f5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:24:25 +0800 Subject: [PATCH 081/150] fix(ooxml): apply MCE to relationship parts --- ooxml/read_parse.mbt | 346 ++++++++++++++++++++++++++++++++++++-- ooxml/read_parse_test.mbt | 91 ++++++++++ 2 files changed, 427 insertions(+), 10 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index fae3bcf0..46f904d7 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -85,6 +85,9 @@ let package_relationships_namespace = "http://schemas.openxmlformats.org/package ///| let package_content_types_namespace = "http://schemas.openxmlformats.org/package/2006/content-types" +///| +let markup_compatibility_namespace = "http://schemas.openxmlformats.org/markup-compatibility/2006" + ///| let max_relationship_records = 1_000_000 @@ -165,6 +168,325 @@ priv enum RelationshipTargetSelection { ExternalTargets } +///| +/// Streaming state for the MCE-effective projection of an OPC Relationships +/// part. Each raw XML depth owns one state; wrapper elements selected by +/// `ProcessContent`, `Choice`, or `Fallback` retain their parent's effective +/// depth so their application children are promoted without materializing a +/// second XML tree. +priv struct RelationshipMceState { + active : Bool + included_in_effective_tree : Bool + element_effective_depth : Int + children_effective_parent_depth : Int + ignorable_namespaces : Set[String] + process_content_names : Set[String] + is_alternate_content : Bool + mut alternate_choice_seen : Bool + mut alternate_fallback_seen : Bool + mut alternate_selected : Bool +} + +///| +fn clone_relationship_mce_set(source : Set[String]) -> Set[String] { + let cloned : Set[String] = Set([]) + for value in source { + cloned.add(value) + } + cloned +} + +///| +fn relationship_mce_tokens( + value : StringView, + label : StringView, +) -> Array[String] raise ParseXmlError { + let normalized = collapse_schema_whitespace(value) + if normalized == "" { + raise InvalidXml(msg="MCE \{label.to_owned()} is empty") + } + let tokens : Array[String] = [] + for token in normalized.split(" ") { + if token == "" || token.length() > max_relationship_id_chars { + raise InvalidXml(msg="MCE \{label.to_owned()} is invalid") + } + if tokens.length() >= 4096 { + raise InvalidXml(msg="MCE \{label.to_owned()} token limit exceeded") + } + tokens.push(token.to_owned()) + } + tokens +} + +///| +fn relationship_mce_resolve_prefix( + scanner : XmlStartTagScanner, + prefix : StringView, + label : StringView, +) -> String raise ParseXmlError { + if prefix == "" || prefix.contains(":") { + raise InvalidXml(msg="MCE \{label.to_owned()} prefix is invalid") + } + scanner.resolve_prefix(prefix, false) +} + +///| +fn relationship_mce_expanded_name_key( + scanner : XmlStartTagScanner, + token : StringView, + label : StringView, +) -> (String, String) raise ParseXmlError { + let parts = token.split(":").collect() + if parts.length() != 2 || parts[0] == "" || parts[1] == "" { + raise InvalidXml(msg="MCE \{label.to_owned()} name is invalid") + } + let namespace_uri = relationship_mce_resolve_prefix(scanner, parts[0], label) + (namespace_uri, namespace_uri + "\u{0}" + parts[1].to_owned()) +} + +///| +fn relationship_mce_scope( + scanner : XmlStartTagScanner, + parent : RelationshipMceState?, +) -> (Set[String], Set[String]) raise ParseXmlError { + let ignorable_attribute = scanner.attribute( + markup_compatibility_namespace, "Ignorable", + ) + let ignorable = match (parent, ignorable_attribute) { + (Some(value), None) => value.ignorable_namespaces + (Some(value), Some(_)) => + clone_relationship_mce_set(value.ignorable_namespaces) + (None, _) => Set([]) + } + match ignorable_attribute { + Some(value) => + for prefix in relationship_mce_tokens(value, "Ignorable") { + let namespace_uri = relationship_mce_resolve_prefix( + scanner, prefix, "Ignorable", + ) + if namespace_uri == "" || + namespace_uri == markup_compatibility_namespace || + namespace_uri == package_relationships_namespace { + raise InvalidXml(msg="MCE Ignorable namespace is invalid") + } + ignorable.add(namespace_uri) + } + None => () + } + let process_content_attribute = scanner.attribute( + markup_compatibility_namespace, "ProcessContent", + ) + let process_content = match (parent, process_content_attribute) { + (Some(value), None) => value.process_content_names + (Some(value), Some(_)) => + clone_relationship_mce_set(value.process_content_names) + (None, _) => Set([]) + } + match process_content_attribute { + Some(value) => + for token in relationship_mce_tokens(value, "ProcessContent") { + let (namespace_uri, key) = relationship_mce_expanded_name_key( + scanner, token, "ProcessContent", + ) + if !ignorable.contains(namespace_uri) { + raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") + } + if token.has_suffix(":*") { + raise InvalidXml(msg="MCE ProcessContent wildcard is invalid") + } + process_content.add(key) + } + None => () + } + match scanner.attribute(markup_compatibility_namespace, "MustUnderstand") { + Some(value) => + for prefix in relationship_mce_tokens(value, "MustUnderstand") { + if relationship_mce_resolve_prefix(scanner, prefix, "MustUnderstand") != + package_relationships_namespace { + raise InvalidXml(msg="MCE MustUnderstand namespace is unsupported") + } + } + None => () + } + for directive in ["PreserveElements", "PreserveAttributes"] { + match scanner.attribute(markup_compatibility_namespace, directive) { + Some(value) => + for token in relationship_mce_tokens(value, directive) { + let (namespace_uri, _) = relationship_mce_expanded_name_key( + scanner, token, directive, + ) + if !ignorable.contains(namespace_uri) { + raise InvalidXml(msg="MCE \{directive} namespace is not ignorable") + } + } + None => () + } + } + (ignorable, process_content) +} + +///| +fn relationship_mce_choice_supported( + scanner : XmlStartTagScanner, +) -> Bool raise ParseXmlError { + let requires = match scanner.attribute("", "Requires") { + Some(value) => value + None => raise InvalidXml(msg="MCE Choice is missing Requires") + } + let mut supported = true + for prefix in relationship_mce_tokens(requires, "Choice Requires") { + if relationship_mce_resolve_prefix(scanner, prefix, "Choice Requires") != + package_relationships_namespace { + supported = false + } + } + supported +} + +///| +fn trim_relationship_mce_states( + states : Array[RelationshipMceState], + next_depth : Int, +) -> Unit raise ParseXmlError { + while states.length() >= next_depth { + let state = states.pop().unwrap() + if state.is_alternate_content && !state.alternate_choice_seen { + raise InvalidXml(msg="MCE AlternateContent has no Choice element") + } + } +} + +///| +fn relationship_mce_state_for_current_element( + scanner : XmlStartTagScanner, + states : Array[RelationshipMceState], +) -> RelationshipMceState raise ParseXmlError { + let depth = scanner.depth() + trim_relationship_mce_states(states, depth) + let parent = if depth > 1 { + if states.length() != depth - 1 { + raise InvalidXml(msg="relationships document nesting is invalid") + } + Some(states[depth - 2]) + } else { + None + } + match parent { + Some(parent_state) if !parent_state.active => + return { + active: false, + included_in_effective_tree: false, + element_effective_depth: parent_state.children_effective_parent_depth, + children_effective_parent_depth: parent_state.children_effective_parent_depth, + ignorable_namespaces: parent_state.ignorable_namespaces, + process_content_names: parent_state.process_content_names, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } + _ => () + } + let (ignorable, process_content) = relationship_mce_scope(scanner, parent) + if depth == 1 { + if scanner.namespace_uri() != package_relationships_namespace || + scanner.local_name() != "Relationships" { + raise InvalidXml(msg="relationships document element invalid") + } + return { + active: true, + included_in_effective_tree: true, + element_effective_depth: 1, + children_effective_parent_depth: 1, + ignorable_namespaces: ignorable, + process_content_names: process_content, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } + } + let parent_state = parent.unwrap() + let namespace_uri = scanner.namespace_uri().to_owned() + let local_name = scanner.local_name().to_owned() + let parent_effective_depth = parent_state.children_effective_parent_depth + let mut active = parent_state.active + let mut included_in_effective_tree = false + let mut element_effective_depth = parent_effective_depth + let mut children_effective_parent_depth = parent_effective_depth + let mut is_alternate_content = false + + if parent_state.is_alternate_content { + if namespace_uri == markup_compatibility_namespace && local_name == "Choice" { + if parent_state.alternate_fallback_seen { + raise InvalidXml(msg="MCE Choice follows Fallback") + } + parent_state.alternate_choice_seen = true + let supported = relationship_mce_choice_supported(scanner) + active = parent_state.active && + !parent_state.alternate_selected && + supported + if active { + parent_state.alternate_selected = true + } + } else if namespace_uri == markup_compatibility_namespace && + local_name == "Fallback" { + if parent_state.alternate_fallback_seen { + raise InvalidXml( + msg="MCE AlternateContent has multiple Fallback elements", + ) + } + parent_state.alternate_fallback_seen = true + active = parent_state.active && !parent_state.alternate_selected + if active { + parent_state.alternate_selected = true + } + } else if ignorable.contains(namespace_uri) { + active = false + } else { + raise InvalidXml(msg="MCE AlternateContent child is invalid") + } + } else if namespace_uri == markup_compatibility_namespace { + if local_name != "AlternateContent" { + raise InvalidXml(msg="MCE element is misplaced") + } + active = parent_state.active + is_alternate_content = true + } else if ignorable.contains(namespace_uri) && + namespace_uri != package_relationships_namespace { + let key = namespace_uri + "\u{0}" + local_name + if process_content.contains(key) { + if scanner.attribute(xml_namespace_uri, "base") is Some(_) || + scanner.attribute(xml_namespace_uri, "lang") is Some(_) || + scanner.attribute(xml_namespace_uri, "space") is Some(_) { + raise InvalidXml( + msg="MCE ProcessContent element has an unsafe xml attribute", + ) + } + active = parent_state.active + } else { + active = false + } + } else { + active = parent_state.active + included_in_effective_tree = true + element_effective_depth = parent_effective_depth + 1 + children_effective_parent_depth = element_effective_depth + } + { + active, + included_in_effective_tree, + element_effective_depth, + children_effective_parent_depth, + ignorable_namespaces: ignorable, + process_content_names: process_content, + is_alternate_content, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } +} + ///| /// Reads relationship targets whose decoded `Type` is in `rel_types`. Element /// names are matched by expanded namespace name, so the package namespace may @@ -178,21 +500,24 @@ fn parse_relationship_targets_by_types_selected( let targets : Map[String, String] = Map([]) let ids : Set[String] = Set([]) let scanner = XmlStartTagScanner::new(xml, cancelled~) - if !scanner.next() || - scanner.depth() != 1 || - scanner.namespace_uri() != package_relationships_namespace || - scanner.local_name() != "Relationships" { - raise InvalidXml(msg="relationships document element invalid") - } + let mce_states : Array[RelationshipMceState] = [] let mut records = 0 let mut retained_chars = 0 while scanner.next() { - if scanner.namespace_uri() != package_relationships_namespace || - scanner.local_name() != "Relationship" { + let mce_state = relationship_mce_state_for_current_element( + scanner, mce_states, + ) + mce_states.push(mce_state) + if mce_state.element_effective_depth == 1 { continue } - if scanner.depth() != 2 { - raise InvalidXml(msg="relationship element is not a direct child") + if !mce_state.active || !mce_state.included_in_effective_tree { + continue + } + if scanner.namespace_uri() != package_relationships_namespace || + scanner.local_name() != "Relationship" || + mce_state.element_effective_depth != 2 { + raise InvalidXml(msg="relationships effective content is invalid") } records = records + 1 if records > max_relationship_records { @@ -240,6 +565,7 @@ fn parse_relationship_targets_by_types_selected( } targets[id] = target } + trim_relationship_mce_states(mce_states, 1) targets } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 2e78f140..7740ffd6 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -106,6 +106,97 @@ test "ooxml read_parse: relationship target modes are explicit and filtered" { ) } +///| +test "ooxml read_parse: relationships apply OPC Markup Compatibility" { + let process_content = + #| + let selected_choice = + #| + let selected_fallback = + #| + let processed = @ooxml.parse_internal_relationship_targets( + process_content, "wanted", + ) + assert_eq(processed.get("processed"), Some("processed.xml")) + assert_eq(processed.length(), 1) + let choice = @ooxml.parse_internal_relationship_targets( + selected_choice, "wanted", + ) + assert_eq(choice.get("choice"), Some("choice.xml")) + assert_eq(choice.length(), 1) + let fallback = @ooxml.parse_internal_relationship_targets( + selected_fallback, "wanted", + ) + assert_eq(fallback.get("fallback"), Some("fallback.xml")) + assert_eq(fallback.length(), 1) +} + +///| +test "ooxml read_parse: invalid relationship MCE fails closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "MCE AlternateContent has no Choice element", + ), + ( + ( + #| + ), + "MCE Choice follows Fallback", + ), + ( + ( + #| + ), + "MCE AlternateContent has multiple Fallback elements", + ), + ( + ( + #| + ), + "MCE MustUnderstand namespace is unsupported", + ), + ( + ( + #| + ), + "MCE ProcessContent wildcard is invalid", + ), + ( + ( + #| + ), + "MCE ProcessContent element has an unsafe xml attribute", + ), + ( + ( + #| + ), + "relationships effective content is invalid", + ), + ] + for case in cases { + let (xml, expected) = case + try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected relationship parse cancellation") + } noraise { + _ => fail("expected invalid relationship MCE") + } + } +} + +///| +test "ooxml read_parse: unselected MCE choices remain opaque" { + let xml = + #| + let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + assert_eq(targets.get("selected"), Some("selected.xml")) + assert_eq(targets.length(), 1) +} + ///| test "ooxml read_parse: invalid target modes fail closed" { let xml = From 317bbf619c3eef75c5cf6b31eddba8768e962884 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:28:33 +0800 Subject: [PATCH 082/150] fix(ooxml): bound part-name trie allocation --- ooxml/opc_part_name.mbt | 165 +++++++++++++++++++++++++++++++--------- 1 file changed, 131 insertions(+), 34 deletions(-) diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt index 3a035756..3bb5ab3c 100644 --- a/ooxml/opc_part_name.mbt +++ b/ooxml/opc_part_name.mbt @@ -258,16 +258,90 @@ pub fn logical_opc_part_name_from_zip_item_name_cancellable( } ///| -fn opc_part_name_identity_unit(unit : UInt16) -> UInt16 { - if unit >= ('A' : UInt16) && unit <= ('Z' : UInt16) { - (unit.to_int() + 32).to_uint16() - } else { - unit +let opc_part_name_identity_chunk_units = 4096 + +///| +priv struct OpcPartNameIdentityToken { + key : String + starts_segment : Bool +} + +///| +fn opc_part_name_identity_chunk( + chunk : StringView, + final_chunk : Bool, + cancelled : () -> Bool, +) -> String raise ParseXmlError { + let output = StringBuilder::new(size_hint=chunk.length() + 1) + output.write_char(if final_chunk { '1' } else { '0' }) + let mut offset = 0 + for character in chunk { + if offset % 4096 == 0 && cancelled() { + raise ReadCancelled + } + output.write_char(character.to_ascii_lowercase()) + offset = offset + (if character.to_int() > 0xffff { 2 } else { 1 }) } + output.to_string() } ///| -type OpcPartNameChildren = @sorted_map.SortedMap[UInt16, Int] +fn opc_part_name_identity_tokens( + name : StringView, + cancelled : () -> Bool, +) -> Array[OpcPartNameIdentityToken] raise ParseXmlError { + let tokens : Array[OpcPartNameIdentityToken] = [] + let mut segment_start = 0 + let mut index = 0 + while index <= name.length() { + if index % 4096 == 0 && cancelled() { + raise ReadCancelled + } + if index == name.length() || name[index] == ('/' : UInt16) { + if segment_start == index { + tokens.push({ + key: opc_part_name_identity_chunk("", true, cancelled), + starts_segment: true, + }) + } else { + let mut chunk_start = segment_start + while chunk_start < index { + let tentative_end = chunk_start + opc_part_name_identity_chunk_units + let mut chunk_end = if tentative_end < index { + tentative_end + } else { + index + } + // Keep a supplementary scalar in one bounded key rather than + // slicing between its UTF-16 surrogate code units. + if chunk_end < index && + name[chunk_end - 1] >= 0xd800 && + name[chunk_end - 1] <= 0xdbff && + name[chunk_end] >= 0xdc00 && + name[chunk_end] <= 0xdfff { + chunk_end = chunk_end - 1 + } + let final_chunk = chunk_end == index + tokens.push({ + key: opc_part_name_identity_chunk( + name[chunk_start:chunk_end], + final_chunk, + cancelled, + ), + starts_segment: chunk_start == segment_start, + }) + chunk_start = chunk_end + } + } + segment_start = index + 1 + } + index = index + 1 + } + tokens +} + +///| +type OpcPartNameChildren = @sorted_map.SortedMap[String, Int] ///| priv struct OpcPartNameTrieNode { @@ -289,10 +363,13 @@ pub(all) enum OpcPartNameConflict { } ///| -/// A UTF-16 code-unit trie for OPC PartName identity and non-derivability checks. -/// Work and storage are linear in aggregate PartName length, while ordered -/// scalar child indexes avoid repeated comparisons of attacker-controlled -/// string prefixes and keep hostile names independent of string-hash behavior. +/// A chunked path-segment trie for OPC PartName identity and non-derivability +/// checks. Each node represents at most 4,096 UTF-16 units, so a very long +/// segment cannot amplify bounded package text into millions of heap objects, +/// while individual ordered-key comparisons remain bounded. Chunk keys encode +/// segment boundaries so character prefixes are never confused with derivable +/// path prefixes. Ordered child indexes also keep hostile names independent of +/// string-hash behavior. pub struct OpcPartNameRegistry { priv nodes : Array[OpcPartNameTrieNode] } @@ -327,44 +404,41 @@ pub fn OpcPartNameRegistry::register_cancellable( if cancelled() { raise ReadCancelled } + let tokens = opc_part_name_identity_tokens(name, cancelled) let mut node_index = 0 - let mut index = 0 - let mut missing_index : Int? = None - while index < name.length() { - if index % 4096 == 0 && cancelled() { + let mut token_index = 0 + let mut missing_token : Int? = None + while token_index < tokens.length() { + if token_index % 4096 == 0 && cancelled() { raise ReadCancelled } - let unit = opc_part_name_identity_unit(name[index]) - if unit == ('/' : UInt16) { + if token_index > 0 && tokens[token_index].starts_segment { match self.nodes[node_index].terminal { Some(existing) => return Some(Derivable(existing)) None => () } } - match self.nodes[node_index].children.get(unit) { + match self.nodes[node_index].children.get(tokens[token_index].key) { Some(child) => node_index = child None => { - missing_index = Some(index) + missing_token = Some(token_index) break } } - index = index + 1 + token_index = token_index + 1 } - if missing_index is None { + if missing_token is None { match self.nodes[node_index].terminal { Some(existing) => return Some(Equivalent(existing)) None => () } - match - self.nodes[node_index].children - .get('/') - .map(child => self.nodes[child].first_part) { - Some(Some(existing)) => return Some(Derivable(existing)) - _ => () + match self.nodes[node_index].first_part { + Some(existing) => return Some(Derivable(existing)) + None => () } } - match missing_index { + match missing_token { None => { if cancelled() { raise ReadCancelled @@ -376,17 +450,14 @@ pub fn OpcPartNameRegistry::register_cancellable( // and cancellation point succeeds do we append and link it once. let base = self.nodes.length() let detached : Array[OpcPartNameTrieNode] = [] - for offset in start.. assert_eq(existing, long_name) + _ => fail("expected equivalent long OPC PartName") + } + assert_true(registry.nodes.length() < 32) +} + +///| +test "OPC PartName chunk boundaries preserve segment semantics" { + let registry = OpcPartNameRegistry::new() + let exact_chunk = "custom/" + "a".repeat(4096) + let longer_segment = exact_chunk + "b" + assert_true(registry.register(exact_chunk, exact_chunk) is None) + assert_true(registry.register(longer_segment, longer_segment) is None) + match registry.register(exact_chunk + "/child", "descendant") { + Some(Derivable(existing)) => assert_eq(existing, exact_chunk) + _ => fail("expected derivable long OPC PartName") + } +} From a11ed3987a3c4bb3b0ef90bd8c5d24dde7becd25 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:36:55 +0800 Subject: [PATCH 083/150] fix(read): avoid eager bounded-buffer copies --- xlsx/cfb.mbt | 7 +++++-- zip/fixed_output.mbt | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/xlsx/cfb.mbt b/xlsx/cfb.mbt index 7a514505..041d745c 100644 --- a/xlsx/cfb.mbt +++ b/xlsx/cfb.mbt @@ -572,7 +572,7 @@ priv struct CfbEntry { ///| priv struct CfbReader { - bytes : Bytes + bytes : BytesView sector_size : Int mini_sector_size : Int fat : Array[Int] @@ -847,7 +847,10 @@ fn cfb_read( mini_stream = Bytes::from_array(data) } { - bytes: Bytes::from_array(bytes.to_array()), + // A BytesView retains its immutable backing allocation. Keep the package + // borrowed instead of duplicating the complete encrypted input after it + // has already passed all structural checks. + bytes, sector_size, mini_sector_size, fat, diff --git a/zip/fixed_output.mbt b/zip/fixed_output.mbt index 0c38d7aa..43f13dd0 100644 --- a/zip/fixed_output.mbt +++ b/zip/fixed_output.mbt @@ -25,12 +25,28 @@ fn FixedByteOutput::allocated(size : Int) -> FixedByteOutput raise ZipError { raise UnsupportedFeature(msg="negative ZIP output size") } { - storage: Some(FixedArray::make(size, b'\x00')), + // Every position is initialized by the exact-size emission pass before + // `finish`. Avoid an eager zero-fill proportional to attacker-bounded ZIP + // output; DEFLATE back-references only read positions below `position`. + storage: Some(uninitialized_fixed_byte_storage(size)), position: 0, limit: Some(size), } } +///| +/// Reinterprets primitive uninitialized storage for incremental initialization. +/// The sink must initialize every element before handing it to immutable Bytes. +fn uninitialized_fixed_byte_storage(size : Int) -> FixedArray[Byte] { + let storage : UninitializedArray[Byte] = UninitializedArray::make(size) + unsafe_uninitialized_byte_storage_to_fixed(storage) +} + +///| +fn unsafe_uninitialized_byte_storage_to_fixed( + storage : UninitializedArray[Byte], +) -> FixedArray[Byte] = "%identity" + ///| fn FixedByteOutput::length(self : FixedByteOutput) -> Int { self.position From e141ea056f8d39f722576330022e2bb04d04ac67 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:46:54 +0800 Subject: [PATCH 084/150] test: guard JavaScript backend --- .github/workflows/ci.yml | 9 +++++++ xlsx/calc_test.mbt | 52 ++++++++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cc4335c..31a1e19b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,12 +163,21 @@ jobs: - name: Type check (wasm) run: moon check --target wasm + # Keep the JavaScript runtime honest as well. Floating-point formatting + # and target-specific package declarations have both regressed here + # without affecting native or Wasm builds. + - name: Type check (js) + run: moon check --target js + # An unfiltered invocation traverses every moon.work member and all of # its packages. Keep this module-wide so new portable child packages are # covered automatically instead of silently receiving root-only tests. - name: Test complete workspace (wasm) run: moon test --target wasm + - name: Test complete workspace (js) + run: moon test --target js + # Run all members' native test suites in one pass. Covers mbtexcel # (including OpenXML SDK schema validation via .NET), docx2html # (converter/reader/fixtures/stress), and pdflite (reader/writer/ diff --git a/xlsx/calc_test.mbt b/xlsx/calc_test.mbt index f09fc10b..61535dae 100644 --- a/xlsx/calc_test.mbt +++ b/xlsx/calc_test.mbt @@ -1,3 +1,26 @@ +///| +/// Numerical solvers and transcendental functions may differ by the final few +/// IEEE-754 digits across MoonBit backends. Compare their numeric meaning while +/// retaining roughly twelve significant decimal digits of precision. +fn assert_calc_numeric_close( + actual : StringView, + expected : StringView, +) -> Unit raise { + let actual_value = @string.parse_double(actual) catch { + _ => fail("expected numeric calculation result, got \{actual}") + } + let expected_value = @string.parse_double(expected) catch { + _ => fail("invalid numeric test expectation \{expected}") + } + let difference = (actual_value - expected_value).abs() + let tolerance = expected_value.abs() * 2.0e-12 + 1.0e-300 + if difference.is_nan() || difference > tolerance { + fail( + "calculation result \{actual} differs from \{expected} by \{difference} (tolerance \{tolerance})", + ) + } +} + ///| test "calc basic operators" { let workbook = @xlsx.Workbook::new() @@ -620,9 +643,9 @@ test "calc chi-square test functions" { sheet.set_cell_formula("H3", "CHITEST(B3:B4,F3:F4)") inspect(workbook.calc_cell_value("Sheet1", "H3"), content="0.152357748933542") sheet.set_cell_formula("H4", "CHITEST(B4:B6,F3:F5)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "H4"), - content="7.070769514407259e-25", + "7.070769514407259e-25", ) sheet.set_cell_formula("H5", "CHISQ.TEST(B3:C5,F3:G5)") inspect( @@ -637,9 +660,9 @@ test "calc chi-square test functions" { sheet.set_cell_formula("H7", "CHISQ.TEST(B3:B4,F3:F4)") inspect(workbook.calc_cell_value("Sheet1", "H7"), content="0.152357748933542") sheet.set_cell_formula("H8", "CHISQ.TEST(B4:B6,F3:F5)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "H8"), - content="7.070769514407259e-25", + "7.070769514407259e-25", ) sheet.set_cell_formula("H9", "CHITEST()") inspect(workbook.calc_cell_value("Sheet1", "H9"), content="#VALUE!") @@ -2248,9 +2271,9 @@ test "calc trig functions" { content="0.7456241416655579", ) sheet.set_cell_formula("U1", "TAN(1.047197551)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "U1"), - content="1.7320508067824865", + "1.7320508067824865", ) sheet.set_cell_formula("U2", "TAN(0)") inspect(workbook.calc_cell_value("Sheet1", "U2"), content="0") @@ -5043,19 +5066,19 @@ test "calc financial functions" { sheet.set_cell_formula("B9", "PV(10%/4,16,2000,0,1)") inspect(workbook.calc_cell_value("Sheet1", "B9"), content="-26762.7554528811") sheet.set_cell_formula("B10", "RATE(60,-1000,50000)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "B10"), - content="0.00618341316212867", + "0.00618341316212867", ) sheet.set_cell_formula("B11", "RATE(24,-800,0,20000,1)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "B11"), - content="0.00325084350160377", + "0.00325084350160377", ) sheet.set_cell_formula("B12", "RATE(48,-200,8000,3,1,0.5)") - inspect( + assert_calc_numeric_close( workbook.calc_cell_value("Sheet1", "B12"), - content="0.00804126658316361", + "0.00804126658316361", ) } @@ -5331,7 +5354,10 @@ test "calc bond functions" { let (formula, expected) = entry let cell = "E\{idx + 1}" sheet.set_cell_formula(cell, formula) - inspect(workbook.calc_cell_value("Sheet1", cell), content=expected) + assert_calc_numeric_close( + workbook.calc_cell_value("Sheet1", cell), + expected, + ) } } From 542c4e812a78794b1c6f1f14171a3c7bacbccaea Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:52:23 +0800 Subject: [PATCH 085/150] fix(xlsx): state async parse cancellation boundary --- xlsx/io.mbt | 56 +++++++++++------------------------------------------ 1 file changed, 11 insertions(+), 45 deletions(-) diff --git a/xlsx/io.mbt b/xlsx/io.mbt index cd0e2c0a..29ddd840 100644 --- a/xlsx/io.mbt +++ b/xlsx/io.mbt @@ -27,26 +27,6 @@ test "bounded file-size telemetry saturates instead of truncating" { inspect(bounded_actual_above_limit(0x7fff_ffff), content="2147483647") } -///| -test "async byte bridge forwards cancellation into semantic parsing" { - let source = Workbook::new() - ignore(source.add_sheet("Sheet1")) - let bytes = write(source) - let result : Result[Workbook, XlsxError] = Ok( - read_workbook_from_bytes( - bytes, - "", - Options::new(), - ReadLimits::new(), - read_io_context(None), - cancelled=() => true, - ), - ) catch { - error => Err(error) - } - inspect(result is Err(ReadCancelled), content="true") -} - ///| async fn[R : @async/io.Reader] read_all_bytes( reader : R, @@ -262,7 +242,6 @@ fn read_workbook_from_bytes( options : Options, limits : ReadLimits, io_context : WorkbookIOContext, - cancelled? : () -> Bool = () => false, ) -> Workbook raise XlsxError { let resolved_password = if password == "" { options.password @@ -277,11 +256,9 @@ fn read_workbook_from_bytes( bytes, options=resolved_options, limits~, - cancelled~, transcoder=value, ) - None => - read_zip_bytes(bytes, options=resolved_options, limits~, cancelled~) + None => read_zip_bytes(bytes, options=resolved_options, limits~) } } else { match io_context.charset_transcoder { @@ -291,7 +268,6 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, - cancelled~, transcoder=value, ) None => @@ -300,7 +276,6 @@ fn read_workbook_from_bytes( resolved_password, options=resolved_options, limits~, - cancelled~, ) } } @@ -308,7 +283,8 @@ fn read_workbook_from_bytes( ///| /// Reads an XLSX package asynchronously from a reader, stopping before the -/// configured compressed package limit is exceeded. Cancellation is preserved. +/// configured compressed package limit is exceeded. Reader I/O is cancellable; +/// the subsequent resource-bounded semantic parse is currently synchronous. pub async fn[R : @async/io.Reader] open_reader( reader : R, password? : String = "", @@ -321,14 +297,13 @@ pub async fn[R : @async/io.Reader] open_reader( maximum=limits.max_package_bytes, resource="package_bytes", ) + // Give scheduled peers and cancellation one explicit turn even when a custom + // in-memory Reader completed every read immediately. Mid-parse scheduler + // cooperation requires the shared resumable parser tracked in issue #174. + @async.pause() let io_context = read_io_context(transcoder) let workbook = read_workbook_from_bytes( - bytes, - password, - options, - limits, - io_context, - cancelled=() => @async.is_being_cancelled(), + bytes, password, options, limits, io_context, ) workbook.set_io_context(io_context) workbook @@ -376,13 +351,9 @@ pub async fn[R : @async/io.Reader] Workbook::read_zip_reader( maximum=limits.max_package_bytes, resource="package_bytes", ) + @async.pause() let workbook = read_workbook_from_bytes( - bytes, - password, - resolved_options, - limits, - io_context, - cancelled=() => @async.is_being_cancelled(), + bytes, password, resolved_options, limits, io_context, ) workbook.set_io_context(io_context) workbook @@ -410,12 +381,7 @@ pub async fn open_file( Some(path), ) let workbook = read_workbook_from_bytes( - bytes, - password, - options, - limits, - io_context, - cancelled=() => @async.is_being_cancelled(), + bytes, password, options, limits, io_context, ) workbook.set_io_context(io_context) workbook From 9b423619dd1879679e615d254abf1742235ee51f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 18:56:09 +0800 Subject: [PATCH 086/150] chore: upgrade async runtime to 0.20.2 --- .github/workflows/ci.yml | 2 +- docx2html/moon.mod | 2 +- moon.mod | 2 +- office/moon.mod | 2 +- pdflite/moon.mod | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a1e19b..ef1ff5e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,7 +110,7 @@ jobs: # The nightly toolchain is required: mbtexcel depends on # moonbitlang/async >= 0.20 (wasm filesystem support) and its xlsx CLI # targets the wasm backend, both of which need nightly. `moon work sync` - # has aligned the workspace members on async@0.20.1 + x@0.4.43. + # has aligned the workspace members on async@0.20.2 + x@0.4.43. - name: Set up MoonBit (nightly) run: | curl -fsSL https://cli.moonbitlang.com/install/unix.sh | bash -s nightly diff --git a/docx2html/moon.mod b/docx2html/moon.mod index e4211180..f1fcec13 100644 --- a/docx2html/moon.mod +++ b/docx2html/moon.mod @@ -26,7 +26,7 @@ description = "Native MoonBit DOCX to HTML/Markdown converter ported from Mammot import { "bobzhang/mbtexcel@0.1.9", "moonbitlang/x@0.4.43", - "moonbitlang/async@0.20.1", + "moonbitlang/async@0.20.2", } preferred_target = "native" diff --git a/moon.mod b/moon.mod index 06e6a2ee..798b17a2 100644 --- a/moon.mod +++ b/moon.mod @@ -3,7 +3,7 @@ name = "bobzhang/mbtexcel" version = "0.1.9" import { - "moonbitlang/async@0.20.1", + "moonbitlang/async@0.20.2", "moonbitlang/x@0.4.43", } diff --git a/office/moon.mod b/office/moon.mod index 520146b1..1320ba8f 100644 --- a/office/moon.mod +++ b/office/moon.mod @@ -15,7 +15,7 @@ description = "Agent-oriented XLSX and DOCX tooling for MoonBit" import { "bobzhang/mbtexcel@0.1.9", "bobzhang/docx2html@0.2.0", - "moonbitlang/async@0.20.1", + "moonbitlang/async@0.20.2", "moonbitlang/x@0.4.43", "tonyfettes/unicode@0.3.3", } diff --git a/pdflite/moon.mod b/pdflite/moon.mod index 4ab25ab4..676a4283 100644 --- a/pdflite/moon.mod +++ b/pdflite/moon.mod @@ -3,7 +3,7 @@ name = "bobzhang/pdflite" version = "0.1.41" import { - "moonbitlang/async@0.20.1", + "moonbitlang/async@0.20.2", "moonbitlang/x@0.4.43", "bobzhang/mbtexcel@0.1.8", } From 6342b83c62ce7fb0757190d2716b82ab6ee9b26e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:16:54 +0800 Subject: [PATCH 087/150] fix(ooxml): bound MCE relationship state --- ooxml/read_parse.mbt | 266 +++++++++++++++++++++++++++--------- ooxml/read_parse_test.mbt | 23 ++++ ooxml/start_tag_scanner.mbt | 18 ++- 3 files changed, 244 insertions(+), 63 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 46f904d7..fc119fd4 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -103,6 +103,9 @@ let max_relationship_target_chars : Int = 16 * 1024 * 1024 ///| let max_relationship_retained_chars : Int = 16 * 1024 * 1024 +///| +let max_relationship_mce_work_chars : Int = 16 * 1024 * 1024 + ///| fn collapse_schema_whitespace(value : StringView) -> String { let output = StringBuilder::new() @@ -168,6 +171,130 @@ priv enum RelationshipTargetSelection { ExternalTargets } +///| +/// A structural MCE expanded name. Namespace identities are allocated by the +/// streaming XML scanner, so a long namespace URI is retained once rather than +/// copied into every `ProcessContent` key. +priv struct RelationshipMceName { + namespace_id : Int + local_name : String +} derive(Eq, Hash) + +///| +priv struct RelationshipMceScopeDelta { + ignorable_namespace_ids : Array[Int] + process_content_names : Array[RelationshipMceName] +} + +///| +/// Active MCE directives are reference-counted and unwound with XML depth. +/// This keeps scope entry O(number of new directives), without cloning every +/// inherited set at each nested element. +priv struct RelationshipMceContext { + ignorable_namespace_counts : Map[Int, Int] + process_content_name_counts : Map[RelationshipMceName, Int] + mut work_chars : Int +} + +///| +fn RelationshipMceContext::new() -> RelationshipMceContext { + { + ignorable_namespace_counts: Map([]), + process_content_name_counts: Map([]), + work_chars: 0, + } +} + +///| +fn RelationshipMceContext::charge( + self : RelationshipMceContext, + value : StringView, +) -> Unit raise ParseXmlError { + if value.length() > max_relationship_mce_work_chars - self.work_chars { + raise InvalidXml(msg="MCE directive work limit exceeded") + } + self.work_chars = self.work_chars + value.length() +} + +///| +test "relationship MCE directive work is cumulative and bounded" { + let context = RelationshipMceContext::new() + context.work_chars = max_relationship_mce_work_chars - 1 + context.charge("a") + try context.charge("b") catch { + InvalidXml(msg~) => + inspect(msg, content="MCE directive work limit exceeded") + _ => fail("unexpected MCE directive work error") + } noraise { + _ => fail("expected cumulative MCE directive work limit") + } +} + +///| +fn RelationshipMceContext::add_ignorable( + self : RelationshipMceContext, + namespace_id : Int, +) -> Unit { + self.ignorable_namespace_counts[namespace_id] = self.ignorable_namespace_counts.get_or_default( + namespace_id, 0, + ) + + 1 +} + +///| +fn RelationshipMceContext::add_process_content( + self : RelationshipMceContext, + name : RelationshipMceName, +) -> Unit { + self.process_content_name_counts[name] = self.process_content_name_counts.get_or_default( + name, 0, + ) + + 1 +} + +///| +fn RelationshipMceContext::is_ignorable( + self : RelationshipMceContext, + namespace_id : Int, +) -> Bool { + self.ignorable_namespace_counts.get_or_default(namespace_id, 0) > 0 +} + +///| +fn RelationshipMceContext::processes( + self : RelationshipMceContext, + name : RelationshipMceName, +) -> Bool { + self.process_content_name_counts.get_or_default(name, 0) > 0 +} + +///| +fn RelationshipMceContext::leave_scope( + self : RelationshipMceContext, + delta : RelationshipMceScopeDelta?, +) -> Unit { + guard delta is Some(delta) else { return } + for namespace_id in delta.ignorable_namespace_ids { + let remaining = self.ignorable_namespace_counts.get_or_default( + namespace_id, 0, + ) - + 1 + if remaining <= 0 { + ignore(self.ignorable_namespace_counts.remove(namespace_id)) + } else { + self.ignorable_namespace_counts[namespace_id] = remaining + } + } + for name in delta.process_content_names { + let remaining = self.process_content_name_counts.get_or_default(name, 0) - 1 + if remaining <= 0 { + ignore(self.process_content_name_counts.remove(name)) + } else { + self.process_content_name_counts[name] = remaining + } + } +} + ///| /// Streaming state for the MCE-effective projection of an OPC Relationships /// part. Each raw XML depth owns one state; wrapper elements selected by @@ -179,23 +306,13 @@ priv struct RelationshipMceState { included_in_effective_tree : Bool element_effective_depth : Int children_effective_parent_depth : Int - ignorable_namespaces : Set[String] - process_content_names : Set[String] + scope_delta : RelationshipMceScopeDelta? is_alternate_content : Bool mut alternate_choice_seen : Bool mut alternate_fallback_seen : Bool mut alternate_selected : Bool } -///| -fn clone_relationship_mce_set(source : Set[String]) -> Set[String] { - let cloned : Set[String] = Set([]) - for value in source { - cloned.add(value) - } - cloned -} - ///| fn relationship_mce_tokens( value : StringView, @@ -223,11 +340,11 @@ fn relationship_mce_resolve_prefix( scanner : XmlStartTagScanner, prefix : StringView, label : StringView, -) -> String raise ParseXmlError { +) -> (String, Int) raise ParseXmlError { if prefix == "" || prefix.contains(":") { raise InvalidXml(msg="MCE \{label.to_owned()} prefix is invalid") } - scanner.resolve_prefix(prefix, false) + scanner.resolve_prefix_identity(prefix, false) } ///| @@ -235,33 +352,33 @@ fn relationship_mce_expanded_name_key( scanner : XmlStartTagScanner, token : StringView, label : StringView, -) -> (String, String) raise ParseXmlError { +) -> (String, RelationshipMceName) raise ParseXmlError { let parts = token.split(":").collect() if parts.length() != 2 || parts[0] == "" || parts[1] == "" { raise InvalidXml(msg="MCE \{label.to_owned()} name is invalid") } - let namespace_uri = relationship_mce_resolve_prefix(scanner, parts[0], label) - (namespace_uri, namespace_uri + "\u{0}" + parts[1].to_owned()) + let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( + scanner, + parts[0], + label, + ) + (namespace_uri, { namespace_id, local_name: parts[1].to_owned() }) } ///| -fn relationship_mce_scope( +fn relationship_mce_enter_scope( scanner : XmlStartTagScanner, - parent : RelationshipMceState?, -) -> (Set[String], Set[String]) raise ParseXmlError { + context : RelationshipMceContext, +) -> RelationshipMceScopeDelta? raise ParseXmlError { + let mut delta : RelationshipMceScopeDelta? = None let ignorable_attribute = scanner.attribute( markup_compatibility_namespace, "Ignorable", ) - let ignorable = match (parent, ignorable_attribute) { - (Some(value), None) => value.ignorable_namespaces - (Some(value), Some(_)) => - clone_relationship_mce_set(value.ignorable_namespaces) - (None, _) => Set([]) - } match ignorable_attribute { - Some(value) => + Some(value) => { + context.charge(value) for prefix in relationship_mce_tokens(value, "Ignorable") { - let namespace_uri = relationship_mce_resolve_prefix( + let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( scanner, prefix, "Ignorable", ) if namespace_uri == "" || @@ -269,74 +386,97 @@ fn relationship_mce_scope( namespace_uri == package_relationships_namespace { raise InvalidXml(msg="MCE Ignorable namespace is invalid") } - ignorable.add(namespace_uri) + context.add_ignorable(namespace_id) + match delta { + Some(value) => value.ignorable_namespace_ids.push(namespace_id) + None => + delta = Some({ + ignorable_namespace_ids: [namespace_id], + process_content_names: [], + }) + } } + } None => () } let process_content_attribute = scanner.attribute( markup_compatibility_namespace, "ProcessContent", ) - let process_content = match (parent, process_content_attribute) { - (Some(value), None) => value.process_content_names - (Some(value), Some(_)) => - clone_relationship_mce_set(value.process_content_names) - (None, _) => Set([]) - } match process_content_attribute { - Some(value) => + Some(value) => { + context.charge(value) for token in relationship_mce_tokens(value, "ProcessContent") { - let (namespace_uri, key) = relationship_mce_expanded_name_key( + let (_, key) = relationship_mce_expanded_name_key( scanner, token, "ProcessContent", ) - if !ignorable.contains(namespace_uri) { + if !context.is_ignorable(key.namespace_id) { raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") } if token.has_suffix(":*") { raise InvalidXml(msg="MCE ProcessContent wildcard is invalid") } - process_content.add(key) + context.add_process_content(key) + match delta { + Some(value) => value.process_content_names.push(key) + None => + delta = Some({ + ignorable_namespace_ids: [], + process_content_names: [key], + }) + } } + } None => () } match scanner.attribute(markup_compatibility_namespace, "MustUnderstand") { - Some(value) => + Some(value) => { + context.charge(value) for prefix in relationship_mce_tokens(value, "MustUnderstand") { - if relationship_mce_resolve_prefix(scanner, prefix, "MustUnderstand") != - package_relationships_namespace { + let (namespace_uri, _) = relationship_mce_resolve_prefix( + scanner, prefix, "MustUnderstand", + ) + if namespace_uri != package_relationships_namespace { raise InvalidXml(msg="MCE MustUnderstand namespace is unsupported") } } + } None => () } for directive in ["PreserveElements", "PreserveAttributes"] { match scanner.attribute(markup_compatibility_namespace, directive) { - Some(value) => + Some(value) => { + context.charge(value) for token in relationship_mce_tokens(value, directive) { - let (namespace_uri, _) = relationship_mce_expanded_name_key( + let (_, name) = relationship_mce_expanded_name_key( scanner, token, directive, ) - if !ignorable.contains(namespace_uri) { + if !context.is_ignorable(name.namespace_id) { raise InvalidXml(msg="MCE \{directive} namespace is not ignorable") } } + } None => () } } - (ignorable, process_content) + delta } ///| fn relationship_mce_choice_supported( scanner : XmlStartTagScanner, + context : RelationshipMceContext, ) -> Bool raise ParseXmlError { let requires = match scanner.attribute("", "Requires") { Some(value) => value None => raise InvalidXml(msg="MCE Choice is missing Requires") } + context.charge(requires) let mut supported = true for prefix in relationship_mce_tokens(requires, "Choice Requires") { - if relationship_mce_resolve_prefix(scanner, prefix, "Choice Requires") != - package_relationships_namespace { + let (namespace_uri, _) = relationship_mce_resolve_prefix( + scanner, prefix, "Choice Requires", + ) + if namespace_uri != package_relationships_namespace { supported = false } } @@ -347,9 +487,11 @@ fn relationship_mce_choice_supported( fn trim_relationship_mce_states( states : Array[RelationshipMceState], next_depth : Int, + context : RelationshipMceContext, ) -> Unit raise ParseXmlError { while states.length() >= next_depth { let state = states.pop().unwrap() + context.leave_scope(state.scope_delta) if state.is_alternate_content && !state.alternate_choice_seen { raise InvalidXml(msg="MCE AlternateContent has no Choice element") } @@ -360,9 +502,10 @@ fn trim_relationship_mce_states( fn relationship_mce_state_for_current_element( scanner : XmlStartTagScanner, states : Array[RelationshipMceState], + context : RelationshipMceContext, ) -> RelationshipMceState raise ParseXmlError { let depth = scanner.depth() - trim_relationship_mce_states(states, depth) + trim_relationship_mce_states(states, depth, context) let parent = if depth > 1 { if states.length() != depth - 1 { raise InvalidXml(msg="relationships document nesting is invalid") @@ -378,8 +521,7 @@ fn relationship_mce_state_for_current_element( included_in_effective_tree: false, element_effective_depth: parent_state.children_effective_parent_depth, children_effective_parent_depth: parent_state.children_effective_parent_depth, - ignorable_namespaces: parent_state.ignorable_namespaces, - process_content_names: parent_state.process_content_names, + scope_delta: None, is_alternate_content: false, alternate_choice_seen: false, alternate_fallback_seen: false, @@ -387,7 +529,7 @@ fn relationship_mce_state_for_current_element( } _ => () } - let (ignorable, process_content) = relationship_mce_scope(scanner, parent) + let scope_delta = relationship_mce_enter_scope(scanner, context) if depth == 1 { if scanner.namespace_uri() != package_relationships_namespace || scanner.local_name() != "Relationships" { @@ -398,8 +540,7 @@ fn relationship_mce_state_for_current_element( included_in_effective_tree: true, element_effective_depth: 1, children_effective_parent_depth: 1, - ignorable_namespaces: ignorable, - process_content_names: process_content, + scope_delta, is_alternate_content: false, alternate_choice_seen: false, alternate_fallback_seen: false, @@ -408,6 +549,7 @@ fn relationship_mce_state_for_current_element( } let parent_state = parent.unwrap() let namespace_uri = scanner.namespace_uri().to_owned() + let namespace_id = scanner.current_namespace_identity() let local_name = scanner.local_name().to_owned() let parent_effective_depth = parent_state.children_effective_parent_depth let mut active = parent_state.active @@ -422,7 +564,7 @@ fn relationship_mce_state_for_current_element( raise InvalidXml(msg="MCE Choice follows Fallback") } parent_state.alternate_choice_seen = true - let supported = relationship_mce_choice_supported(scanner) + let supported = relationship_mce_choice_supported(scanner, context) active = parent_state.active && !parent_state.alternate_selected && supported @@ -441,7 +583,7 @@ fn relationship_mce_state_for_current_element( if active { parent_state.alternate_selected = true } - } else if ignorable.contains(namespace_uri) { + } else if context.is_ignorable(namespace_id) { active = false } else { raise InvalidXml(msg="MCE AlternateContent child is invalid") @@ -452,10 +594,10 @@ fn relationship_mce_state_for_current_element( } active = parent_state.active is_alternate_content = true - } else if ignorable.contains(namespace_uri) && + } else if context.is_ignorable(namespace_id) && namespace_uri != package_relationships_namespace { - let key = namespace_uri + "\u{0}" + local_name - if process_content.contains(key) { + let key : RelationshipMceName = { namespace_id, local_name } + if context.processes(key) { if scanner.attribute(xml_namespace_uri, "base") is Some(_) || scanner.attribute(xml_namespace_uri, "lang") is Some(_) || scanner.attribute(xml_namespace_uri, "space") is Some(_) { @@ -478,8 +620,7 @@ fn relationship_mce_state_for_current_element( included_in_effective_tree, element_effective_depth, children_effective_parent_depth, - ignorable_namespaces: ignorable, - process_content_names: process_content, + scope_delta, is_alternate_content, alternate_choice_seen: false, alternate_fallback_seen: false, @@ -501,11 +642,12 @@ fn parse_relationship_targets_by_types_selected( let ids : Set[String] = Set([]) let scanner = XmlStartTagScanner::new(xml, cancelled~) let mce_states : Array[RelationshipMceState] = [] + let mce_context = RelationshipMceContext::new() let mut records = 0 let mut retained_chars = 0 while scanner.next() { let mce_state = relationship_mce_state_for_current_element( - scanner, mce_states, + scanner, mce_states, mce_context, ) mce_states.push(mce_state) if mce_state.element_effective_depth == 1 { @@ -565,7 +707,7 @@ fn parse_relationship_targets_by_types_selected( } targets[id] = target } - trim_relationship_mce_states(mce_states, 1) + trim_relationship_mce_states(mce_states, 1, mce_context) targets } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 7740ffd6..949e398e 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -131,6 +131,29 @@ test "ooxml read_parse: relationships apply OPC Markup Compatibility" { assert_eq(fallback.length(), 1) } +///| +test "ooxml read_parse: MCE names retain one copy of a long namespace URI" { + let names = StringBuilder::new() + for index in 0..<4096 { + if index > 0 { + names.write_char(' ') + } + names.write_string("v:n\{index}") + } + let namespace_uri = "urn:" + "v".repeat(100_000) + let xml = "" + let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + assert_eq(targets.length(), 0) +} + ///| test "ooxml read_parse: invalid relationship MCE fails closed" { let cases : Array[(String, String)] = [ diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 951ce27e..3341ac26 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -75,6 +75,7 @@ pub struct XmlStartTagScanner { priv mut current_tag_end : Int priv mut current_depth : Int priv mut current_namespace_uri : String + priv mut current_namespace_id : Int priv capture_element_namespaces : Array[String] priv capture_element_local_name : String priv retain_element_namespaces : Array[String] @@ -1226,6 +1227,7 @@ fn xml_start_tag_scanner_with_rewrites( current_tag_end: 0, current_depth: 0, current_namespace_uri: "", + current_namespace_id: 0, capture_element_namespaces, capture_element_local_name, retain_element_namespaces, @@ -1498,7 +1500,11 @@ pub fn XmlStartTagScanner::next( Some(value) => self.xml[name_start:value] None => self.xml[name_start:name_start] } - self.current_namespace_uri = self.resolve_prefix(prefix, false) + let (current_namespace_uri, current_namespace_id) = self.resolve_prefix_identity( + prefix, false, + ) + self.current_namespace_uri = current_namespace_uri + self.current_namespace_id = current_namespace_id let local_name_start = match colon { Some(value) => value + 1 None => name_start @@ -1598,6 +1604,16 @@ pub fn XmlStartTagScanner::namespace_uri( self.current_namespace_uri } +///| +/// Returns the scanner-local structural identity of the current element's +/// namespace. The identity is stable for the lifetime of the in-scope +/// namespace binding and avoids copying long namespace URIs into parser keys. +fn XmlStartTagScanner::current_namespace_identity( + self : XmlStartTagScanner, +) -> Int { + self.current_namespace_id +} + ///| /// Returns the one-based nesting depth of the current start tag. pub fn XmlStartTagScanner::depth(self : XmlStartTagScanner) -> Int { From 80abe03c8395f6b520698e363c2aaa9b9a6a31bd Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:18:22 +0800 Subject: [PATCH 088/150] fix(ooxml): honor MCE directive semantics --- ooxml/read_parse.mbt | 48 ++++++++++++++++++++++++++++++--------- ooxml/read_parse_test.mbt | 25 ++++++++++++++++++-- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index fc119fd4..f342c537 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -260,12 +260,35 @@ fn RelationshipMceContext::is_ignorable( self.ignorable_namespace_counts.get_or_default(namespace_id, 0) > 0 } +///| +fn relationship_mce_namespace_is_understood(namespace_uri : StringView) -> Bool { + namespace_uri == package_relationships_namespace || + namespace_uri == markup_compatibility_namespace +} + +///| +fn RelationshipMceContext::is_effectively_ignorable( + self : RelationshipMceContext, + namespace_id : Int, + namespace_uri : StringView, +) -> Bool { + self.is_ignorable(namespace_id) && + !relationship_mce_namespace_is_understood(namespace_uri) +} + ///| fn RelationshipMceContext::processes( self : RelationshipMceContext, name : RelationshipMceName, ) -> Bool { - self.process_content_name_counts.get_or_default(name, 0) > 0 + if self.process_content_name_counts.get_or_default(name, 0) > 0 { + return true + } + self.process_content_name_counts.get_or_default( + { namespace_id: name.namespace_id, local_name: "*" }, + 0, + ) > + 0 } ///| @@ -317,9 +340,13 @@ priv struct RelationshipMceState { fn relationship_mce_tokens( value : StringView, label : StringView, + allow_empty? : Bool = true, ) -> Array[String] raise ParseXmlError { let normalized = collapse_schema_whitespace(value) if normalized == "" { + if allow_empty { + return [] + } raise InvalidXml(msg="MCE \{label.to_owned()} is empty") } let tokens : Array[String] = [] @@ -381,9 +408,7 @@ fn relationship_mce_enter_scope( let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( scanner, prefix, "Ignorable", ) - if namespace_uri == "" || - namespace_uri == markup_compatibility_namespace || - namespace_uri == package_relationships_namespace { + if namespace_uri == "" { raise InvalidXml(msg="MCE Ignorable namespace is invalid") } context.add_ignorable(namespace_id) @@ -412,9 +437,6 @@ fn relationship_mce_enter_scope( if !context.is_ignorable(key.namespace_id) { raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") } - if token.has_suffix(":*") { - raise InvalidXml(msg="MCE ProcessContent wildcard is invalid") - } context.add_process_content(key) match delta { Some(value) => value.process_content_names.push(key) @@ -472,7 +494,12 @@ fn relationship_mce_choice_supported( } context.charge(requires) let mut supported = true - for prefix in relationship_mce_tokens(requires, "Choice Requires") { + for + prefix in relationship_mce_tokens( + requires, + "Choice Requires", + allow_empty=false, + ) { let (namespace_uri, _) = relationship_mce_resolve_prefix( scanner, prefix, "Choice Requires", ) @@ -583,7 +610,7 @@ fn relationship_mce_state_for_current_element( if active { parent_state.alternate_selected = true } - } else if context.is_ignorable(namespace_id) { + } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { active = false } else { raise InvalidXml(msg="MCE AlternateContent child is invalid") @@ -594,8 +621,7 @@ fn relationship_mce_state_for_current_element( } active = parent_state.active is_alternate_content = true - } else if context.is_ignorable(namespace_id) && - namespace_uri != package_relationships_namespace { + } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { let key : RelationshipMceName = { namespace_id, local_name } if context.processes(key) { if scanner.attribute(xml_namespace_uri, "base") is Some(_) || diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 949e398e..dc509dfa 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -114,6 +114,12 @@ test "ooxml read_parse: relationships apply OPC Markup Compatibility" { #| let selected_fallback = #| + let wildcard_process_content = + #| + let empty_directives = + #| + let understood_ignorable = + #| let processed = @ooxml.parse_internal_relationship_targets( process_content, "wanted", ) @@ -129,6 +135,21 @@ test "ooxml read_parse: relationships apply OPC Markup Compatibility" { ) assert_eq(fallback.get("fallback"), Some("fallback.xml")) assert_eq(fallback.length(), 1) + let wildcard = @ooxml.parse_internal_relationship_targets( + wildcard_process_content, "wanted", + ) + assert_eq(wildcard.get("wildcard"), Some("wildcard.xml")) + assert_eq(wildcard.length(), 1) + let empty = @ooxml.parse_internal_relationship_targets( + empty_directives, "wanted", + ) + assert_eq(empty.get("empty"), Some("empty.xml")) + assert_eq(empty.length(), 1) + let understood = @ooxml.parse_internal_relationship_targets( + understood_ignorable, "wanted", + ) + assert_eq(understood.get("understood"), Some("understood.xml")) + assert_eq(understood.length(), 1) } ///| @@ -183,9 +204,9 @@ test "ooxml read_parse: invalid relationship MCE fails closed" { ), ( ( - #| + #| ), - "MCE ProcessContent wildcard is invalid", + "MCE Choice Requires is empty", ), ( ( From 82cc5b453c11444346d3e777386b8d0ff7ab1ba8 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:33:07 +0800 Subject: [PATCH 089/150] fix(xlsx): bound Excel date conversion --- xlsx/cell_time.mbt | 31 ++++++++++ xlsx/cell_value_test.mbt | 19 ++++++ xlsx/date_convert.mbt | 99 ++++++++++++++++++-------------- xlsx/date_convert_test.mbt | 41 +++++++++++-- xlsx/formula_builtins.mbt | 46 +++++++++------ xlsx/formula_builtins_wbtest.mbt | 36 ++++++------ xlsx/formula_eval.mbt | 30 +++++----- xlsx/value_format.mbt | 57 ------------------ 8 files changed, 205 insertions(+), 154 deletions(-) diff --git a/xlsx/cell_time.mbt b/xlsx/cell_time.mbt index e69537e8..7e27edf8 100644 --- a/xlsx/cell_time.mbt +++ b/xlsx/cell_time.mbt @@ -11,6 +11,37 @@ fn days_from_civil(year : Int, month : Int, day : Int) -> Int { era * 146097 + doe - 719468 } +///| +/// Converts days since 1970-01-01 to a proleptic-Gregorian calendar date in +/// constant time (the inverse of `days_from_civil`). +fn civil_from_days(days : Int) -> (Int, Int, Int) { + let shifted = days + 719468 + let era = (if shifted >= 0 { shifted } else { shifted - 146096 }) / 146097 + let day_of_era = shifted - era * 146097 + let year_of_era = ( + day_of_era - day_of_era / 1460 + day_of_era / 36524 - day_of_era / 146096 + ) / + 365 + let mut year = year_of_era + era * 400 + let day_of_year = day_of_era - + (365 * year_of_era + year_of_era / 4 - year_of_era / 100) + let month_prime = (5 * day_of_year + 2) / 153 + let day = day_of_year - (153 * month_prime + 2) / 5 + 1 + let month = month_prime + (if month_prime < 10 { 3 } else { -9 }) + if month <= 2 { + year = year + 1 + } + (year, month, day) +} + +///| +test "civil day conversion round-trips supported Excel boundaries" { + for date in [(1899, 12, 31), (1904, 1, 1), (1970, 1, 1), (9999, 12, 31)] { + let (year, month, day) = date + assert_eq(civil_from_days(days_from_civil(year, month, day)), date) + } +} + ///| let nanos_per_day : Double = 86_400_000_000_000.0 diff --git a/xlsx/cell_value_test.mbt b/xlsx/cell_value_test.mbt index 8839fe6f..560e5b0f 100644 --- a/xlsx/cell_value_test.mbt +++ b/xlsx/cell_value_test.mbt @@ -82,6 +82,25 @@ test "formatted percent and date" { ) } +///| +test "built-in date formatting is constant-time at numeric boundaries" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + let date_style = workbook.add_style(@xlsx.Style::builtin_number_format(14)) + let cases : Array[(String, Double, String)] = [ + ("A1", 2_958_465.0, "12-31-99"), + ("A2", 2_958_466.0, "2958466"), + ("A3", 2_000_000_000.0, "2000000000"), + ("A4", 2_147_483_647.0, "2147483647"), + ] + for case in cases { + let (cell, value, expected) = case + workbook.set_cell_value("Sheet1", cell, Numeric(value)) + workbook.set_cell_style("Sheet1", cell, date_style) + assert_eq(workbook.get_cell_value("Sheet1", cell), Some(expected)) + } +} + ///| test "formatted dates honor the workbook 1904 date system" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/date_convert.mbt b/xlsx/date_convert.mbt index 269a3d21..4e8ab4af 100644 --- a/xlsx/date_convert.mbt +++ b/xlsx/date_convert.mbt @@ -1,55 +1,70 @@ ///| -fn add_days_to_date( - year : Int, - month : Int, - day : Int, - days : Int, -) -> (Int, Int, Int) { - let mut y = year - let mut m = month - let mut d = day - let mut remaining = days - while remaining > 0 { - let dim = days_in_month(y, m) - if d + remaining <= dim { - d = d + remaining - remaining = 0 - } else { - remaining = remaining - (dim - d + 1) - d = 1 - m = m + 1 - if m > 12 { - m = 1 - y = y + 1 - } - } - } - (y, m, d) -} +let maximum_excel_serial_1900 = 2_958_465 ///| -fn excel_serial_to_parts_1904( +let maximum_excel_serial_1904 = 2_957_003 + +///| +/// Splits a finite, supported Excel serial after rounding its clock to whole +/// seconds. The returned day is also rounded when 23:59:59.5 spills into the +/// next date, and can therefore be checked before calendar conversion. +fn excel_serial_day_and_clock( value : Double, -) -> (Int, Int, Int, Int, Int, Int)? { - if value < 0.0 { + maximum_day : Int, +) -> (Int, Int, Int, Int)? { + if value.is_nan() || + value.is_inf() || + value < 0.0 || + value >= Double::from_int(maximum_day + 1) { return None } - let whole = Double::to_int(Double::floor(value)) - let days = whole - let mut fraction = value - Double::from_int(whole) - if fraction < 0.0 { - fraction = 0.0 - } + let mut serial_day = Double::to_int(Double::floor(value)) + let fraction = value - Double::from_int(serial_day) let mut total_seconds = Double::to_int(Double::round(fraction * 86400.0)) - let mut add_day = 0 if total_seconds >= 86400 { total_seconds = total_seconds - 86400 - add_day = 1 + serial_day = serial_day + 1 + } + if serial_day > maximum_day { + return None + } + Some( + ( + serial_day, + total_seconds / 3600, + total_seconds % 3600 / 60, + total_seconds % 60, + ), + ) +} + +///| +fn excel_serial_to_parts(value : Double) -> (Int, Int, Int, Int, Int, Int)? { + guard excel_serial_day_and_clock(value, maximum_excel_serial_1900) + is Some((serial_day, hour, minute, second)) else { + return None + } + if serial_day == 60 { + return Some((1900, 2, 29, hour, minute, second)) + } + let civil_day = if serial_day > 59 { serial_day - 1 } else { serial_day } + let (year, month, day) = civil_from_days( + days_from_civil(1899, 12, 31) + civil_day, + ) + Some((year, month, day, hour, minute, second)) +} + +///| +fn excel_serial_to_parts_1904( + value : Double, +) -> (Int, Int, Int, Int, Int, Int)? { + guard excel_serial_day_and_clock(value, maximum_excel_serial_1904) + is Some((serial_day, hour, minute, second)) else { + return None } - let (year, month, day) = add_days_to_date(1904, 1, 1, days + add_day) - let hour = total_seconds / 3600 - let minute = total_seconds % 3600 / 60 - let second = total_seconds % 60 + let (year, month, day) = civil_from_days( + days_from_civil(1904, 1, 1) + serial_day, + ) Some((year, month, day, hour, minute, second)) } diff --git a/xlsx/date_convert_test.mbt b/xlsx/date_convert_test.mbt index e22718e4..d6ee6c70 100644 --- a/xlsx/date_convert_test.mbt +++ b/xlsx/date_convert_test.mbt @@ -49,12 +49,45 @@ test "excel date to time 1904" { ///| test "excel date to time invalid" { - let result : Result[@time.ZonedDateTime, Error] = Ok( - @xlsx.excel_date_to_time(-1.0), + for + value in [ + -1.0, + 2_958_465.9999999, + 2_958_466.0, + 2_000_000_000.0, + 2_147_483_647.0, + 1.0 / 0.0, + 0.0 / 0.0, + ] { + let result : Result[@time.ZonedDateTime, Error] = Ok( + @xlsx.excel_date_to_time(value), + ) catch { + e => Err(e) + } + inspect(result is Err(_), content="true") + } + let latest = @xlsx.excel_date_to_time(2_958_465.0) + debug_inspect( + (latest.year(), latest.month(), latest.day()), + content="(9999, 12, 31)", + ) + let latest_1904 = @xlsx.excel_date_to_time(2_957_003.0, use_1904_format=true) + debug_inspect( + (latest_1904.year(), latest_1904.month(), latest_1904.day()), + content="(9999, 12, 31)", + ) + let after_1904 : Result[@time.ZonedDateTime, Error] = Ok( + @xlsx.excel_date_to_time(2_957_004.0, use_1904_format=true), ) catch { - e => Err(e) + error => Err(error) } - inspect(result is Err(_), content="true") + inspect(after_1904 is Err(_), content="true") + let spill_1904 : Result[@time.ZonedDateTime, Error] = Ok( + @xlsx.excel_date_to_time(2_957_003.9999999, use_1904_format=true), + ) catch { + error => Err(error) + } + inspect(spill_1904 is Err(_), content="true") } ///| diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index b4fe4dcc..b84ac734 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -3693,15 +3693,22 @@ fn eval_function( Ok(text) => text.to_lower() Err(err) => return err } - // value_as_date_serial rejects negatives; serial-to-date conversion is total on non-negative serials. - let (sy, sm, sd) = date_parts_from_serial( - start_serial, - use_1904_dates=ctx.use_1904_dates, - ).unwrap() - let (ey, em, ed) = date_parts_from_serial( - end_serial, - use_1904_dates=ctx.use_1904_dates, - ).unwrap() + let (sy, sm, sd) = match + date_parts_from_serial( + start_serial, + use_1904_dates=ctx.use_1904_dates, + ) { + Some(parts) => parts + None => return Error(formula_error_num) + } + let (ey, em, ed) = match + date_parts_from_serial( + end_serial, + use_1904_dates=ctx.use_1904_dates, + ) { + Some(parts) => parts + None => return Error(formula_error_num) + } match unit { "y" => { let mut diff = ey - sy @@ -3791,14 +3798,19 @@ fn eval_function( Ok(value) => value Err(err) => return err } - let (sy, sm, sd) = date_parts_from_serial( - start_serial, - use_1904_dates=ctx.use_1904_dates, - ).unwrap() - let (ey, em, ed) = date_parts_from_serial( - end_serial, - use_1904_dates=ctx.use_1904_dates, - ).unwrap() + let (sy, sm, sd) = match + date_parts_from_serial( + start_serial, + use_1904_dates=ctx.use_1904_dates, + ) { + Some(parts) => parts + None => return Error(formula_error_num) + } + let (ey, em, ed) = match + date_parts_from_serial(end_serial, use_1904_dates=ctx.use_1904_dates) { + Some(parts) => parts + None => return Error(formula_error_num) + } let mut start_day = sd let mut end_day = ed let end_year = ey diff --git a/xlsx/formula_builtins_wbtest.mbt b/xlsx/formula_builtins_wbtest.mbt index 60187fc9..43a8dd7a 100644 --- a/xlsx/formula_builtins_wbtest.mbt +++ b/xlsx/formula_builtins_wbtest.mbt @@ -2404,31 +2404,29 @@ test "formula builtins wb: date-time and lookup guard branches part 7" { content="true", ) inspect( - match + is_formula_error( eval_function( workbook, "Sheet1", "DATEDIF", [Number(1.0e20), Number(1.0e20 + 1.0), String("D")], ctx, - ) { - Number(_) => true - _ => false - }, + ), + formula_error_num, + ), content="true", ) inspect( - match + is_formula_error( eval_function( workbook, "Sheet1", "DATEDIF", [Number(1.0), Number(1.0e20), String("D")], ctx, - ) { - Number(_) => true - _ => false - }, + ), + formula_error_num, + ), content="true", ) inspect( @@ -2578,31 +2576,29 @@ test "formula builtins wb: date-time and lookup guard branches part 7" { content="true", ) inspect( - match + is_formula_error( eval_function( workbook, "Sheet1", "DAYS360", [Number(1.0e20), String("2024-01-31"), Bool(true)], ctx, - ) { - Number(_) => true - _ => false - }, + ), + formula_error_num, + ), content="true", ) inspect( - match + is_formula_error( eval_function( workbook, "Sheet1", "DAYS360", [String("2024-01-31"), Number(1.0e20), Bool(true)], ctx, - ) { - Number(_) => true - _ => false - }, + ), + formula_error_num, + ), content="true", ) inspect( diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index d2ff33a6..b8ec9b61 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -5958,20 +5958,27 @@ fn date_parts_from_serial( } } +///| +fn validated_date_serial( + serial : Double, + use_1904_dates : Bool, +) -> Result[Double, FormulaValue] { + match date_parts_from_serial(serial, use_1904_dates~) { + Some(_) => Ok(serial) + None => Err(Error(formula_error_num)) + } +} + ///| fn value_as_date_serial( value : FormulaValue, use_1904_dates? : Bool = false, ) -> Result[Double, FormulaValue] { match normalize_scalar(value) { - Number(num) => - if num < 0.0 { - Err(Error(formula_error_num)) - } else { - Ok(num) - } - Bool(flag) => Ok(if flag { 1.0 } else { 0.0 }) - Empty => Ok(0.0) + Number(num) => validated_date_serial(num, use_1904_dates) + Bool(flag) => + validated_date_serial(if flag { 1.0 } else { 0.0 }, use_1904_dates) + Empty => validated_date_serial(0.0, use_1904_dates) String(text) => { let trimmed = text.trim().to_owned() match parse_date_parts(trimmed) { @@ -5982,12 +5989,7 @@ fn value_as_date_serial( } None => match parse_value_number(trimmed) { - Some(num) => - if num < 0.0 { - Err(Error(formula_error_num)) - } else { - Ok(num) - } + Some(num) => validated_date_serial(num, use_1904_dates) None => Err(Error(formula_error_value)) } } diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index daa74c31..1f090f2d 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -1175,63 +1175,6 @@ fn format_excel_date( } } -///| -fn excel_serial_to_parts(value : Double) -> (Int, Int, Int, Int, Int, Int)? { - if value < 0.0 { - return None - } - let whole = Double::to_int(Double::floor(value)) - let mut fraction = value - Double::from_int(whole) - if fraction < 0.0 { - fraction = 0.0 - } - let mut total_seconds = Double::to_int(Double::round(fraction * 86400.0)) - let mut add_day = 0 - if total_seconds >= 86400 { - total_seconds = total_seconds - 86400 - add_day = 1 - } - if whole == 60 { - let hour = total_seconds / 3600 - let minute = total_seconds % 3600 / 60 - let second = total_seconds % 60 - return Some((1900, 2, 29, hour, minute, second)) - } - let mut days = whole - if days > 59 { - days = days - 1 - } - let (year, month, day) = add_days_to_base(days + add_day) - let hour = total_seconds / 3600 - let minute = total_seconds % 3600 / 60 - let second = total_seconds % 60 - Some((year, month, day, hour, minute, second)) -} - -///| -fn add_days_to_base(days : Int) -> (Int, Int, Int) { - let mut year = 1899 - let mut month = 12 - let mut day = 31 - let mut remaining = days - while remaining > 0 { - let dim = days_in_month(year, month) - if day + remaining <= dim { - day = day + remaining - remaining = 0 - } else { - remaining = remaining - (dim - day + 1) - day = 1 - month = month + 1 - if month > 12 { - month = 1 - year = year + 1 - } - } - } - (year, month, day) -} - ///| fn is_leap_year(year : Int) -> Bool { if year % 400 == 0 { From 0b5f4fcbcd514777c52919f5317a139f20ca1775 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:35:15 +0800 Subject: [PATCH 090/150] fix(xlsx): resolve relationship targets strictly --- xlsx/read.mbt | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index c1848695..8806d58c 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3383,21 +3383,29 @@ fn actual_relationship_target_path( logical_archive_part_path(source_part), target, ) - match part_names.get(@ooxml.package_part_name_key(source_relative)) { - Some(actual) => actual - None => { - // Some producers emit package-root paths without the leading slash. - // Standards-compliant source-relative lookup wins when both exist; this - // fallback retains tolerant reads only when the root spelling names an - // actual archive part. - let package_root = try normalize_rel_part_path(target) catch { - _ => None - } noraise { - path => part_names.get(@ooxml.package_part_name_key(path)) - } - package_root.unwrap_or(source_relative) - } - } + part_names + .get(@ooxml.package_part_name_key(source_relative)) + .unwrap_or(source_relative) +} + +///| +test "relationship targets never fall back from source-relative to package root" { + let root_media = "xl/media/image1.png" + let part_names : Map[String, String] = Map([ + (@ooxml.package_part_name_key(root_media), root_media), + ]) + inspect( + actual_relationship_target_path( + "xl/drawings/drawing1.xml", root_media, part_names, + ), + content="xl/drawings/xl/media/image1.png", + ) + inspect( + actual_relationship_target_path( + "xl/drawings/drawing1.xml", "../media/image1.png", part_names, + ), + content="xl/media/image1.png", + ) } ///| From 0d83988af6d3c0478150aaf578038d6a8de3e2e4 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:36:30 +0800 Subject: [PATCH 091/150] fix(ooxml): reject uppercase hex references --- ooxml/read_parse_test.mbt | 20 ++++++++++++++++++++ ooxml/start_tag_scanner.mbt | 4 ++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index dc509dfa..f244e808 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -74,6 +74,26 @@ test "ooxml read_parse: prefixed relationships decode entities exactly once" { ) } +///| +test "ooxml read_parse: uppercase-X character references never change relationship identity" { + for + xml in [ + ( + #| + ), + ( + #| + ), + ] { + try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + InvalidXml(msg~) => inspect(msg, content="XML entity is invalid") + _ => fail("unexpected uppercase-X entity error") + } noraise { + _ => fail("expected uppercase-X entity rejection") + } + } +} + ///| test "ooxml read_parse: relationship schema whitespace is collapsed" { let xml = diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 3341ac26..a0722f21 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -654,7 +654,7 @@ fn xml_entity_value(entity : StringView) -> Char? { _ => { let (digits, base) = if entity.length() >= 2 && entity[0] == ('#' : UInt16) && - (entity[1] == ('x' : UInt16) || entity[1] == ('X' : UInt16)) { + entity[1] == ('x' : UInt16) { (entity[2:], 16) } else if entity.length() >= 1 && entity[0] == ('#' : UInt16) { (entity[1:], 10) @@ -1989,7 +1989,7 @@ test "start tag scanner restores namespace scopes after self-closing tags" { test "XML attributes decode once and reject invalid entities" { inspect(decode_xml_attribute("a&#x20;b"), content="a b") inspect(decode_xml_attribute("a\r\nb"), content="a b") - for value in ["&unknown;", "�", "�", "&"] { + for value in ["&unknown;", "�", "�", "1", "&"] { try decode_xml_attribute(value) catch { InvalidXml(_) => () _ => fail("unexpected attribute cancellation") From d4fb63667e681f677694c135c14ff26de7a85114 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:39:41 +0800 Subject: [PATCH 092/150] fix(xlsx): calculate ISO weeks across date epochs --- xlsx/calc_1904_date_test.mbt | 4 ++ xlsx/formula_builtins.mbt | 3 +- xlsx/formula_eval.mbt | 95 +++++++++++++----------------------- 3 files changed, 38 insertions(+), 64 deletions(-) diff --git a/xlsx/calc_1904_date_test.mbt b/xlsx/calc_1904_date_test.mbt index 9167daa0..a17db6d1 100644 --- a/xlsx/calc_1904_date_test.mbt +++ b/xlsx/calc_1904_date_test.mbt @@ -25,6 +25,10 @@ test "calculated dates honor the workbook 1904 date system" { ("DAYS(DATE(1904,1,3),DATE(1904,1,1))", "2"), ("NETWORKDAYS(DATE(1904,1,1),DATE(1904,1,4))", "2"), ("TEXT(WORKDAY(DATE(1904,1,1),1),\"yyyy-mm-dd\")", "1904-01-04"), + ("ISOWEEKNUM(DATE(1904,1,1))", "53"), + ("ISOWEEKNUM(DATE(1904,1,2))", "53"), + ("ISOWEEKNUM(DATE(1904,1,3))", "53"), + ("ISOWEEKNUM(DATE(1904,1,4))", "1"), ("INT(NOW())=TODAY()", "TRUE"), ("TODAY()=DATE(YEAR(TODAY()),MONTH(TODAY()),DAY(TODAY()))", "TRUE"), ("DATE(1903,12,31)", "#NUM!"), diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index b84ac734..389a26d3 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -3858,8 +3858,7 @@ fn eval_function( Ok(parts) => parts Err(err) => return err } - match - iso_week_number(year, month, day, use_1904_dates=ctx.use_1904_dates) { + match iso_week_number(year, month, day) { Some(week) => Number(Double::from_int(week)) None => Error(formula_error_num) } diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index b8ec9b61..ace30365 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -5849,35 +5849,6 @@ fn normalize_date_parts(year : Int, month : Int, day : Int) -> (Int, Int, Int) { (y, m, d) } -///| -fn days_before_year(year : Int) -> Int { - let mut days = 0 - let mut y = 1900 - if year >= 1900 { - while y < year { - let inc = if is_leap_year(y) { 366 } else { 365 } - days = days + inc - y = y + 1 - } - } else { - while y > year { - y = y - 1 - let dec = if is_leap_year(y) { 366 } else { 365 } - days = days - dec - } - } - days -} - -///| -fn days_before_month(year : Int, month : Int) -> Int { - let mut days = 0 - for m in 1..= 60 { + let civil_day = days_from_civil(y, m, d) + let mut value = civil_day - days_from_civil(1899, 12, 31) + if civil_day >= days_from_civil(1900, 3, 1) { value = value + 1 } value @@ -6025,15 +5994,13 @@ fn days_in_year(year : Int) -> Int { ///| fn day_of_year(year : Int, month : Int, day : Int) -> Int { - days_before_month(year, month) + day + days_from_civil(year, month, day) - days_from_civil(year, 1, 1) + 1 } ///| fn weekday_monday1(year : Int, month : Int, day : Int) -> Int { - let mut days = days_before_year(year) - days = days + days_before_month(year, month) - days = days + (day - 1) - let mut idx = days % 7 + // 1970-01-01 was Thursday, or weekday 4 when Monday is 1. + let mut idx = (days_from_civil(year, month, day) + 3) % 7 if idx < 0 { idx = idx + 7 } @@ -6051,32 +6018,36 @@ fn weekday_sun1(year : Int, month : Int, day : Int) -> Int { } ///| -fn iso_week_number( - year : Int, - month : Int, - day : Int, - use_1904_dates? : Bool = false, -) -> Int? { - let serial = match excel_serial_from_date(year, month, day, use_1904_dates~) { - Some(value) => value - None => return None +fn iso_weeks_in_year(year : Int) -> Int { + let january_first = weekday_monday1(year, 1, 1) + if january_first == 4 || (january_first == 3 && is_leap_year(year)) { + 53 + } else { + 52 } - let weekday = weekday_monday1(year, month, day) - let thursday_serial = serial + Double::from_int(4 - weekday) - let (iso_year, _, _) = match - date_parts_from_serial(thursday_serial, use_1904_dates~) { - Some(parts) => parts - None => return None +} + +///| +fn iso_week_number(year : Int, month : Int, day : Int) -> Int? { + if year < 1 || + year > 9999 || + month < 1 || + month > 12 || + day < 1 || + day > days_in_month(year, month) { + return None } - let week1_serial = match - excel_serial_from_date(iso_year, 1, 4, use_1904_dates~) { - Some(value) => value - None => return None + let weekday = weekday_monday1(year, month, day) + let mut week = (day_of_year(year, month, day) - weekday + 10) / 7 + if week < 1 { + if year == 1 { + return None + } + week = iso_weeks_in_year(year - 1) + } else if week > iso_weeks_in_year(year) { + week = 1 } - let week1_weekday = weekday_monday1(iso_year, 1, 4) - let week1_thursday = week1_serial + Double::from_int(4 - week1_weekday) - let offset = (thursday_serial - week1_thursday) / 7.0 - Some(Double::to_int(Double::floor(offset)) + 1) + Some(week) } ///| From 90ee0c9232534eb6933359bab7eeb8988772ccd4 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:53:20 +0800 Subject: [PATCH 093/150] test(xlsx): use strict pivot relationship targets --- xlsx/ooxml_rels_error_test.mbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index d06bcbd5..20b8aa64 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -329,7 +329,7 @@ test "ooxml rels: pivot cache rels tolerate stale first entries and mixed target "pivotCacheGhostRecords.xml", ).replace_all( old="Target=\"pivotCacheRecords1.xml\"", - new="Target=\"xl/pivotCache/pivotCacheRecords1.xml\"", + new="Target=\"/xl/pivotCache/pivotCacheRecords1.xml\"", ) let updated = rewrite_zip_entry( archive_after_pivot_rels, From 47688c94e7f2a0a2b0fc449bf3bf6adabe18754d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 20:55:27 +0800 Subject: [PATCH 094/150] fix(xlsx): preserve ISO week epoch semantics --- xlsx/formula_builtins.mbt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index 389a26d3..fc3f9877 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -3853,11 +3853,21 @@ fn eval_function( } "ISOWEEKNUM" => if values.length() == 1 { - let (year, month, day) = match - date_parts_from_value(values[0], use_1904_dates=ctx.use_1904_dates) { - Ok(parts) => parts + let serial = match + value_as_date_serial(values[0], use_1904_dates=ctx.use_1904_dates) { + Ok(value) => value Err(err) => return err } + // Serial zero is the 1900 system's artificial day before its epoch, + // but is the real 1904-01-01 epoch in a date-1904 workbook. + if !ctx.use_1904_dates && serial < 1.0 { + return Error(formula_error_num) + } + let (year, month, day) = match + date_parts_from_serial(serial, use_1904_dates=ctx.use_1904_dates) { + Some(parts) => parts + None => return Error(formula_error_num) + } match iso_week_number(year, month, day) { Some(week) => Number(Double::from_int(week)) None => Error(formula_error_num) From 148b9a83a507fc2008d3a63d846dd284d8a47d3c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 21:51:31 +0800 Subject: [PATCH 095/150] fix(xlsx): bound fraction number formatting --- office/cmd/office/xlsx_read_wbtest.mbt | 27 +++++++++++++ xlsx/value_format.mbt | 52 ++++++++++++++++++++++---- xlsx/value_format_test.mbt | 21 +++++++++++ 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index cb290090..e7275e76 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -890,6 +890,33 @@ async test "XLSX formatting rejects text amplification before materialization" { ) } +///| +async test "XLSX structured formatting rejects non-finite percent scaling" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_value("Data", "A1", Numeric(0.0)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("# ?/?" + "%".repeat(155)), + ) + workbook.set_cell_style("Data", "A1", style_id) + let source = xlsx_bounded_test_source( + "non-finite-format.xlsx", + @xlsx.write(workbook), + ) + let projection = make_xlsx_projection( + source.file, + open_xlsx_read_package(source), + 10, + ) + let resolved = resolve_xlsx_selector( + projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]", + ) + let error = xlsx_cli_error_async(() => { + ignore(xlsx_get_data(projection, resolved)) + }) + assert_eq(error.code, "office.invalid_package") +} + ///| async test "XLSX outline and get use office schemas with canonical cell records" { let projection = xlsx_read_test_projection() diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index 1f090f2d..680ba115 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -582,13 +582,19 @@ fn format_number_pattern( pattern : String, add_minus : Bool, currency_prefix : String, -) -> String { +) -> String raise XlsxError { let value = @string.parse_double(raw) catch { _ => return raw } + if value.is_nan() || value.is_inf() { + raise InvalidXml(msg="cell number must be finite") + } let negative = value < 0.0 let abs_value = if negative { -value } else { value } let (prefix, suffix, info) = parse_number_pattern(pattern) let scaled = abs_value * @math.pow(100.0, Double::from_int(info.percent_count)) + if scaled.is_nan() || scaled.is_inf() { + raise InvalidXml(msg="number format scale must remain finite") + } let formatted = if info.has_fraction { format_fraction_number(scaled, info) } else if info.has_scientific { @@ -911,6 +917,10 @@ fn float_to_frac_use_continued_fraction( value : Double, denominator_limit : Int64, ) -> (Int64, Int64) { + if value.is_nan() || value.is_inf() || value <= 0.0 || denominator_limit <= 1L { + return (0L, 1L) + } + let maximum_int64 = 9_223_372_036_854_775_807L let mut p1 : Int64 = 1L let mut q1 : Int64 = 0L let mut p2 : Int64 = 0L @@ -918,18 +928,34 @@ fn float_to_frac_use_continued_fraction( let mut lasta : Int64 = 0L let mut lastb : Int64 = 0L let mut r = value - while true { - let a = Double::to_int64(Double::floor(r)) + // A finite Double has at most 53 significant binary digits, so 128 + // continued-fraction steps are more than enough to reach its exact rational + // representation. The hard bound also makes this helper total if a future + // backend changes floating-point corner-case behavior. + for _ in 0..<128 { + if r.is_nan() || r.is_inf() || r < 0.0 { + return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } + } + let floored = Double::floor(r) + if floored > maximum_int64.to_double() { + return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } + } + let a = Double::to_int64(floored) + if a < 0L || + (p1 > 0L && a > (maximum_int64 - p2) / p1) || + (q1 > 0L && a > (maximum_int64 - q2) / q1) { + return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } + } let curra = a * p1 + p2 let currb = a * q1 + q2 + if currb <= 0L || currb >= denominator_limit { + return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } + } p2 = p1 q2 = q1 p1 = curra q1 = currb let frac = r - a.to_double() - if currb >= denominator_limit { - return (lasta, lastb) - } if frac.abs() < 0.000000000001 { return (curra, currb) } @@ -937,7 +963,11 @@ fn float_to_frac_use_continued_fraction( lastb = currb r = 1.0 / frac } - (0L, 1L) + if lastb > 0L { + (lasta, lastb) + } else { + (0L, 1L) + } } ///| @@ -958,9 +988,15 @@ fn format_number_simple( decimals : Int, use_group : Bool, percent : Bool, -) -> String { +) -> String raise XlsxError { let value = @string.parse_double(raw) catch { _ => return raw } + if value.is_nan() || value.is_inf() { + raise InvalidXml(msg="cell number must be finite") + } let scaled = if percent { value * 100.0 } else { value } + if scaled.is_nan() || scaled.is_inf() { + raise InvalidXml(msg="number format scale must remain finite") + } let info = { min_int_digits: 1, required_decimals: decimals, diff --git a/xlsx/value_format_test.mbt b/xlsx/value_format_test.mbt index 437862bc..84859602 100644 --- a/xlsx/value_format_test.mbt +++ b/xlsx/value_format_test.mbt @@ -24,6 +24,27 @@ fn format_text(value : String, fmt : String) -> String { } } +///| +test "fraction formatting rejects non-finite percent scaling" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Sheet1")) + workbook.set_cell_value("Sheet1", "A1", Numeric(0.0)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("# ?/?" + "%".repeat(155)), + ) + workbook.set_cell_style("Sheet1", "A1", style_id) + let result : Result[String?, Error] = Ok( + workbook.get_cell_value("Sheet1", "A1"), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.XlsxError::InvalidXml(msg~)) => + inspect(msg, content="number format scale must remain finite") + _ => fail("expected non-finite number-format scaling rejection") + } +} + ///| test "general and literal formats" { inspect(format_numeric(43543.5448726851, "General"), content="43543.54487") From 185ab9ce49967b5c8aa13f476375387b8a4171b2 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 21:58:54 +0800 Subject: [PATCH 096/150] fix(ooxml): bound OPC part-name registry growth --- ooxml/opc_part_name.mbt | 107 ++++++++++++++++++++++++++++++++++++--- ooxml/pkg.generated.mbti | 6 ++- xlsx/read.mbt | 39 ++++++++++++-- 3 files changed, 140 insertions(+), 12 deletions(-) diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt index 3bb5ab3c..dab164c6 100644 --- a/ooxml/opc_part_name.mbt +++ b/ooxml/opc_part_name.mbt @@ -340,6 +340,51 @@ fn opc_part_name_identity_tokens( tokens } +///| +/// Returns the number of bounded trie nodes needed to represent `name` in an +/// OPC PartName registry. The scan does not allocate identity tokens, allowing +/// callers to reserve cumulative parser capacity before registration starts. +pub fn opc_part_name_identity_node_count_cancellable( + name : StringView, + cancelled? : () -> Bool = () => false, +) -> Int raise ParseXmlError { + let mut count = 0 + let mut segment_start = 0 + let mut index = 0 + while index <= name.length() { + if index % 4096 == 0 && cancelled() { + raise ReadCancelled + } + if index == name.length() || name[index] == ('/' : UInt16) { + if segment_start == index { + count = count + 1 + } else { + let mut chunk_start = segment_start + while chunk_start < index { + let tentative_end = chunk_start + opc_part_name_identity_chunk_units + let mut chunk_end = if tentative_end < index { + tentative_end + } else { + index + } + if chunk_end < index && + name[chunk_end - 1] >= 0xd800 && + name[chunk_end - 1] <= 0xdbff && + name[chunk_end] >= 0xdc00 && + name[chunk_end] <= 0xdfff { + chunk_end = chunk_end - 1 + } + count = count + 1 + chunk_start = chunk_end + } + } + segment_start = index + 1 + } + index = index + 1 + } + count +} + ///| type OpcPartNameChildren = @sorted_map.SortedMap[String, Int] @@ -372,24 +417,36 @@ pub(all) enum OpcPartNameConflict { /// string-hash behavior. pub struct OpcPartNameRegistry { priv nodes : Array[OpcPartNameTrieNode] + priv maximum_identity_nodes : Int } ///| -/// Creates an empty OPC PartName registry. -pub fn OpcPartNameRegistry::new() -> OpcPartNameRegistry { - { nodes: [new_opc_part_name_trie_node()] } +/// Creates an empty OPC PartName registry with a cumulative identity-node +/// ceiling. The root bookkeeping node is not counted against the ceiling. +pub fn OpcPartNameRegistry::new( + maximum_identity_nodes? : Int = 2_000_000, +) -> OpcPartNameRegistry { + { + nodes: [new_opc_part_name_trie_node()], + maximum_identity_nodes: if maximum_identity_nodes < 0 { + 0 + } else { + maximum_identity_nodes + }, + } } ///| /// Registers `name`, using `display` in any returned conflict. Identity is /// ASCII-case-insensitive and a name conflicts when either side is derivable -/// from the other by appending one or more path segments. +/// from the other by appending one or more path segments. Raises `InvalidXml` +/// before mutation if registration would exceed the registry's node ceiling. pub fn OpcPartNameRegistry::register( self : OpcPartNameRegistry, name : StringView, display : String, -) -> OpcPartNameConflict? { - try! self.register_cancellable(name, display) +) -> OpcPartNameConflict? raise ParseXmlError { + self.register_cancellable(name, display) } ///| @@ -446,6 +503,11 @@ pub fn OpcPartNameRegistry::register_cancellable( self.nodes[node_index].terminal = Some(display) } Some(start) => { + let missing_nodes = tokens.length() - start + let existing_nodes = self.nodes.length() - 1 + if missing_nodes > self.maximum_identity_nodes - existing_nodes { + raise InvalidXml(msg="OPC PartName registry node limit exceeded") + } // Construct the missing suffix off-registry. Only after every allocation // and cancellation point succeeds do we append and link it once. let base = self.nodes.length() @@ -553,6 +615,29 @@ test "OPC PartName registry allocates per bounded chunk, not per code unit" { assert_true(registry.nodes.length() < 32) } +///| +test "OPC PartName registry enforces its cumulative node ceiling atomically" { + let registry = OpcPartNameRegistry::new(maximum_identity_nodes=2) + try registry.register_cancellable("a/b/c", "too-large") catch { + InvalidXml(msg~) => + inspect(msg, content="OPC PartName registry node limit exceeded") + _ => fail("unexpected OPC PartName registry limit error") + } noraise { + _ => fail("expected OPC PartName registry node limit") + } + assert_eq(registry.nodes.length(), 1) + assert_true(registry.register("a/b", "a/b") is None) + assert_eq(registry.nodes.length(), 3) + try registry.register_cancellable("a/c", "still-too-large") catch { + InvalidXml(msg~) => + inspect(msg, content="OPC PartName registry node limit exceeded") + _ => fail("unexpected OPC PartName registry limit error") + } noraise { + _ => fail("expected cumulative OPC PartName registry node limit") + } + assert_eq(registry.nodes.length(), 3) +} + ///| test "OPC PartName chunk boundaries preserve segment semantics" { let registry = OpcPartNameRegistry::new() @@ -564,4 +649,14 @@ test "OPC PartName chunk boundaries preserve segment semantics" { Some(Derivable(existing)) => assert_eq(existing, exact_chunk) _ => fail("expected derivable long OPC PartName") } + + let scalar_boundary = "custom/" + "a".repeat(4095) + "😀" + "a".repeat(4095) + let identity_nodes = opc_part_name_identity_node_count_cancellable( + scalar_boundary, + ) + let scalar_registry = OpcPartNameRegistry::new() + assert_true( + scalar_registry.register(scalar_boundary, scalar_boundary) is None, + ) + assert_eq(scalar_registry.nodes.length() - 1, identity_nodes) } diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index c7a4529c..b6e46b5d 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -26,6 +26,8 @@ pub fn logical_opc_part_name_from_zip_item_name(StringView) -> String? pub fn logical_opc_part_name_from_zip_item_name_cancellable(StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError +pub fn opc_part_name_identity_node_count_cancellable(StringView, cancelled? : () -> Bool) -> Int raise ParseXmlError + pub fn package_part_name_key(StringView) -> String pub fn package_part_name_key_cancellable(StringView, cancelled? : () -> Bool) -> String raise ParseXmlError @@ -61,8 +63,8 @@ pub(all) enum OpcPartNameConflict { pub struct OpcPartNameRegistry { // private fields } -pub fn OpcPartNameRegistry::new() -> Self -pub fn OpcPartNameRegistry::register(Self, StringView, String) -> OpcPartNameConflict? +pub fn OpcPartNameRegistry::new(maximum_identity_nodes? : Int) -> Self +pub fn OpcPartNameRegistry::register(Self, StringView, String) -> OpcPartNameConflict? raise ParseXmlError pub fn OpcPartNameRegistry::register_cancellable(Self, StringView, String, cancelled? : () -> Bool) -> OpcPartNameConflict? raise ParseXmlError type WorkbookManifest diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 8806d58c..5d1a23fc 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3156,9 +3156,20 @@ fn budgeted_register_opc_part_name( budget : ReadBudget, ) -> @ooxml.OpcPartNameConflict? raise XlsxError { budget.checkpoint() - // Registration performs a mutation-free lookup pass and, for a new branch, - // constructs a detached suffix before linking it. Charge both linear passes - // before either can run. + // Count bounded identity nodes without allocating them, then reserve their + // cumulative parser capacity before registration can materialize either its + // token array or a detached trie suffix. + budget.charge_work(name.length()) + let identity_nodes = @ooxml.opc_part_name_identity_node_count_cancellable( + name, + cancelled=budget.cancelled, + ) catch { + InvalidXml(msg~) => raise InvalidPackage(msg~) + ReadCancelled => raise ReadCancelled + } + budget.charge_items(identity_nodes) + // Registration performs a tokenization pass followed by a mutation-free + // lookup/allocation pass. Charge both before either can run. budget.charge_work(name.length()) budget.charge_work(name.length()) registry.register_cancellable(name, display, cancelled=budget.cancelled) catch { @@ -3174,7 +3185,9 @@ fn archive_part_name_index( ) -> Map[String, String] raise XlsxError { let budget = budget.unwrap_or(ReadBudget::new(ReadLimits::new())) let names : Map[String, String] = Map([]) - let registry = @ooxml.OpcPartNameRegistry::new() + let registry = @ooxml.OpcPartNameRegistry::new( + maximum_identity_nodes=budget.limits.max_parser_items, + ) let content_types_key = @ooxml.package_part_name_key(content_types_part_path) for entry in archive.entries() { budget.checkpoint() @@ -3322,6 +3335,24 @@ test "read archive index charges physical OPC name work before decoding" { } } +///| +test "read archive index charges OPC identity nodes as parser items" { + let archive = @zip.Archive::new() + archive.add("a/b/c.xml", b"part") + let budget = ReadBudget::new(ReadLimits::with_values(max_parser_items=2)) + try archive_part_name_index(archive, budget~) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + inspect(kind, content="parser_items") + assert_eq(limit, 2) + assert_true(actual > limit) + } + _ => fail("unexpected archive identity-node budget error") + } noraise { + _ => fail("expected archive identity-node parser limit") + } + assert_eq(budget.parser_items, 0) +} + ///| test "read archive index polls cancellation inside long OPC names" { let archive = @zip.Archive::new() From ce5030e73aacb7af2453951a4a39d5a23ce43d83 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 22:00:00 +0800 Subject: [PATCH 097/150] fix(ooxml): reject MCE namespace directives --- ooxml/read_parse.mbt | 6 +++++- ooxml/read_parse_test.mbt | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index f342c537..e21c3b75 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -408,7 +408,8 @@ fn relationship_mce_enter_scope( let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( scanner, prefix, "Ignorable", ) - if namespace_uri == "" { + if namespace_uri == "" || + namespace_uri == markup_compatibility_namespace { raise InvalidXml(msg="MCE Ignorable namespace is invalid") } context.add_ignorable(namespace_id) @@ -503,6 +504,9 @@ fn relationship_mce_choice_supported( let (namespace_uri, _) = relationship_mce_resolve_prefix( scanner, prefix, "Choice Requires", ) + if namespace_uri == markup_compatibility_namespace { + raise InvalidXml(msg="MCE Choice Requires namespace is invalid") + } if namespace_uri != package_relationships_namespace { supported = false } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index f244e808..50a1539b 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -228,6 +228,36 @@ test "ooxml read_parse: invalid relationship MCE fails closed" { ), "MCE Choice Requires is empty", ), + ( + ( + #| + ), + "MCE Ignorable namespace is invalid", + ), + ( + ( + #| + ), + "MCE Ignorable namespace is invalid", + ), + ( + ( + #| + ), + "MCE Ignorable namespace is invalid", + ), + ( + ( + #| + ), + "MCE Ignorable namespace is invalid", + ), + ( + ( + #| + ), + "MCE Choice Requires namespace is invalid", + ), ( ( #| From 002c95a978611a09a1d8c55e15cfdbfc8923b0a1 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 22:02:42 +0800 Subject: [PATCH 098/150] fix(xlsx): validate root relationship targets --- xlsx/read_package_parts.mbt | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index 126bdc0f..b94174a2 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -32,6 +32,7 @@ fn workbook_part_from_root_relationships( } match first_relationship_target(targets) { Some(target) => { + validate_relationship_target_uri(target) let part_name = archive_path_from_part_name(target) normalize_rel_part_path(part_name) } @@ -158,6 +159,45 @@ test "read package parts: workbook path follows the root relationship" { ) } +///| +fn workbook_part_from_root_target_for_test( + target : StringView, +) -> String raise XlsxError { + let archive = @zip.Archive::new() + let root_relationships = "" + + "" + archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + workbook_part_from_root_relationships( + archive, + archive_part_name_index(archive), + decode_utf8_or_invalid_xml, + ) +} + +///| +test "read package parts: root target rejects absolute URI syntax" { + for target in ["urn:book.xml", "a:b/c"] { + try workbook_part_from_root_target_for_test(target) catch { + InvalidXml(msg~) => inspect(msg, content="relationship target invalid") + _ => fail("unexpected absolute root-target error") + } noraise { + _ => fail("expected absolute root-target rejection") + } + } +} + +///| +test "read package parts: explicit relative and package-absolute root targets remain valid" { + for target in ["./urn:book.xml", "/urn:book.xml"] { + inspect( + workbook_part_from_root_target_for_test(target), + content="urn:book.xml", + ) + } +} + ///| test "read package parts: external workbook relationship is never resolved as a part" { let archive = @zip.Archive::new() From 59db85467be115c76a660104c210920271d8c52c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 22:08:54 +0800 Subject: [PATCH 099/150] fix(xlsx): preserve phantom ISO week --- xlsx/calc_test.mbt | 4 ++++ xlsx/formula_builtins.mbt | 6 ++++++ xlsx/formula_builtins_wbtest.mbt | 9 +++++++++ xlsx/formula_eval.mbt | 5 +++++ xlsx/formula_eval_wbtest.mbt | 10 ++++++++++ 5 files changed, 34 insertions(+) diff --git a/xlsx/calc_test.mbt b/xlsx/calc_test.mbt index 61535dae..ed5c0e58 100644 --- a/xlsx/calc_test.mbt +++ b/xlsx/calc_test.mbt @@ -3651,6 +3651,10 @@ test "calc date and time functions" { inspect(workbook.calc_cell_value("Sheet1", "G4"), content="0.5") sheet.set_cell_formula("G5", "YEARFRAC(DATE(2020,1,1),DATE(2021,1,1),1)") inspect(workbook.calc_cell_value("Sheet1", "G5"), content="1") + sheet.set_cell_formula("G6", "ISOWEEKNUM(60)") + inspect(workbook.calc_cell_value("Sheet1", "G6"), content="9") + sheet.set_cell_formula("G7", "ISOWEEKNUM(60.5)") + inspect(workbook.calc_cell_value("Sheet1", "G7"), content="9") sheet.set_cell_formula("H1", "WEEKDAY(DATE(2020,1,1))") inspect(workbook.calc_cell_value("Sheet1", "H1"), content="4") sheet.set_cell_formula("H2", "WEEKDAY(DATE(2020,1,1),2)") diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index fc3f9877..a4805cce 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -3863,6 +3863,12 @@ fn eval_function( if !ctx.use_1904_dates && serial < 1.0 { return Error(formula_error_num) } + // Excel assigns the fictitious 1900-02-29 (including its time + // fraction) to ISO week 9. It cannot pass through the Gregorian helper + // below because 1900 was not actually a leap year. + if is_1900_phantom_date_serial(serial, ctx.use_1904_dates) { + return Number(9.0) + } let (year, month, day) = match date_parts_from_serial(serial, use_1904_dates=ctx.use_1904_dates) { Some(parts) => parts diff --git a/xlsx/formula_builtins_wbtest.mbt b/xlsx/formula_builtins_wbtest.mbt index 43a8dd7a..2b44393d 100644 --- a/xlsx/formula_builtins_wbtest.mbt +++ b/xlsx/formula_builtins_wbtest.mbt @@ -18548,6 +18548,15 @@ test "formula builtins wb: residual hotspot probes part 75" { ), content="true", ) + for serial in [60.0, 60.5] { + inspect( + is_formula_number( + eval_function(workbook, "Sheet1", "ISOWEEKNUM", [Number(serial)], ctx), + 9.0, + ), + content="true", + ) + } inspect( match eval_function( diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index ace30365..5c86e610 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -5927,6 +5927,11 @@ fn date_parts_from_serial( } } +///| +fn is_1900_phantom_date_serial(serial : Double, use_1904_dates : Bool) -> Bool { + !use_1904_dates && serial >= 60.0 && serial < 61.0 +} + ///| fn validated_date_serial( serial : Double, diff --git a/xlsx/formula_eval_wbtest.mbt b/xlsx/formula_eval_wbtest.mbt index 7f723feb..a888ba59 100644 --- a/xlsx/formula_eval_wbtest.mbt +++ b/xlsx/formula_eval_wbtest.mbt @@ -2069,6 +2069,16 @@ test "formula eval wb: date serial and weekend helper branches part 4" { ) inspect(date_parts_from_serial(-1.0) is None, content="true") + debug_inspect( + ( + is_1900_phantom_date_serial(60.0, false), + is_1900_phantom_date_serial(60.5, false), + is_1900_phantom_date_serial(59.999, false), + is_1900_phantom_date_serial(61.0, false), + is_1900_phantom_date_serial(60.0, true), + ), + content="(true, true, false, false, false)", + ) inspect( match date_parts_from_value(Error(formula_error_ref)) { Err(Error(err)) => err == formula_error_ref From af174434a4fad8613a62e61a8a654f5041ba5e34 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 23:13:25 +0800 Subject: [PATCH 100/150] fix(ooxml): enforce relationship MCE grammar --- ooxml/read_parse.mbt | 82 ++++++++++++++++++++++++++++++++++--- ooxml/read_parse_test.mbt | 60 +++++++++++++++++++++++++++ ooxml/start_tag_scanner.mbt | 77 ++++++++++++++++++++++++++++++++++ 3 files changed, 213 insertions(+), 6 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index e21c3b75..85f06803 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -186,6 +186,12 @@ priv struct RelationshipMceScopeDelta { process_content_names : Array[RelationshipMceName] } +///| +priv struct RelationshipMceScopeEntry { + delta : RelationshipMceScopeDelta? + must_understand_supported : Bool +} + ///| /// Active MCE directives are reference-counted and unwound with XML depth. /// This keeps scope entry O(number of new directives), without cloning every @@ -368,7 +374,7 @@ fn relationship_mce_resolve_prefix( prefix : StringView, label : StringView, ) -> (String, Int) raise ParseXmlError { - if prefix == "" || prefix.contains(":") { + if !is_xml_ncname(prefix) { raise InvalidXml(msg="MCE \{label.to_owned()} prefix is invalid") } scanner.resolve_prefix_identity(prefix, false) @@ -381,7 +387,9 @@ fn relationship_mce_expanded_name_key( label : StringView, ) -> (String, RelationshipMceName) raise ParseXmlError { let parts = token.split(":").collect() - if parts.length() != 2 || parts[0] == "" || parts[1] == "" { + if parts.length() != 2 || + !is_xml_ncname(parts[0]) || + (parts[1] != "*" && !is_xml_ncname(parts[1])) { raise InvalidXml(msg="MCE \{label.to_owned()} name is invalid") } let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( @@ -396,8 +404,9 @@ fn relationship_mce_expanded_name_key( fn relationship_mce_enter_scope( scanner : XmlStartTagScanner, context : RelationshipMceContext, -) -> RelationshipMceScopeDelta? raise ParseXmlError { +) -> RelationshipMceScopeEntry raise ParseXmlError { let mut delta : RelationshipMceScopeDelta? = None + let mut must_understand_supported = true let ignorable_attribute = scanner.attribute( markup_compatibility_namespace, "Ignorable", ) @@ -459,7 +468,7 @@ fn relationship_mce_enter_scope( scanner, prefix, "MustUnderstand", ) if namespace_uri != package_relationships_namespace { - raise InvalidXml(msg="MCE MustUnderstand namespace is unsupported") + must_understand_supported = false } } } @@ -481,7 +490,42 @@ fn relationship_mce_enter_scope( None => () } } - delta + { delta, must_understand_supported } +} + +///| +fn relationship_mce_require_must_understand( + entry : RelationshipMceScopeEntry, +) -> Unit raise ParseXmlError { + if !entry.must_understand_supported { + raise InvalidXml(msg="MCE MustUnderstand namespace is unsupported") + } +} + +///| +fn relationship_mce_validate_container_attributes( + scanner : XmlStartTagScanner, + context : RelationshipMceContext, + element_label : StringView, + allow_requires : Bool, +) -> Unit raise ParseXmlError { + for attribute in scanner.expanded_attribute_names() { + if allow_requires && + attribute.namespace_uri == "" && + attribute.local_name == "Requires" { + continue + } + if attribute.namespace_uri != xml_namespace_uri && + ( + attribute.namespace_uri == markup_compatibility_namespace || + context.is_ignorable(attribute.namespace_id) + ) { + continue + } + raise InvalidXml( + msg="MCE \{element_label.to_owned()} has a disallowed attribute", + ) + } } ///| @@ -560,12 +604,14 @@ fn relationship_mce_state_for_current_element( } _ => () } - let scope_delta = relationship_mce_enter_scope(scanner, context) + let scope_entry = relationship_mce_enter_scope(scanner, context) + let scope_delta = scope_entry.delta if depth == 1 { if scanner.namespace_uri() != package_relationships_namespace || scanner.local_name() != "Relationships" { raise InvalidXml(msg="relationships document element invalid") } + relationship_mce_require_must_understand(scope_entry) return { active: true, included_in_effective_tree: true, @@ -591,6 +637,9 @@ fn relationship_mce_state_for_current_element( if parent_state.is_alternate_content { if namespace_uri == markup_compatibility_namespace && local_name == "Choice" { + relationship_mce_validate_container_attributes( + scanner, context, "Choice", true, + ) if parent_state.alternate_fallback_seen { raise InvalidXml(msg="MCE Choice follows Fallback") } @@ -601,9 +650,13 @@ fn relationship_mce_state_for_current_element( supported if active { parent_state.alternate_selected = true + relationship_mce_require_must_understand(scope_entry) } } else if namespace_uri == markup_compatibility_namespace && local_name == "Fallback" { + relationship_mce_validate_container_attributes( + scanner, context, "Fallback", false, + ) if parent_state.alternate_fallback_seen { raise InvalidXml( msg="MCE AlternateContent has multiple Fallback elements", @@ -613,8 +666,13 @@ fn relationship_mce_state_for_current_element( active = parent_state.active && !parent_state.alternate_selected if active { parent_state.alternate_selected = true + relationship_mce_require_must_understand(scope_entry) } } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { + let key : RelationshipMceName = { namespace_id, local_name } + if context.processes(key) { + raise InvalidXml(msg="MCE AlternateContent child is invalid") + } active = false } else { raise InvalidXml(msg="MCE AlternateContent child is invalid") @@ -623,8 +681,14 @@ fn relationship_mce_state_for_current_element( if local_name != "AlternateContent" { raise InvalidXml(msg="MCE element is misplaced") } + relationship_mce_validate_container_attributes( + scanner, context, "AlternateContent", false, + ) active = parent_state.active is_alternate_content = true + if active { + relationship_mce_require_must_understand(scope_entry) + } } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { let key : RelationshipMceName = { namespace_id, local_name } if context.processes(key) { @@ -636,6 +700,9 @@ fn relationship_mce_state_for_current_element( ) } active = parent_state.active + if active { + relationship_mce_require_must_understand(scope_entry) + } } else { active = false } @@ -644,6 +711,9 @@ fn relationship_mce_state_for_current_element( included_in_effective_tree = true element_effective_depth = parent_effective_depth + 1 children_effective_parent_depth = element_effective_depth + if active { + relationship_mce_require_must_understand(scope_entry) + } } { active, diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 50a1539b..890b03ce 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -291,6 +291,66 @@ test "ooxml read_parse: unselected MCE choices remain opaque" { assert_eq(targets.length(), 1) } +///| +test "ooxml read_parse: unselected Choice does not enforce MustUnderstand" { + let xml = + #| + let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + assert_eq(targets.get("ok"), Some("ok.xml")) + assert_eq(targets.length(), 1) +} + +///| +test "ooxml read_parse: MCE container grammar fails closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "MCE ProcessContent name is invalid", + ), + ( + ( + #| + ), + "MCE ProcessContent name is invalid", + ), + ( + ( + #| + ), + "MCE AlternateContent has a disallowed attribute", + ), + ( + ( + #| + ), + "MCE Choice has a disallowed attribute", + ), + ( + ( + #| + ), + "MCE Fallback has a disallowed attribute", + ), + ( + ( + #| + ), + "MCE AlternateContent child is invalid", + ), + ] + for case in cases { + let (xml, expected) = case + try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected relationship parse cancellation") + } noraise { + _ => fail("expected invalid relationship MCE") + } + } +} + ///| test "ooxml read_parse: invalid target modes fail closed" { let xml = diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index a0722f21..cb1de242 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -142,6 +142,31 @@ fn is_xml_name_scalar(code : Int) -> Bool { (code >= 0x203f && code <= 0x2040) } +///| +/// Validates an XML NCName using the same Unicode scalar rules as the scanner's +/// QName validation. MCE directive tokens are attribute *values*, so they do +/// not pass through `validate_xml_qname` automatically. +fn is_xml_ncname(value : StringView) -> Bool { + if value.length() == 0 { + return false + } + let mut index = 0 + while index < value.length() { + guard xml_name_scalar_at(value, index, value.length()) is Some((code, next)) else { + return false + } + if index == 0 { + if !is_xml_name_start_scalar(code) { + return false + } + } else if !is_xml_name_scalar(code) { + return false + } + index = next + } + true +} + ///| fn validate_xml_qname( xml : StringView, @@ -1868,6 +1893,58 @@ pub fn XmlStartTagScanner::attribute( } } +///| +priv struct XmlExpandedAttributeName { + namespace_uri : String + namespace_id : Int + local_name : String +} + +///| +/// Enumerates non-namespace-declaration attributes by expanded name. Scanner +/// validation has already bounded the start-tag attribute count and rejected +/// duplicate expanded names before this helper is called. +fn XmlStartTagScanner::expanded_attribute_names( + self : XmlStartTagScanner, +) -> Array[XmlExpandedAttributeName] raise ParseXmlError { + let output : Array[XmlExpandedAttributeName] = [] + let mut next = self.current_name_end + while next_xml_attribute( + self.xml, + next, + self.current_tag_end, + cancelled=self.cancelled, + ) + is Some(attribute) { + next = attribute.next + if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { + continue + } + let colon = validate_xml_qname( + self.xml, + attribute.name_start, + attribute.name_end, + ) + let (prefix, local_start) = match colon { + Some(value) => (self.xml[attribute.name_start:value], value + 1) + None => + ( + self.xml[attribute.name_start:attribute.name_start], + attribute.name_start, + ) + } + let (namespace_uri, namespace_id) = self.resolve_prefix_identity( + prefix, true, + ) + output.push({ + namespace_uri, + namespace_id, + local_name: self.xml[local_start:attribute.name_end].to_owned(), + }) + } + output +} + ///| test "start tag scanner resolves namespaces and decodes attributes once" { let xml = From 96c67eec4e58438d8cafe56a3bd72662db1fd997 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 23:26:36 +0800 Subject: [PATCH 101/150] fix(xlsx): project SpreadsheetML MCE fallbacks --- ooxml/mce_projection.mbt | 612 ++++++++++++++++++++++++++++++++++ ooxml/mce_projection_test.mbt | 78 +++++ ooxml/pkg.generated.mbti | 2 + ooxml/start_tag_scanner.mbt | 116 ++++++- xlsx/read.mbt | 48 ++- xlsx/read_namespace_test.mbt | 34 ++ xlsx/read_xml_namespaces.mbt | 52 +++ 7 files changed, 922 insertions(+), 20 deletions(-) create mode 100644 ooxml/mce_projection.mbt create mode 100644 ooxml/mce_projection_test.mbt diff --git a/ooxml/mce_projection.mbt b/ooxml/mce_projection.mbt new file mode 100644 index 00000000..9fc36648 --- /dev/null +++ b/ooxml/mce_projection.mbt @@ -0,0 +1,612 @@ +///| +let max_xml_mce_tokens = 4096 + +///| +let max_xml_mce_token_chars = 1024 + +///| +priv struct XmlMceName { + namespace_id : Int + local_name : String +} derive(Eq, Hash) + +///| +priv struct XmlMceScopeDelta { + ignorable_namespace_ids : Array[Int] + process_content_names : Array[XmlMceName] +} + +///| +priv struct XmlMceScopeEntry { + delta : XmlMceScopeDelta? + must_understand_supported : Bool +} + +///| +priv struct XmlMceContext { + understood_namespaces : Set[String] + ignorable_namespace_counts : Map[Int, Int] + process_content_name_counts : Map[XmlMceName, Int] + max_work_chars : Int + mut work_chars : Int +} + +///| +fn XmlMceContext::new( + understood_namespaces : ArrayView[String], + max_work_chars : Int, +) -> XmlMceContext { + let understood : Set[String] = Set([]) + for namespace_uri in understood_namespaces { + understood.add(namespace_uri) + } + { + understood_namespaces: understood, + ignorable_namespace_counts: Map([]), + process_content_name_counts: Map([]), + max_work_chars, + work_chars: 0, + } +} + +///| +fn XmlMceContext::charge( + self : XmlMceContext, + value : StringView, +) -> Unit raise ParseXmlError { + if value.length() > self.max_work_chars - self.work_chars { + raise InvalidXml(msg="MCE directive work limit exceeded") + } + self.work_chars += value.length() +} + +///| +fn XmlMceContext::understands( + self : XmlMceContext, + namespace_uri : StringView, +) -> Bool { + self.understood_namespaces.contains(namespace_uri.to_owned()) +} + +///| +fn XmlMceContext::add_ignorable( + self : XmlMceContext, + namespace_id : Int, +) -> Unit { + self.ignorable_namespace_counts[namespace_id] = self.ignorable_namespace_counts.get_or_default( + namespace_id, 0, + ) + + 1 +} + +///| +fn XmlMceContext::add_process_content( + self : XmlMceContext, + name : XmlMceName, +) -> Unit { + self.process_content_name_counts[name] = self.process_content_name_counts.get_or_default( + name, 0, + ) + + 1 +} + +///| +fn XmlMceContext::is_ignorable( + self : XmlMceContext, + namespace_id : Int, +) -> Bool { + self.ignorable_namespace_counts.get_or_default(namespace_id, 0) > 0 +} + +///| +fn XmlMceContext::is_effectively_ignorable( + self : XmlMceContext, + namespace_id : Int, + namespace_uri : StringView, +) -> Bool { + self.is_ignorable(namespace_id) && !self.understands(namespace_uri) +} + +///| +fn XmlMceContext::processes(self : XmlMceContext, name : XmlMceName) -> Bool { + self.process_content_name_counts.get_or_default(name, 0) > 0 || + self.process_content_name_counts.get_or_default( + { namespace_id: name.namespace_id, local_name: "*" }, + 0, + ) > + 0 +} + +///| +fn XmlMceContext::leave_scope( + self : XmlMceContext, + delta : XmlMceScopeDelta?, +) -> Unit { + guard delta is Some(delta) else { return } + for namespace_id in delta.ignorable_namespace_ids { + let remaining = self.ignorable_namespace_counts.get_or_default( + namespace_id, 0, + ) - + 1 + if remaining <= 0 { + ignore(self.ignorable_namespace_counts.remove(namespace_id)) + } else { + self.ignorable_namespace_counts[namespace_id] = remaining + } + } + for name in delta.process_content_names { + let remaining = self.process_content_name_counts.get_or_default(name, 0) - 1 + if remaining <= 0 { + ignore(self.process_content_name_counts.remove(name)) + } else { + self.process_content_name_counts[name] = remaining + } + } +} + +///| +priv enum XmlMceBindingScope { + BindingScope(Array[(String, String)], XmlMceBindingScope?) +} + +///| +fn xml_mce_extend_bindings( + parent : XmlMceBindingScope?, + declarations : Array[(String, String)], +) -> XmlMceBindingScope? { + if declarations.length() == 0 { + parent + } else { + Some(BindingScope(declarations, parent)) + } +} + +///| +priv struct XmlMceProjectionState { + active : Bool + scope_delta : XmlMceScopeDelta? + pending_bindings : XmlMceBindingScope? + is_alternate_content : Bool + mut alternate_choice_seen : Bool + mut alternate_fallback_seen : Bool + mut alternate_selected : Bool +} + +///| +fn xml_mce_tokens( + value : StringView, + label : StringView, + allow_empty? : Bool = true, +) -> Array[String] raise ParseXmlError { + let normalized = collapse_schema_whitespace(value) + if normalized == "" { + if allow_empty { + return [] + } + raise InvalidXml(msg="MCE \{label.to_owned()} is empty") + } + let output : Array[String] = [] + for token in normalized.split(" ") { + if token == "" || token.length() > max_xml_mce_token_chars { + raise InvalidXml(msg="MCE \{label.to_owned()} is invalid") + } + if output.length() >= max_xml_mce_tokens { + raise InvalidXml(msg="MCE \{label.to_owned()} token limit exceeded") + } + output.push(token.to_owned()) + } + output +} + +///| +fn xml_mce_resolve_prefix( + scanner : XmlStartTagScanner, + prefix : StringView, + label : StringView, +) -> (String, Int) raise ParseXmlError { + if !is_xml_ncname(prefix) { + raise InvalidXml(msg="MCE \{label.to_owned()} prefix is invalid") + } + scanner.resolve_prefix_identity(prefix, false) +} + +///| +fn xml_mce_expanded_name( + scanner : XmlStartTagScanner, + token : StringView, + label : StringView, +) -> (String, XmlMceName) raise ParseXmlError { + let parts = token.split(":").collect() + if parts.length() != 2 || + !is_xml_ncname(parts[0]) || + (parts[1] != "*" && !is_xml_ncname(parts[1])) { + raise InvalidXml(msg="MCE \{label.to_owned()} name is invalid") + } + let (namespace_uri, namespace_id) = xml_mce_resolve_prefix( + scanner, + parts[0], + label, + ) + (namespace_uri, { namespace_id, local_name: parts[1].to_owned() }) +} + +///| +fn xml_mce_enter_scope( + scanner : XmlStartTagScanner, + context : XmlMceContext, +) -> XmlMceScopeEntry raise ParseXmlError { + let mut delta : XmlMceScopeDelta? = None + let mut must_understand_supported = true + match scanner.attribute(markup_compatibility_namespace, "Ignorable") { + Some(value) => { + context.charge(value) + for prefix in xml_mce_tokens(value, "Ignorable") { + let (namespace_uri, namespace_id) = xml_mce_resolve_prefix( + scanner, prefix, "Ignorable", + ) + if namespace_uri == "" || + namespace_uri == markup_compatibility_namespace { + raise InvalidXml(msg="MCE Ignorable namespace is invalid") + } + context.add_ignorable(namespace_id) + match delta { + Some(value) => value.ignorable_namespace_ids.push(namespace_id) + None => + delta = Some({ + ignorable_namespace_ids: [namespace_id], + process_content_names: [], + }) + } + } + } + None => () + } + match scanner.attribute(markup_compatibility_namespace, "ProcessContent") { + Some(value) => { + context.charge(value) + for token in xml_mce_tokens(value, "ProcessContent") { + let (_, name) = xml_mce_expanded_name(scanner, token, "ProcessContent") + if !context.is_ignorable(name.namespace_id) { + raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") + } + context.add_process_content(name) + match delta { + Some(value) => value.process_content_names.push(name) + None => + delta = Some({ + ignorable_namespace_ids: [], + process_content_names: [name], + }) + } + } + } + None => () + } + match scanner.attribute(markup_compatibility_namespace, "MustUnderstand") { + Some(value) => { + context.charge(value) + for prefix in xml_mce_tokens(value, "MustUnderstand") { + let (namespace_uri, _) = xml_mce_resolve_prefix( + scanner, prefix, "MustUnderstand", + ) + if namespace_uri == markup_compatibility_namespace || + !context.understands(namespace_uri) { + must_understand_supported = false + } + } + } + None => () + } + for directive in ["PreserveElements", "PreserveAttributes"] { + match scanner.attribute(markup_compatibility_namespace, directive) { + Some(value) => { + context.charge(value) + for token in xml_mce_tokens(value, directive) { + let (_, name) = xml_mce_expanded_name(scanner, token, directive) + if !context.is_ignorable(name.namespace_id) { + raise InvalidXml(msg="MCE \{directive} namespace is not ignorable") + } + } + } + None => () + } + } + { delta, must_understand_supported } +} + +///| +fn xml_mce_require_must_understand( + entry : XmlMceScopeEntry, +) -> Unit raise ParseXmlError { + if !entry.must_understand_supported { + raise InvalidXml(msg="MCE MustUnderstand namespace is unsupported") + } +} + +///| +fn xml_mce_validate_container_attributes( + scanner : XmlStartTagScanner, + context : XmlMceContext, + label : StringView, + allow_requires : Bool, +) -> Unit raise ParseXmlError { + for attribute in scanner.expanded_attribute_names() { + if allow_requires && + attribute.namespace_uri == "" && + attribute.local_name == "Requires" { + continue + } + if attribute.namespace_uri != xml_namespace_uri && + ( + attribute.namespace_uri == markup_compatibility_namespace || + context.is_ignorable(attribute.namespace_id) + ) { + continue + } + raise InvalidXml(msg="MCE \{label.to_owned()} has a disallowed attribute") + } +} + +///| +fn xml_mce_choice_supported( + scanner : XmlStartTagScanner, + context : XmlMceContext, +) -> Bool raise ParseXmlError { + let requires = match scanner.attribute("", "Requires") { + Some(value) => value + None => raise InvalidXml(msg="MCE Choice is missing Requires") + } + context.charge(requires) + let mut supported = true + for prefix in xml_mce_tokens(requires, "Choice Requires", allow_empty=false) { + let (namespace_uri, _) = xml_mce_resolve_prefix( + scanner, prefix, "Choice Requires", + ) + if namespace_uri == markup_compatibility_namespace { + raise InvalidXml(msg="MCE Choice Requires namespace is invalid") + } + if !context.understands(namespace_uri) { + supported = false + } + } + supported +} + +///| +fn xml_mce_namespace_attributes( + scanner : XmlStartTagScanner, + pending : XmlMceBindingScope?, +) -> String { + guard pending is Some(_) else { return "" } + let seen : Set[String] = Set([]) + for declaration in scanner.current_namespace_declarations() { + let (prefix, _) = declaration + seen.add(prefix) + } + let output = StringBuilder::new() + let mut current = pending + while current is Some(BindingScope(declarations, parent)) { + for declaration in declarations { + let (prefix, namespace_uri) = declaration + if prefix == "xml" || seen.contains(prefix) { + continue + } + seen.add(prefix) + output.write_string(" xmlns") + if prefix != "" { + output.write_char(':') + output.write_string(prefix) + } + output.write_string("=\"") + output.write_string(escape_xml_attr(namespace_uri)) + output.write_char('"') + } + current = parent + } + output.to_string() +} + +///| +fn trim_xml_mce_states( + states : Array[XmlMceProjectionState], + next_depth : Int, + context : XmlMceContext, +) -> Unit raise ParseXmlError { + while states.length() >= next_depth { + let state = states.pop().unwrap() + context.leave_scope(state.scope_delta) + if state.is_alternate_content && !state.alternate_choice_seen { + raise InvalidXml(msg="MCE AlternateContent has no Choice element") + } + } +} + +///| +fn xml_mce_state_for_current_element( + scanner : XmlStartTagScanner, + states : Array[XmlMceProjectionState], + context : XmlMceContext, +) -> XmlMceProjectionState raise ParseXmlError { + let depth = scanner.depth() + trim_xml_mce_states(states, depth, context) + let parent = if depth > 1 { + if states.length() != depth - 1 { + raise InvalidXml(msg="MCE projection nesting is invalid") + } + Some(states[depth - 2]) + } else { + None + } + match parent { + Some(parent_state) if !parent_state.active => + return { + active: false, + scope_delta: None, + pending_bindings: None, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } + _ => () + } + let scope_entry = xml_mce_enter_scope(scanner, context) + let declarations = scanner.current_namespace_declarations() + if depth == 1 { + if scanner.namespace_uri() == markup_compatibility_namespace { + raise InvalidXml(msg="MCE document element is invalid") + } + xml_mce_require_must_understand(scope_entry) + return { + active: true, + scope_delta: scope_entry.delta, + pending_bindings: None, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } + } + let parent_state = parent.unwrap() + let namespace_uri = scanner.namespace_uri().to_owned() + let namespace_id = scanner.current_namespace_identity() + let local_name = scanner.local_name().to_owned() + let mut active = parent_state.active + let mut pending_bindings : XmlMceBindingScope? = None + let mut is_alternate_content = false + if parent_state.is_alternate_content { + if namespace_uri == markup_compatibility_namespace && local_name == "Choice" { + xml_mce_validate_container_attributes(scanner, context, "Choice", true) + if parent_state.alternate_fallback_seen { + raise InvalidXml(msg="MCE Choice follows Fallback") + } + parent_state.alternate_choice_seen = true + let supported = xml_mce_choice_supported(scanner, context) + active = !parent_state.alternate_selected && supported + if active { + parent_state.alternate_selected = true + xml_mce_require_must_understand(scope_entry) + scanner.unwrap_current_element() + pending_bindings = xml_mce_extend_bindings( + parent_state.pending_bindings, + declarations, + ) + } else { + scanner.remove_current_subtree() + } + } else if namespace_uri == markup_compatibility_namespace && + local_name == "Fallback" { + xml_mce_validate_container_attributes(scanner, context, "Fallback", false) + if parent_state.alternate_fallback_seen { + raise InvalidXml( + msg="MCE AlternateContent has multiple Fallback elements", + ) + } + parent_state.alternate_fallback_seen = true + active = !parent_state.alternate_selected + if active { + parent_state.alternate_selected = true + xml_mce_require_must_understand(scope_entry) + scanner.unwrap_current_element() + pending_bindings = xml_mce_extend_bindings( + parent_state.pending_bindings, + declarations, + ) + } else { + scanner.remove_current_subtree() + } + } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { + if context.processes({ namespace_id, local_name }) { + raise InvalidXml(msg="MCE AlternateContent child is invalid") + } + active = false + scanner.remove_current_subtree() + } else { + raise InvalidXml(msg="MCE AlternateContent child is invalid") + } + } else if namespace_uri == markup_compatibility_namespace { + if local_name != "AlternateContent" { + raise InvalidXml(msg="MCE element is misplaced") + } + xml_mce_validate_container_attributes( + scanner, context, "AlternateContent", false, + ) + xml_mce_require_must_understand(scope_entry) + scanner.unwrap_current_element() + pending_bindings = xml_mce_extend_bindings( + parent_state.pending_bindings, + declarations, + ) + is_alternate_content = true + } else if context.is_effectively_ignorable(namespace_id, namespace_uri) { + let name : XmlMceName = { namespace_id, local_name } + if context.processes(name) { + if scanner.attribute(xml_namespace_uri, "base") is Some(_) || + scanner.attribute(xml_namespace_uri, "lang") is Some(_) || + scanner.attribute(xml_namespace_uri, "space") is Some(_) { + raise InvalidXml( + msg="MCE ProcessContent element has an unsafe xml attribute", + ) + } + xml_mce_require_must_understand(scope_entry) + scanner.unwrap_current_element() + pending_bindings = xml_mce_extend_bindings( + parent_state.pending_bindings, + declarations, + ) + } else { + active = false + scanner.remove_current_subtree() + } + } else { + xml_mce_require_must_understand(scope_entry) + scanner.insert_current_attributes( + xml_mce_namespace_attributes(scanner, parent_state.pending_bindings), + ) + } + { + active, + scope_delta: scope_entry.delta, + pending_bindings, + is_alternate_content, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + } +} + +///| +/// Applies ECMA-376 Markup Compatibility selection to an XML part and returns +/// its effective projection. Unsupported `Choice` branches and ignorable +/// subtrees are removed, selected `Choice`/`Fallback` and `ProcessContent` +/// wrappers are unwrapped, and namespace declarations from removed wrappers +/// are retained on promoted children. Processing is streaming, cancellable, +/// and bounded by `max_output_chars`. +pub fn project_xml_markup_compatibility( + xml : StringView, + understood_namespaces : ArrayView[String], + max_output_chars? : Int = 16 * 1024 * 1024, + cancelled? : () -> Bool = () => false, +) -> String raise ParseXmlError { + check_xml_cancelled(cancelled) + if max_output_chars < 0 { + raise InvalidXml(msg="MCE output limit is invalid") + } + let scanner = xml_start_tag_scanner_with_rewrites( + xml, + Map([]), + Map([]), + Map([]), + Map([]), + false, + max_output_chars, + cancelled~, + ) + let context = XmlMceContext::new(understood_namespaces, max_output_chars) + let states : Array[XmlMceProjectionState] = [] + while scanner.next() { + states.push(xml_mce_state_for_current_element(scanner, states, context)) + } + trim_xml_mce_states(states, 1, context) + scanner.finish_rewritten_xml() +} diff --git a/ooxml/mce_projection_test.mbt b/ooxml/mce_projection_test.mbt new file mode 100644 index 00000000..d5933573 --- /dev/null +++ b/ooxml/mce_projection_test.mbt @@ -0,0 +1,78 @@ +///| +test "MCE projection selects fallback and preserves promoted namespaces" { + let xml = + #| + let projected = @ooxml.project_xml_markup_compatibility(xml, ["urn:app"]) + assert_false(projected.contains("AlternateContent")) + assert_false(projected.contains("Future")) + assert_true(projected.contains("q:Item")) + let scanner = @ooxml.XmlStartTagScanner::new(projected) + assert_true(scanner.next()) + assert_true(scanner.next()) + inspect(scanner.namespace_uri(), content="urn:app") + inspect(scanner.local_name(), content="Item") + debug_inspect(scanner.attribute("urn:app", "id"), content="Some(\"kept\")") + assert_false(scanner.next()) +} + +///| +test "MCE projection selects the first understood Choice" { + let xml = + #| + let projected = @ooxml.project_xml_markup_compatibility(xml, ["urn:app"]) + assert_true(projected.contains(" + let projected = @ooxml.project_xml_markup_compatibility(xml, ["urn:app"]) + assert_false(projected.contains("v:Drop")) + assert_false(projected.contains(" + let projected = @ooxml.project_xml_markup_compatibility(xml, ["urn:app"]) + assert_true(projected.contains("a:Kept")) +} + +///| +test "MCE projection output and cancellation are bounded" { + let xml = + #| + try + @ooxml.project_xml_markup_compatibility( + xml, + ["urn:app"], + max_output_chars=8, + ) + catch { + InvalidXml(msg~) => + inspect(msg, content="XML canonical output limit exceeded") + _ => fail("unexpected MCE projection cancellation") + } noraise { + _ => fail("expected MCE projection output limit") + } + let checks = [0] + try + @ooxml.project_xml_markup_compatibility( + "" + "x".repeat(32 * 1024) + "", + ["urn:app"], + cancelled=() => { + checks[0] += 1 + checks[0] >= 8 + }, + ) + catch { + ReadCancelled => assert_true(checks[0] >= 8) + _ => fail("unexpected bounded MCE projection error") + } noraise { + _ => fail("expected MCE projection cancellation") + } +} diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index b6e46b5d..93369095 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -40,6 +40,8 @@ pub fn parse_internal_relationship_targets(StringView, StringView, cancelled? : pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError +pub fn project_xml_markup_compatibility(StringView, ArrayView[String], max_output_chars? : Int, cancelled? : () -> Bool) -> String raise ParseXmlError + pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError pub fn remove_xml_foreign_namespace_subtrees(StringView, ArrayView[String], cancelled? : () -> Bool) -> String? raise ParseXmlError diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index cb1de242..4403edc1 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -57,6 +57,7 @@ pub struct XmlStartTagScanner { priv open_local_name_starts : Array[Int] priv open_local_name_ends : Array[Int] priv open_namespace_uris : Array[String] + priv open_remove_end_tags : Array[Bool] priv rewrite_element_prefixes : Map[String, String] priv reserved_element_prefixes : Map[String, String] priv rewrite_attribute_prefixes : Map[String, String] @@ -72,7 +73,10 @@ pub struct XmlStartTagScanner { priv mut root_closed : Bool priv mut current_name_start : Int priv mut current_name_end : Int + priv mut current_tag_start : Int priv mut current_tag_end : Int + priv mut current_attribute_insert : Int + priv mut current_self_closing : Bool priv mut current_depth : Int priv mut current_namespace_uri : String priv mut current_namespace_id : Int @@ -1234,6 +1238,7 @@ fn xml_start_tag_scanner_with_rewrites( open_local_name_starts: [], open_local_name_ends: [], open_namespace_uris: [], + open_remove_end_tags: [], rewrite_element_prefixes: element_prefixes, reserved_element_prefixes, rewrite_attribute_prefixes: attribute_prefixes, @@ -1249,7 +1254,10 @@ fn xml_start_tag_scanner_with_rewrites( root_closed: false, current_name_start: 0, current_name_end: 0, + current_tag_start: 0, current_tag_end: 0, + current_attribute_insert: 0, + current_self_closing: false, current_depth: 0, current_namespace_uri: "", current_namespace_id: 0, @@ -1318,20 +1326,24 @@ fn XmlStartTagScanner::consume_end_tag( if !xml_spans_equal(self.xml, name_start, name_end, open_start, open_end) { raise InvalidXml(msg="XML start and end tags do not match") } - let replacement = self.open_name_replacements[self.open_name_replacements.length() - - 1] - match replacement { - Some(value) => self.record_rewrite(name_start, name_end, value) - None => () - } + let stack_index = self.open_name_replacements.length() - 1 + let replacement = self.open_name_replacements[stack_index] + let remove_end_tag = self.open_remove_end_tags[stack_index] match self.capture_start { - Some(capture_start) => - if self.capture_depth == self.depth { - self.record_rewrite(capture_start, tag_end + 1, "") - self.capture_start = None - self.capture_depth = 0 + Some(capture_start) if self.capture_depth == self.depth => { + self.record_rewrite(capture_start, tag_end + 1, "") + self.capture_start = None + self.capture_depth = 0 + } + _ => + if remove_end_tag { + self.record_rewrite(start, tag_end + 1, "") + } else { + match replacement { + Some(value) => self.record_rewrite(name_start, name_end, value) + None => () + } } - None => () } ignore(self.open_name_starts.pop()) ignore(self.open_name_ends.pop()) @@ -1339,6 +1351,7 @@ fn XmlStartTagScanner::consume_end_tag( ignore(self.open_local_name_starts.pop()) ignore(self.open_local_name_ends.pop()) ignore(self.open_namespace_uris.pop()) + ignore(self.open_remove_end_tags.pop()) self.pop_namespace_depth(self.depth) self.depth = self.depth - 1 if self.depth == 0 { @@ -1551,7 +1564,14 @@ pub fn XmlStartTagScanner::next( self.validate_attributes(name_end, tag_end) self.current_name_start = local_name_start self.current_name_end = name_end + self.current_tag_start = start self.current_tag_end = tag_end + self.current_attribute_insert = if self_closing { + before_end - 1 + } else { + before_end + } + self.current_self_closing = self_closing self.current_depth = scope_depth if self.capture_start is None && self.capture_element_local_name != "" && @@ -1602,6 +1622,7 @@ pub fn XmlStartTagScanner::next( self.open_local_name_starts.push(self.current_name_start) self.open_local_name_ends.push(self.current_name_end) self.open_namespace_uris.push(self.current_namespace_uri) + self.open_remove_end_tags.push(false) } return true } @@ -1945,6 +1966,77 @@ fn XmlStartTagScanner::expanded_attribute_names( output } +///| +fn XmlStartTagScanner::current_namespace_declarations( + self : XmlStartTagScanner, +) -> Array[(String, String)] { + let output : Array[(String, String)] = [] + for binding in self.namespace_bindings { + if binding.depth == self.current_depth { + output.push((binding.prefix, binding.uri)) + } + } + output +} + +///| +fn XmlStartTagScanner::remove_current_subtree( + self : XmlStartTagScanner, +) -> Unit raise ParseXmlError { + if self.capture_start is Some(_) { + return + } + if self.current_self_closing { + self.record_rewrite(self.current_tag_start, self.current_tag_end + 1, "") + } else { + self.capture_start = Some(self.current_tag_start) + self.capture_depth = self.current_depth + } +} + +///| +fn XmlStartTagScanner::unwrap_current_element( + self : XmlStartTagScanner, +) -> Unit raise ParseXmlError { + self.record_rewrite(self.current_tag_start, self.current_tag_end + 1, "") + if !self.current_self_closing { + let index = self.open_remove_end_tags.length() - 1 + if index < 0 { + raise InvalidXml(msg="XML unwrap state is invalid") + } + self.open_remove_end_tags[index] = true + } +} + +///| +fn XmlStartTagScanner::insert_current_attributes( + self : XmlStartTagScanner, + attributes : String, +) -> Unit raise ParseXmlError { + if attributes != "" { + self.record_rewrite( + self.current_attribute_insert, + self.current_attribute_insert, + attributes, + ) + } +} + +///| +fn XmlStartTagScanner::finish_rewritten_xml( + self : XmlStartTagScanner, +) -> String raise ParseXmlError { + if self.rewrite_count == 0 { + if self.xml.length() > self.rewrite_output_limit { + raise InvalidXml(msg="XML canonical output limit exceeded") + } + return self.xml.to_owned() + } + self.charge_rewrite_output(self.xml.length() - self.rewrite_cursor) + self.write_rewrite_source(self.rewrite_cursor, self.xml.length()) + self.rewrite_output.to_string() +} + ///| test "start tag scanner resolves namespaces and decodes attributes once" { let xml = diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 5d1a23fc..313807f4 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3523,8 +3523,16 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path=workbook_xml_part_path) } - let workbook_source_xml = decode_source(workbook_bytes) - read_budget.charge_work(workbook_source_xml.length()) + let workbook_raw_xml = decode_source(workbook_bytes) + read_budget.charge_work(workbook_raw_xml.length()) + let workbook_source_xml = project_xlsx_markup_compatibility( + workbook_raw_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + if workbook_source_xml != workbook_raw_xml { + read_budget.charge_work(workbook_source_xml.length()) + } let workbook_xml = canonicalize_xlsx_xml( workbook_source_xml, max_output_chars=limits.max_xml_part_bytes, @@ -3590,8 +3598,16 @@ fn read_zip_archive_core_unchecked( Some(path) => match archive.get(path) { Some(value) => { - let source = decode_source(value) - read_budget.charge_work(source.length()) + let raw_source = decode_source(value) + read_budget.charge_work(raw_source.length()) + let source = project_xlsx_markup_compatibility( + raw_source, + limits.max_xml_part_bytes, + cancelled~, + ) + if source != raw_source { + read_budget.charge_work(source.length()) + } let core_source = match xlsx_core_source_without_foreign(source, "sst", cancelled~) { Some(filtered) => filtered @@ -3637,8 +3653,16 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path~) } - let styles_source_xml = decode_source(value) - read_budget.charge_work(styles_source_xml.length()) + let styles_raw_xml = decode_source(value) + read_budget.charge_work(styles_raw_xml.length()) + let styles_source_xml = project_xlsx_markup_compatibility( + styles_raw_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + if styles_source_xml != styles_raw_xml { + read_budget.charge_work(styles_source_xml.length()) + } let styles_xml = canonicalize_xlsx_xml( styles_source_xml, max_output_chars=limits.max_xml_part_bytes, @@ -3911,8 +3935,16 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path=sheet_path) } - let sheet_source_xml = decode_source(sheet_bytes) - read_budget.charge_work(sheet_source_xml.length()) + let sheet_raw_xml = decode_source(sheet_bytes) + read_budget.charge_work(sheet_raw_xml.length()) + let sheet_source_xml = project_xlsx_markup_compatibility( + sheet_raw_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + if sheet_source_xml != sheet_raw_xml { + read_budget.charge_work(sheet_source_xml.length()) + } let sheet_xml = canonicalize_xlsx_xml( sheet_source_xml, max_output_chars=limits.max_xml_part_bytes, diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 23fb2c10..9e774463 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -124,6 +124,40 @@ fn unicode_part_name_xlsx_fixture() -> Bytes raise { @zip.write(archive) } +///| +fn mce_fallback_xlsx_fixture() -> Bytes raise { + let content_types = + #| + let root_relationships = + #| + let workbook = + #| + let workbook_relationships = + #| + let worksheet = + #|fallback-value + let archive = @zip.Archive::new() + archive.add("[Content_Types].xml", @encoding/utf8.encode(content_types)) + archive.add("_rels/.rels", @encoding/utf8.encode(root_relationships)) + archive.add("xl/workbook.xml", @encoding/utf8.encode(workbook)) + archive.add( + "xl/_rels/workbook.xml.rels", + @encoding/utf8.encode(workbook_relationships), + ) + archive.add("xl/worksheets/sheet1.xml", @encoding/utf8.encode(worksheet)) + @zip.write(archive) +} + +///| +test "workbook and worksheet MCE fallbacks read end to end" { + let workbook = @xlsx.read(mce_fallback_xlsx_fixture()) + debug_inspect(workbook.get_sheet_list(), content="[\"Data\"]") + debug_inspect( + workbook.get_cell_value("Data", "A1"), + content="Some(\"fallback-value\")", + ) +} + ///| test "physical ZIP names follow Unicode logical OPC part names end to end" { let workbook = @xlsx.read(unicode_part_name_xlsx_fixture()) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 748a6752..ffaca78a 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -31,6 +31,33 @@ let drawing_slicer_namespace = "http://schemas.microsoft.com/office/drawing/2010 ///| let markup_compatibility_namespace = "http://schemas.openxmlformats.org/markup-compatibility/2006" +///| +let xlsx_mce_understood_namespaces : Array[String] = [ + transitional_spreadsheet_namespace, strict_spreadsheet_namespace, transitional_relationship_attribute_namespace, + strict_relationship_attribute_namespace, xlsx_extension_namespace_x14, xlsx_extension_namespace_x15, + xlsx_extension_namespace_xm, +] + +///| +/// Produces the effective SpreadsheetML view before namespace filtering or +/// lexical feature parsing. In particular, core content promoted from an MCE +/// Fallback must be selected while its namespace context is still available. +fn project_xlsx_markup_compatibility( + source : StringView, + max_output_chars : Int, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + @ooxml.project_xml_markup_compatibility( + source, + xlsx_mce_understood_namespaces, + max_output_chars~, + cancelled~, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } +} + ///| /// Converts namespace-qualified OOXML element names into the lexical form used /// by the existing bounded feature parsers. The source is fully namespace- and @@ -361,6 +388,31 @@ test "XLSX XML canonicalization handles Strict elements and relationship attribu ) } +///| +test "SpreadsheetML MCE fallback is projected before core filtering" { + let workbook_source = + #| + let workbook_projected = project_xlsx_markup_compatibility( + workbook_source, default_max_xml_part_bytes, + ) + let sheets = parse_workbook_sheets(workbook_projected, 10) + assert_eq(sheets.length(), 1) + assert_eq(sheets[0].name, "Data") + assert_eq(sheets[0].rel_id, "rId1") + let worksheet_source = + #|kept + let worksheet_projected = project_xlsx_markup_compatibility( + worksheet_source, default_max_xml_part_bytes, + ) + let worksheet_full = canonicalize_xlsx_xml(worksheet_projected) + let worksheet_core = canonicalize_xlsx_worksheet_core( + worksheet_projected, worksheet_full, default_max_xml_part_bytes, + ) + assert_true(worksheet_core.contains("")) + assert_true(worksheet_core.contains("kept")) + assert_false(worksheet_core.contains("futureData")) +} + ///| test "XLSX XML canonicalization follows extension namespace identity" { let source = From 240d0fed4649d0eb19b5160ec8cf2f2903fd166e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 23:29:45 +0800 Subject: [PATCH 102/150] fix(xlsx): floor serials for calendar formulas --- xlsx/calc_1904_date_test.mbt | 4 ++++ xlsx/calc_test.mbt | 4 ++++ xlsx/formula_builtins.mbt | 15 +++++++++------ xlsx/formula_eval.mbt | 23 +++++++++++++++++++++-- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/xlsx/calc_1904_date_test.mbt b/xlsx/calc_1904_date_test.mbt index a17db6d1..0297dda2 100644 --- a/xlsx/calc_1904_date_test.mbt +++ b/xlsx/calc_1904_date_test.mbt @@ -28,7 +28,11 @@ test "calculated dates honor the workbook 1904 date system" { ("ISOWEEKNUM(DATE(1904,1,1))", "53"), ("ISOWEEKNUM(DATE(1904,1,2))", "53"), ("ISOWEEKNUM(DATE(1904,1,3))", "53"), + ("ISOWEEKNUM(DATE(1904,1,3)+0.9999999)", "53"), ("ISOWEEKNUM(DATE(1904,1,4))", "1"), + ("YEAR(DATE(1904,12,31)+0.9999999)", "1904"), + ("MONTH(DATE(1904,1,31)+0.9999999)", "1"), + ("DAY(DATE(1904,1,31)+0.9999999)", "31"), ("INT(NOW())=TODAY()", "TRUE"), ("TODAY()=DATE(YEAR(TODAY()),MONTH(TODAY()),DAY(TODAY()))", "TRUE"), ("DATE(1903,12,31)", "#NUM!"), diff --git a/xlsx/calc_test.mbt b/xlsx/calc_test.mbt index ed5c0e58..a918d9eb 100644 --- a/xlsx/calc_test.mbt +++ b/xlsx/calc_test.mbt @@ -3655,6 +3655,10 @@ test "calc date and time functions" { inspect(workbook.calc_cell_value("Sheet1", "G6"), content="9") sheet.set_cell_formula("G7", "ISOWEEKNUM(60.5)") inspect(workbook.calc_cell_value("Sheet1", "G7"), content="9") + sheet.set_cell_formula("G8", "ISOWEEKNUM(59.9999999)") + inspect(workbook.calc_cell_value("Sheet1", "G8"), content="9") + sheet.set_cell_formula("G9", "DAY(31.9999999)") + inspect(workbook.calc_cell_value("Sheet1", "G9"), content="31") sheet.set_cell_formula("H1", "WEEKDAY(DATE(2020,1,1))") inspect(workbook.calc_cell_value("Sheet1", "H1"), content="4") sheet.set_cell_formula("H2", "WEEKDAY(DATE(2020,1,1),2)") diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index a4805cce..76cced62 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -4183,8 +4183,9 @@ fn eval_function( } "YEAR" => if values.length() == 1 { - match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { - Ok((year, _, _, _, _, _)) => Number(Double::from_int(year)) + match + excel_serial_date_parts(values[0], use_1904_dates=ctx.use_1904_dates) { + Ok((year, _, _)) => Number(Double::from_int(year)) Err(err) => err } } else { @@ -4192,8 +4193,9 @@ fn eval_function( } "MONTH" => if values.length() == 1 { - match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { - Ok((_, month, _, _, _, _)) => Number(Double::from_int(month)) + match + excel_serial_date_parts(values[0], use_1904_dates=ctx.use_1904_dates) { + Ok((_, month, _)) => Number(Double::from_int(month)) Err(err) => err } } else { @@ -4201,8 +4203,9 @@ fn eval_function( } "DAY" => if values.length() == 1 { - match excel_serial_parts(values[0], use_1904_dates=ctx.use_1904_dates) { - Ok((_, _, day, _, _, _)) => Number(Double::from_int(day)) + match + excel_serial_date_parts(values[0], use_1904_dates=ctx.use_1904_dates) { + Ok((_, _, day)) => Number(Double::from_int(day)) Err(err) => err } } else { diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index 5c86e610..d7936a37 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -5916,10 +5916,14 @@ fn date_parts_from_serial( serial : Double, use_1904_dates? : Bool = false, ) -> (Int, Int, Int)? { + // Calendar-only formulas use the serial's containing day. Clock rounding is + // intentionally confined to time-bearing conversions: 23:59:59.99 must not + // make YEAR, DAY, WEEKNUM, or ISOWEEKNUM observe the following date. + let date_serial = Double::floor(serial) let parts = if use_1904_dates { - excel_serial_to_parts_1904(serial) + excel_serial_to_parts_1904(date_serial) } else { - excel_serial_to_parts(serial) + excel_serial_to_parts(date_serial) } match parts { Some((year, month, day, _, _, _)) => Some((year, month, day)) @@ -5927,6 +5931,21 @@ fn date_parts_from_serial( } } +///| +fn excel_serial_date_parts( + value : FormulaValue, + use_1904_dates? : Bool = false, +) -> Result[(Int, Int, Int), FormulaValue] { + match value_as_number(value) { + Ok(serial) => + match date_parts_from_serial(serial, use_1904_dates~) { + Some(parts) => Ok(parts) + None => Err(Error(formula_error_num)) + } + Err(err) => Err(err) + } +} + ///| fn is_1900_phantom_date_serial(serial : Double, use_1904_dates : Bool) -> Bool { !use_1904_dates && serial >= 60.0 && serial < 61.0 From 83f72fabba02d09daddf0680bea34a02bf4cb651 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 18 Jul 2026 23:33:32 +0800 Subject: [PATCH 103/150] fix(xlsx): contain TEXT formatter failures --- xlsx/calc_test.mbt | 19 +++++++++++++++++++ xlsx/formula_builtins.mbt | 20 +++++++++++--------- xlsx/value_format.mbt | 10 ++++++++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/xlsx/calc_test.mbt b/xlsx/calc_test.mbt index a918d9eb..ec087ece 100644 --- a/xlsx/calc_test.mbt +++ b/xlsx/calc_test.mbt @@ -7819,3 +7819,22 @@ test "calc text b functions use DBCS semantics without mojibake" { sheet.set_cell_formula("A10", "REPLACEB(C1,5,5,\"!\")") inspect(workbook.calc_cell_value("Sheet1", "A10"), content="你好!") } + +///| +test "TEXT contains formatter failures at the formula boundary" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Sheet1") + let format = "# ?/?" + "%".repeat(155) + sheet.set_cell_formula("A1", "TEXT(0,\"" + format + "\")") + match workbook.calc_cell_typed("Sheet1", "A1") { + Some(String(text)) => assert_true(text.contains("%")) + other => fail("zero TEXT scaling should stay finite, got \{to_repr(other)}") + } + sheet.set_cell_formula("A2", "TEXT(1,\"" + format + "\")") + inspect(workbook.calc_cell_value("Sheet1", "A2"), content="#VALUE!") + match workbook.calc_cell_typed("Sheet1", "A2") { + Some(Error(code)) => inspect(code, content="#VALUE!") + other => + fail("TEXT formatter failure should be an error, got \{to_repr(other)}") + } +} diff --git a/xlsx/formula_builtins.mbt b/xlsx/formula_builtins.mbt index 76cced62..97bf5376 100644 --- a/xlsx/formula_builtins.mbt +++ b/xlsx/formula_builtins.mbt @@ -3351,15 +3351,17 @@ fn eval_function( Err(err) => return err } match value_as_number_text(values[0]) { - Ok(num) => - String( - format_number_code( - format_number(num), - format_text, - Options::new(), - use_1904_format=ctx.use_1904_dates, - ), - ) + Ok(num) => { + let formatted = format_number_code( + format_number(num), + format_text, + Options::new(), + use_1904_format=ctx.use_1904_dates, + ) catch { + _ => return Error(formula_error_value) + } + String(formatted) + } Err(err) => err } } else { diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index 680ba115..e24b37ff 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -590,8 +590,14 @@ fn format_number_pattern( let negative = value < 0.0 let abs_value = if negative { -value } else { value } let (prefix, suffix, info) = parse_number_pattern(pattern) - let scaled = abs_value * - @math.pow(100.0, Double::from_int(info.percent_count)) + // IEEE-754 defines 0 * infinity as NaN. A long but syntactically valid run + // of percent markers can overflow the scale factor, yet formatting numeric + // zero must remain exactly zero rather than manufacturing a formatter error. + let scaled = if abs_value == 0.0 { + 0.0 + } else { + abs_value * @math.pow(100.0, Double::from_int(info.percent_count)) + } if scaled.is_nan() || scaled.is_inf() { raise InvalidXml(msg="number format scale must remain finite") } From 93685dbe5f97f6ac46a2a0dbf96961624f9b8b98 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 00:00:22 +0800 Subject: [PATCH 104/150] test(xlsx): align percent overflow expectations --- xlsx/value_format_test.mbt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/xlsx/value_format_test.mbt b/xlsx/value_format_test.mbt index 84859602..0a629672 100644 --- a/xlsx/value_format_test.mbt +++ b/xlsx/value_format_test.mbt @@ -25,7 +25,7 @@ fn format_text(value : String, fmt : String) -> String { } ///| -test "fraction formatting rejects non-finite percent scaling" { +test "fraction formatting keeps zero finite across percent scaling" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Sheet1")) workbook.set_cell_value("Sheet1", "A1", Numeric(0.0)) @@ -33,6 +33,22 @@ test "fraction formatting rejects non-finite percent scaling" { @xlsx.Style::number_format("# ?/?" + "%".repeat(155)), ) workbook.set_cell_style("Sheet1", "A1", style_id) + guard workbook.get_cell_value("Sheet1", "A1") is Some(formatted) else { + fail("expected formatted numeric zero") + } + assert_true(formatted.has_prefix("0")) + assert_true(formatted.has_suffix("%".repeat(155))) +} + +///| +test "fraction formatting rejects non-finite nonzero percent scaling" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Sheet1")) + workbook.set_cell_value("Sheet1", "A1", Numeric(1.0)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("# ?/?" + "%".repeat(155)), + ) + workbook.set_cell_style("Sheet1", "A1", style_id) let result : Result[String?, Error] = Ok( workbook.get_cell_value("Sheet1", "A1"), ) catch { From 658f9f0c233873040506dc8de38e7db916da986b Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 00:01:48 +0800 Subject: [PATCH 105/150] fix(xlsx): bound shared formula materialization --- xlsx/formula_builtins_stats.mbt | 9 +- xlsx/formula_eval_types.mbt | 1 + xlsx/formula_parse.mbt | 10 +- xlsx/pkg.generated.mbti | 36 +-- xlsx/shared_formula_ranges.mbt | 72 +++--- xlsx/shared_formula_translate.mbt | 290 +++++++++++++++++++++++- xlsx/shared_formula_validation_test.mbt | 177 +++++++++++++++ xlsx/workbook.mbt | 93 ++++++-- xlsx/worksheet.mbt | 241 +++++++++++++++----- 9 files changed, 798 insertions(+), 131 deletions(-) diff --git a/xlsx/formula_builtins_stats.mbt b/xlsx/formula_builtins_stats.mbt index 52a35268..5d9162b4 100644 --- a/xlsx/formula_builtins_stats.mbt +++ b/xlsx/formula_builtins_stats.mbt @@ -1925,8 +1925,13 @@ fn calc_cell_value_internal( let formula = match (formula, cell.formula_type, cell.formula_shared_index) { ("", Some(Shared), Some(shared_index)) => - match sheet.shared_formula_master(shared_index) { - Some(master) => master.translate_to(row, col) + match + sheet.shared_formula_master( + shared_index, + cancelled=ctx.shared_formula_budget.cancelled, + ) { + Some(master) => + ctx.shared_formula_budget.translate(master, row, col) None => raise InvalidXml(msg="shared formula follower has no master") } diff --git a/xlsx/formula_eval_types.mbt b/xlsx/formula_eval_types.mbt index 9762e068..73487393 100644 --- a/xlsx/formula_eval_types.mbt +++ b/xlsx/formula_eval_types.mbt @@ -169,6 +169,7 @@ priv struct CalcContext { mut current_sheet : String mut current_ref : String use_1904_dates : Bool + shared_formula_budget : SharedFormulaMaterializationBudget } ///| diff --git a/xlsx/formula_parse.mbt b/xlsx/formula_parse.mbt index e0ab9d7d..0c93b84e 100644 --- a/xlsx/formula_parse.mbt +++ b/xlsx/formula_parse.mbt @@ -1,5 +1,9 @@ ///| -fn CalcContext::new(use_1904_dates? : Bool = false) -> CalcContext { +fn CalcContext::new( + use_1904_dates? : Bool = false, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, +) -> CalcContext { { visiting: Map([]), depth: 0, @@ -7,6 +11,10 @@ fn CalcContext::new(use_1904_dates? : Bool = false) -> CalcContext { current_sheet: "", current_ref: "", use_1904_dates, + shared_formula_budget: SharedFormulaMaterializationBudget::new( + formula_limits, + cancelled~, + ), } } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index c471983e..59d9a59c 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1355,6 +1355,16 @@ pub struct ShapeLine { pub fn ShapeLine::to_repr(Self) -> @debug.Repr pub fn ShapeLine::with_values(color? : String, width? : Double) -> Self +pub struct SharedFormulaLimits { + // private fields +} +pub fn SharedFormulaLimits::max_input_chars(Self) -> Int +pub fn SharedFormulaLimits::max_output_chars(Self) -> Int +pub fn SharedFormulaLimits::max_total_output_chars(Self) -> Int +pub fn SharedFormulaLimits::max_work_units(Self) -> Int +pub fn SharedFormulaLimits::new() -> Self +pub fn SharedFormulaLimits::with_values(max_input_chars? : Int, max_output_chars? : Int, max_total_output_chars? : Int, max_work_units? : Int) -> Self raise XlsxError + pub struct SharedFormulaMaster { // private fields } @@ -1804,10 +1814,10 @@ pub fn Workbook::add_vml_drawing_xml(Self, StringView, String) -> Unit raise Xls pub fn Workbook::app_properties(Self) -> AppProperties pub fn Workbook::app_props(Self) -> AppProperties pub fn Workbook::auto_filter(Self, StringView, String, ArrayView[AutoFilterOption]) -> Unit raise XlsxError -pub fn Workbook::calc_cell_typed(Self, StringView, StringView) -> CellValue? raise XlsxError -pub fn Workbook::calc_cell_typed_rc(Self, StringView, Int, Int) -> CellValue? raise XlsxError -pub fn Workbook::calc_cell_value(Self, StringView, StringView, raw? : Bool, options? : Options) -> String raise XlsxError -pub fn Workbook::calc_cell_value_rc(Self, StringView, Int, Int, raw? : Bool, options? : Options) -> String raise XlsxError +pub fn Workbook::calc_cell_typed(Self, StringView, StringView, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> CellValue? raise XlsxError +pub fn Workbook::calc_cell_typed_rc(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> CellValue? raise XlsxError +pub fn Workbook::calc_cell_value(Self, StringView, StringView, raw? : Bool, options? : Options, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> String raise XlsxError +pub fn Workbook::calc_cell_value_rc(Self, StringView, Int, Int, raw? : Bool, options? : Options, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> String raise XlsxError pub fn Workbook::charset_transcoder(Self, (String, Bytes) -> String raise XlsxError) -> Self pub fn Workbook::chart_sheet(Self, StringView) -> ChartSheet? pub fn Workbook::chart_sheets(Self) -> ArrayView[ChartSheet] @@ -1831,8 +1841,8 @@ pub fn Workbook::delete_sheet(Self, StringView) -> Unit raise XlsxError pub fn Workbook::delete_slicer(Self, StringView) -> Unit raise XlsxError pub fn Workbook::delete_table(Self, StringView) -> Unit raise XlsxError pub fn Workbook::doc_properties(Self) -> CoreProperties -pub fn Workbook::duplicate_row(Self, StringView, Int) -> Unit raise XlsxError -pub fn Workbook::duplicate_row_to(Self, StringView, Int, Int) -> Unit raise XlsxError +pub fn Workbook::duplicate_row(Self, StringView, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError +pub fn Workbook::duplicate_row_to(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError pub fn Workbook::get_active_sheet_index(Self) -> Int pub fn Workbook::get_app_props(Self) -> AppProperties pub fn Workbook::get_auto_filter(Self, StringView) -> AutoFilter? raise XlsxError @@ -1902,9 +1912,9 @@ pub fn Workbook::get_style(Self, Int) -> Style raise XlsxError pub fn Workbook::get_tables(Self, StringView) -> Array[Table] raise XlsxError pub fn Workbook::get_workbook_props(Self) -> WorkbookPropsOptions pub fn Workbook::group_sheets(Self, Array[String]) -> Unit raise XlsxError -pub fn Workbook::insert_cols(Self, StringView, Int, Int) -> Unit raise XlsxError +pub fn Workbook::insert_cols(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError pub fn Workbook::insert_page_break(Self, StringView, StringView) -> Unit raise XlsxError -pub fn Workbook::insert_rows(Self, StringView, Int, Int) -> Unit raise XlsxError +pub fn Workbook::insert_rows(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError pub fn Workbook::merge_cell(Self, StringView, StringView, StringView) -> Unit raise XlsxError pub fn Workbook::merge_cells(Self, StringView, String) -> Unit raise XlsxError pub fn Workbook::move_sheet(Self, StringView, StringView) -> Unit raise XlsxError @@ -1917,11 +1927,11 @@ pub fn Workbook::protect_sheet(Self, StringView, SheetProtectionOptions) -> Unit pub fn Workbook::protect_workbook(Self, options? : WorkbookProtectionOptions) -> Unit raise XlsxError pub async fn[R : @io.Reader] Workbook::read_zip_reader(Self, R, password? : String, options? : Options, limits? : ReadLimits, transcoder? : (String, Bytes) -> String raise XlsxError) -> Self pub fn Workbook::remove_cell_hyperlink(Self, StringView, StringView) -> Unit raise XlsxError -pub fn Workbook::remove_col(Self, StringView, Int) -> Unit raise XlsxError -pub fn Workbook::remove_cols(Self, StringView, Int, Int) -> Unit raise XlsxError +pub fn Workbook::remove_col(Self, StringView, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError +pub fn Workbook::remove_cols(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError pub fn Workbook::remove_page_break(Self, StringView, StringView) -> Unit raise XlsxError -pub fn Workbook::remove_row(Self, StringView, Int) -> Unit raise XlsxError -pub fn Workbook::remove_rows(Self, StringView, Int, Int) -> Unit raise XlsxError +pub fn Workbook::remove_row(Self, StringView, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError +pub fn Workbook::remove_rows(Self, StringView, Int, Int, formula_limits? : SharedFormulaLimits, cancelled? : () -> Bool) -> Unit raise XlsxError pub fn Workbook::row_outline_level(Self, StringView, Int) -> Int raise XlsxError pub fn Workbook::row_stream(Self, StringView) -> RowStream raise XlsxError pub fn Workbook::row_visible(Self, StringView, Int) -> Bool raise XlsxError @@ -2186,7 +2196,7 @@ pub fn Worksheet::set_row_style(Self, Int, Int) -> Unit raise XlsxError pub fn Worksheet::set_row_visible(Self, Int, Bool) -> Unit raise XlsxError pub fn Worksheet::set_sheet_background(Self, Bytes, String) -> Unit raise XlsxError pub fn Worksheet::shapes(Self) -> ArrayView[Shape] -pub fn Worksheet::shared_formula_master(Self, UInt) -> SharedFormulaMaster? raise XlsxError +pub fn Worksheet::shared_formula_master(Self, UInt, cancelled? : () -> Bool) -> SharedFormulaMaster? raise XlsxError pub fn Worksheet::shared_formula_masters(Self) -> Map[UInt, SharedFormulaMaster] raise XlsxError pub fn Worksheet::sheet_background(Self) -> SheetBackground? pub fn Worksheet::sheet_protection(Self) -> SheetProtection? diff --git a/xlsx/shared_formula_ranges.mbt b/xlsx/shared_formula_ranges.mbt index 4fbb6cf7..a6ad0caf 100644 --- a/xlsx/shared_formula_ranges.mbt +++ b/xlsx/shared_formula_ranges.mbt @@ -6,6 +6,17 @@ priv struct SharedFormulaRangeEvent { delta : Int } +///| +fn shared_formula_range_checkpoint( + budget : ReadBudget?, + cancelled : () -> Bool, +) -> Unit raise XlsxError { + match budget { + Some(value) => value.checkpoint() + None => if cancelled() { raise ReadCancelled } + } +} + ///| fn compare_shared_formula_range_events( left : SharedFormulaRangeEvent, @@ -32,27 +43,24 @@ fn compare_shared_formula_range_events( fn sort_shared_formula_range_events( events : Array[SharedFormulaRangeEvent], budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { if events.length() < 2 { return } + shared_formula_range_checkpoint(budget, cancelled) match budget { - Some(value) => { - // Account for and observe cancellation before allocating/copying the - // O(events) merge scratch buffer. - value.checkpoint() + Some(value) => + // Account before allocating/copying the O(events) merge scratch buffer. value.charge_work(events.length()) - } None => () } let scratch = events.copy() let mut width = 1 while width < events.length() { + shared_formula_range_checkpoint(budget, cancelled) match budget { - Some(value) => { - value.checkpoint() - value.charge_work(events.length()) - } + Some(value) => value.charge_work(events.length()) None => () } let mut run = 0 @@ -72,10 +80,7 @@ fn sort_shared_formula_range_events( let mut right = middle while left < middle || right < end { if (output & 4095) == 0 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } if right >= end || ( @@ -93,19 +98,14 @@ fn sort_shared_formula_range_events( } run = end } + shared_formula_range_checkpoint(budget, cancelled) match budget { - Some(value) => { - value.checkpoint() - value.charge_work(events.length()) - } + Some(value) => value.charge_work(events.length()) None => () } for index in 0.. value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } events[index] = scratch[index] } @@ -115,10 +115,7 @@ fn sort_shared_formula_range_events( width * 2 } } - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } ///| @@ -242,18 +239,16 @@ fn shared_formula_range_tree_maximum( fn validate_shared_formula_master_ranges( masters : Map[UInt, SharedFormulaMaster], budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { if masters.length() < 2 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) return } let events : Array[SharedFormulaRangeEvent] = [] + shared_formula_range_checkpoint(budget, cancelled) match budget { Some(value) => { - value.checkpoint() value.charge_work(masters.length()) value.charge_work(masters.length()) } @@ -262,10 +257,7 @@ fn validate_shared_formula_master_ranges( let mut master_index = 0 for _, master in masters { if (master_index & 4095) == 0 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } events.push({ row: master.row_lo, @@ -281,7 +273,7 @@ fn validate_shared_formula_master_ranges( }) master_index += 1 } - sort_shared_formula_range_events(events, budget?) + sort_shared_formula_range_events(events, budget?, cancelled~) let tree_size = cell_ref_max_cols * 4 + 8 let maximums = Array::make(tree_size, 0) let pending_additions = Array::make(tree_size, 0) @@ -299,10 +291,7 @@ fn validate_shared_formula_master_ranges( let mut event_index = 0 for event in events { if (event_index & 4095) == 0 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } if event.delta > 0 && shared_formula_range_tree_maximum( @@ -329,10 +318,7 @@ fn validate_shared_formula_master_ranges( ) event_index += 1 } - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_range_checkpoint(budget, cancelled) } ///| diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index 7e025029..76b27036 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -12,6 +12,279 @@ pub struct SharedFormulaMaster { priv column_hi : Int } +///| +let default_max_shared_formula_input_chars : Int = 64 * 1024 + +///| +let default_max_shared_formula_output_chars : Int = 256 * 1024 + +///| +let default_max_shared_formula_total_output_chars : Int = 64 * 1024 * 1024 + +///| +let default_max_shared_formula_work_units : Int = 256 * 1024 * 1024 + +///| +/// Resource policy for resolving shared-formula followers. +/// +/// `max_input_chars` and `max_output_chars` apply to one translated formula. +/// `max_total_output_chars` and `max_work_units` are cumulative across one +/// structural edit or one top-level calculation, so a compact shared master +/// cannot fan out into an unbounded amount of text or translation work. +pub struct SharedFormulaLimits { + priv max_input_chars : Int + priv max_output_chars : Int + priv max_total_output_chars : Int + priv max_work_units : Int +} + +///| +/// Returns the production shared-formula policy: 64 Ki characters per master, +/// 256 Ki characters per translated follower, 64 Mi characters of cumulative +/// output, and 256 Mi cumulative translation work units per operation. +pub fn SharedFormulaLimits::new() -> SharedFormulaLimits { + { + max_input_chars: default_max_shared_formula_input_chars, + max_output_chars: default_max_shared_formula_output_chars, + max_total_output_chars: default_max_shared_formula_total_output_chars, + max_work_units: default_max_shared_formula_work_units, + } +} + +///| +/// Builds a validated shared-formula policy. Every limit must be positive and +/// the per-formula output ceiling cannot exceed the cumulative output ceiling. +pub fn SharedFormulaLimits::with_values( + max_input_chars? : Int = default_max_shared_formula_input_chars, + max_output_chars? : Int = default_max_shared_formula_output_chars, + max_total_output_chars? : Int = default_max_shared_formula_total_output_chars, + max_work_units? : Int = default_max_shared_formula_work_units, +) -> SharedFormulaLimits raise XlsxError { + if max_input_chars <= 0 || + max_output_chars <= 0 || + max_total_output_chars <= 0 || + max_work_units <= 0 { + raise InvalidOptions(msg="shared formula limits must be positive") + } + if max_output_chars > max_total_output_chars { + raise InvalidOptions( + msg="shared formula output limit exceeds cumulative output limit", + ) + } + { max_input_chars, max_output_chars, max_total_output_chars, max_work_units } +} + +///| +/// Returns the maximum accepted character count for one shared master. +pub fn SharedFormulaLimits::max_input_chars(self : SharedFormulaLimits) -> Int { + self.max_input_chars +} + +///| +/// Returns the maximum output character count for one translated follower. +pub fn SharedFormulaLimits::max_output_chars(self : SharedFormulaLimits) -> Int { + self.max_output_chars +} + +///| +/// Returns the cumulative translated-output ceiling for one operation. +pub fn SharedFormulaLimits::max_total_output_chars( + self : SharedFormulaLimits, +) -> Int { + self.max_total_output_chars +} + +///| +/// Returns the cumulative translation-work ceiling for one operation. +pub fn SharedFormulaLimits::max_work_units(self : SharedFormulaLimits) -> Int { + self.max_work_units +} + +///| +/// One cumulative budget is shared by every follower resolved during a +/// structural edit or top-level calculation. +priv struct SharedFormulaMaterializationBudget { + limits : SharedFormulaLimits + cancelled : () -> Bool + mut output_chars : Int + mut work_units : Int +} + +///| +fn SharedFormulaMaterializationBudget::new( + limits : SharedFormulaLimits, + cancelled? : () -> Bool = () => false, +) -> SharedFormulaMaterializationBudget { + { limits, cancelled, output_chars: 0, work_units: 0 } +} + +///| +fn SharedFormulaMaterializationBudget::checkpoint( + self : SharedFormulaMaterializationBudget, +) -> Unit raise XlsxError { + if (self.cancelled)() { + raise ReadCancelled + } +} + +///| +fn shared_formula_projected_total( + current : Int, + chars_per_formula : Int, + formula_count : Int, + limit : Int, +) -> (Int, Bool) { + if chars_per_formula <= 0 || formula_count <= 0 { + return (current, false) + } + let remaining = limit - current + if formula_count > remaining / chars_per_formula { + return (if limit < 0x7fffffff { limit + 1 } else { limit }, true) + } + (current + chars_per_formula * formula_count, false) +} + +///| +/// Rejects an oversized fan-out before allocating any translated formula. +/// Counting by master makes the check explicitly overflow-safe for +/// `follower_count * formula_size`, while actual translation still accounts +/// for reference expansion and scanner work precisely. +fn SharedFormulaMaterializationBudget::preflight( + self : SharedFormulaMaterializationBudget, + formula_counts : Map[UInt, Int], + masters : Map[UInt, SharedFormulaMaster], +) -> Unit raise XlsxError { + self.checkpoint() + let mut projected_output = self.output_chars + let mut projected_work = self.work_units + for shared_index, formula_count in formula_counts { + self.checkpoint() + let master = match masters.get(shared_index) { + Some(value) => value + None => raise InvalidXml(msg="shared formula cell has no master") + } + let input_chars = master.formula.length() + if input_chars > self.limits.max_input_chars { + raise ResourceLimitExceeded( + kind="shared_formula_input_chars", + limit=self.limits.max_input_chars, + actual=input_chars, + ) + } + let (next_output, output_exceeded) = shared_formula_projected_total( + projected_output, + input_chars, + formula_count, + self.limits.max_total_output_chars, + ) + if output_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_total_output_chars", + limit=self.limits.max_total_output_chars, + actual=next_output, + ) + } + projected_output = next_output + let (next_work, work_exceeded) = shared_formula_projected_total( + projected_work, + input_chars, + formula_count, + self.limits.max_work_units, + ) + if work_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_work_units", + limit=self.limits.max_work_units, + actual=next_work, + ) + } + projected_work = next_work + } + self.checkpoint() +} + +///| +fn SharedFormulaMaterializationBudget::translate( + self : SharedFormulaMaterializationBudget, + master : SharedFormulaMaster, + row : Int, + column : Int, +) -> String raise XlsxError { + self.checkpoint() + let input_chars = master.formula.length() + if input_chars > self.limits.max_input_chars { + raise ResourceLimitExceeded( + kind="shared_formula_input_chars", + limit=self.limits.max_input_chars, + actual=input_chars, + ) + } + let (projected_output, projected_output_exceeded) = shared_formula_projected_total( + self.output_chars, + input_chars, + 1, + self.limits.max_total_output_chars, + ) + if projected_output_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_total_output_chars", + limit=self.limits.max_total_output_chars, + actual=projected_output, + ) + } + let (projected_work, projected_work_exceeded) = shared_formula_projected_total( + self.work_units, + input_chars, + 1, + self.limits.max_work_units, + ) + if projected_work_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_work_units", + limit=self.limits.max_work_units, + actual=projected_work, + ) + } + let remaining_output = self.limits.max_total_output_chars - self.output_chars + let remaining_work = self.limits.max_work_units - self.work_units + let (translated, work) = master.translate_to_limited( + row, + column, + maximum_input_chars=self.limits.max_input_chars, + maximum_output_chars=self.limits.max_output_chars.min(remaining_output), + maximum_work_units=remaining_work, + cancelled=self.cancelled, + ) + let (output_chars, output_exceeded) = read_budget_actual( + self.output_chars, + translated.length(), + self.limits.max_total_output_chars, + ) + if output_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_total_output_chars", + limit=self.limits.max_total_output_chars, + actual=output_chars, + ) + } + let (work_units, work_exceeded) = read_budget_actual( + self.work_units, + work, + self.limits.max_work_units, + ) + if work_exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_work_units", + limit=self.limits.max_work_units, + actual=work_units, + ) + } + self.output_chars = output_chars + self.work_units = work_units + self.checkpoint() + translated +} + ///| priv struct FormulaCellToken { end : Int @@ -813,13 +1086,14 @@ fn translate_shared_formula( column_delta : Int, row_delta : Int, ) -> String { + let limits = SharedFormulaLimits::new() let (translated, _) = try! translate_shared_formula_limited( formula, column_delta, row_delta, - 0x7fffffff, - 0x7fffffff, - 0x7fffffff, + limits.max_input_chars, + limits.max_output_chars, + limits.max_work_units, () => false, ) translated @@ -868,18 +1142,20 @@ pub fn SharedFormulaMaster::translate_to_limited( ///| /// Resolves the formula text represented by this shared master at a follower /// coordinate. Coordinates outside the master's declared shared range are -/// rejected as malformed OOXML. +/// rejected as malformed OOXML. Production per-formula limits are applied; +/// use `translate_to_limited` when a stricter policy or cancellation is needed. pub fn SharedFormulaMaster::translate_to( self : SharedFormulaMaster, row : Int, column : Int, ) -> String raise XlsxError { + let limits = SharedFormulaLimits::new() let (translated, _) = self.translate_to_limited( row, column, - maximum_input_chars=0x7fffffff, - maximum_output_chars=0x7fffffff, - maximum_work_units=0x7fffffff, + maximum_input_chars=limits.max_input_chars, + maximum_output_chars=limits.max_output_chars, + maximum_work_units=limits.max_work_units, ) translated } diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt index 978aa7b9..73e4dc12 100644 --- a/xlsx/shared_formula_validation_test.mbt +++ b/xlsx/shared_formula_validation_test.mbt @@ -223,6 +223,183 @@ test "row and column edits materialize shared formulas before moving cells" { assert_structural_edit_deshared(duplicate_row, ["C1", "C2", "C3", "C4"]) } +///| +test "shared formula limits reject aggregate structural fan-out before mutation" { + let workbook = shared_formula_edit_fixture() + let limits = @xlsx.SharedFormulaLimits::with_values( + max_input_chars=64, + max_output_chars=14, + max_total_output_chars=14, + max_work_units=1024, + ) + let result : Result[Unit, Error] = Ok( + workbook.insert_rows("Data", 1, 1, formula_limits=limits), + ) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + assert_eq(kind, "shared_formula_total_output_chars") + assert_eq(limit, 14) + assert_eq(actual, 15) + } + Err(error) => fail("unexpected shared-formula fan-out error: \{error}") + Ok(_) => fail("expected aggregate shared-formula fan-out rejection") + } + guard workbook.sheet("Data") is Some(sheet) else { + fail("missing unchanged shared-formula worksheet") + } + assert_eq(sheet.shared_formula_masters().length(), 1) + assert_true(sheet.get_cell_formula_info_rc(1, 3) is Some(_)) + assert_true(sheet.get_cell_formula_info_rc(3, 3) is Some(_)) + assert_true(sheet.get_cell_formula_info_rc(4, 3) is None) + + let work_limited = shared_formula_edit_fixture() + let work_limits = @xlsx.SharedFormulaLimits::with_values( + max_input_chars=64, + max_output_chars=64, + max_total_output_chars=64, + max_work_units=14, + ) + try work_limited.insert_cols("Data", 1, 1, formula_limits=work_limits) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "shared_formula_work_units") + assert_eq(limit, 14) + assert_eq(actual, 15) + } + _ => fail("unexpected aggregate shared-formula work error") + } noraise { + _ => fail("expected aggregate shared-formula work rejection") + } +} + +///| +test "default shared formula limits stop the reported large fan-out" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + let formula = "1+".repeat(4094) + "A1" + sheet.set_cell_formula_opts( + "A1", + formula, + opts=@xlsx.FormulaOpts::shared("A1:A8200"), + ) + let result : Result[Unit, Error] = Ok(workbook.insert_rows("Data", 1, 1)) catch { + error => Err(error) + } + match result { + Err(@xlsx.ResourceLimitExceeded(kind~, limit~, actual~)) => { + assert_eq(kind, "shared_formula_total_output_chars") + assert_eq( + limit, + @xlsx.SharedFormulaLimits::new().max_total_output_chars(), + ) + assert_true(actual > limit) + } + Err(error) => fail("unexpected default fan-out error: \{error}") + Ok(_) => fail("expected default shared-formula fan-out rejection") + } + assert_eq(sheet.shared_formula_masters().length(), 1) + assert_true(sheet.get_cell_formula_info_rc(1, 1) is Some(_)) + assert_true(sheet.get_cell_formula_info_rc(8200, 1) is Some(_)) + assert_true(sheet.get_cell_formula_info_rc(8201, 1) is None) +} + +///| +test "shared formula structural edits are cancellable and atomic" { + let workbook = shared_formula_edit_fixture() + let checks = [0] + let result : Result[Unit, Error] = Ok( + workbook.insert_rows("Data", 1, 1, cancelled=() => { + checks[0] += 1 + true + }), + ) catch { + error => Err(error) + } + assert_eq(checks[0], 1) + assert_true(result is Err(@xlsx.ReadCancelled)) + guard workbook.sheet("Data") is Some(sheet) else { + fail("missing cancelled shared-formula worksheet") + } + assert_eq(sheet.shared_formula_masters().length(), 1) + assert_true(sheet.get_cell_formula_info_rc(1, 3) is Some(_)) + assert_true(sheet.get_cell_formula_info_rc(4, 3) is None) +} + +///| +test "row removal does not translate shared formulas it discards" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "A1", + "B1+AN_OVERSIZED_LITERAL", + opts=@xlsx.FormulaOpts::shared("A1:A3"), + ) + let tiny = @xlsx.SharedFormulaLimits::with_values( + max_input_chars=1, + max_output_chars=1, + max_total_output_chars=1, + max_work_units=1, + ) + workbook.remove_rows("Data", 1, 3, formula_limits=tiny) + let (max_row, max_col) = sheet.used_bounds_limited(maximum_stored_cells=1) + assert_eq(max_row, 0) + assert_eq(max_col, 0) + assert_eq(sheet.shared_formula_masters().length(), 0) + + let column_workbook = @xlsx.Workbook::new() + let column_sheet = column_workbook.add_sheet("Data") + column_sheet.set_cell_formula_opts( + "A1", + "A2+AN_OVERSIZED_LITERAL", + opts=@xlsx.FormulaOpts::shared("A1:C1"), + ) + column_workbook.remove_cols("Data", 1, 3, formula_limits=tiny) + let (column_max_row, column_max_col) = column_sheet.used_bounds_limited( + maximum_stored_cells=1, + ) + assert_eq(column_max_row, 0) + assert_eq(column_max_col, 0) + assert_eq(column_sheet.shared_formula_masters().length(), 0) +} + +///| +test "calculation shares formula output and cancellation budgets" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_value("A1", Numeric(1.0)) + sheet.set_cell_value("A2", Numeric(2.0)) + sheet.set_cell_value("A3", Numeric(3.0)) + sheet.set_cell_formula_opts( + "B1", + "A1", + opts=@xlsx.FormulaOpts::shared("B1:B3"), + ) + sheet.set_cell_formula("C1", "SUM(B1:B3)") + let tiny = @xlsx.SharedFormulaLimits::with_values( + max_input_chars=16, + max_output_chars=3, + max_total_output_chars=3, + max_work_units=1024, + ) + try workbook.calc_cell_value("Data", "C1", formula_limits=tiny) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "shared_formula_total_output_chars") + assert_eq(limit, 3) + assert_eq(actual, 4) + } + _ => fail("unexpected calculated fan-out error") + } noraise { + _ => fail("expected aggregate calculation fan-out rejection") + } + try workbook.calc_cell_value("Data", "B2", cancelled=() => true) catch { + ReadCancelled => () + _ => fail("unexpected calculated cancellation error") + } noraise { + _ => fail("expected calculated shared-formula cancellation") + } +} + ///| test "XLSX read rejects overlapping shared-formula master ranges" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index cd578ff6..5d9fdfc1 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1913,12 +1913,17 @@ pub fn Workbook::get_cell_value_styled_from_worksheet_rc( } ///| +/// Calculates and formats one cell. Shared-formula followers resolved during +/// the calculation share `formula_limits`; `cancelled` is polled while their +/// masters are validated and translated. pub fn Workbook::calc_cell_value( self : Workbook, sheet_name : StringView, reference : StringView, raw? : Bool = false, options? : Options, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { check_sheet_name(sheet_name) let resolved_options = match options { @@ -1931,7 +1936,11 @@ pub fn Workbook::calc_cell_value( } let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) - let ctx = CalcContext::new(use_1904_dates=self.uses_1904_date_system()) + let ctx = CalcContext::new( + use_1904_dates=self.uses_1904_date_system(), + formula_limits~, + cancelled~, + ) let value = resolve_cell_value(self, sheet.name(), canonical, ctx) let (value_type, raw_value) = formula_value_to_cell(value) if raw || resolved_options.raw_cell_value { @@ -1964,12 +1973,28 @@ pub fn Workbook::calc_cell_value_rc( col : Int, raw? : Bool = false, options? : Options, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let reference = cell_ref_from(row, col) match options { Some(value) => - self.calc_cell_value(sheet_name, reference, raw~, options=value) - None => self.calc_cell_value(sheet_name, reference, raw~) + self.calc_cell_value( + sheet_name, + reference, + raw~, + options=value, + formula_limits~, + cancelled~, + ) + None => + self.calc_cell_value( + sheet_name, + reference, + raw~, + formula_limits~, + cancelled~, + ) } } @@ -1982,10 +2007,14 @@ pub fn Workbook::calc_cell_value_rc( /// formula error surfaces as `Some(Error("#…"))` (Excel stores those); only /// structural problems (missing sheet, invalid reference) raise `XlsxError`. /// Like `calc_cell_value`, this recomputes on every call with no memoization. +/// Shared-formula resolution uses the cumulative `formula_limits` policy and +/// polls `cancelled` during validation and translation. pub fn Workbook::calc_cell_typed( self : Workbook, sheet_name : StringView, reference : StringView, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> CellValue? raise XlsxError { check_sheet_name(sheet_name) let sheet = match self.sheet(sheet_name) { @@ -1994,7 +2023,11 @@ pub fn Workbook::calc_cell_typed( } let (row, col) = cell_ref_to_rc(reference) let canonical = cell_ref_from(row, col) - let ctx = CalcContext::new(use_1904_dates=self.uses_1904_date_system()) + let ctx = CalcContext::new( + use_1904_dates=self.uses_1904_date_system(), + formula_limits~, + cancelled~, + ) formula_value_to_cell_value( resolve_cell_value(self, sheet.name(), canonical, ctx), ) @@ -2006,8 +2039,15 @@ pub fn Workbook::calc_cell_typed_rc( sheet_name : StringView, row : Int, col : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> CellValue? raise XlsxError { - self.calc_cell_typed(sheet_name, cell_ref_from(row, col)) + self.calc_cell_typed( + sheet_name, + cell_ref_from(row, col), + formula_limits~, + cancelled~, + ) } ///| @@ -3869,6 +3909,9 @@ pub fn Workbook::get_col_style( /// page breaks. Defined names referring to the sheet are adjusted to follow the /// shift. /// +/// Shared formulas are materialized transactionally under one cumulative +/// `formula_limits` budget; `cancelled` is polled before and during fan-out. +/// /// Raises `XlsxError` if the sheet name is invalid or no matching sheet exists, /// `StreamModeConflict` if the sheet is in stream-writer mode, if `row` is less /// than 1, or if `count` is not positive. @@ -3877,6 +3920,8 @@ pub fn Workbook::insert_rows( sheet_name : StringView, row : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) if count > cell_ref_max_rows { @@ -3896,7 +3941,7 @@ pub fn Workbook::insert_rows( ) -> String? raise XlsxError { adjust_ref_after_row_insert(ref_text, row, count) }) - sheet.insert_rows(row, count) + sheet.insert_rows(row, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -3905,6 +3950,8 @@ pub fn Workbook::insert_rows( /// shifting the rows below up and adjusting row dimensions, merged ranges, /// hyperlinks, auto filter, tables, sparklines, images, charts, page breaks, /// and defined names accordingly. +/// Shared formulas that survive the removal are materialized under one +/// cumulative `formula_limits` budget; removed cells are never translated. /// /// Raises `XlsxError` if the sheet name is invalid or no matching sheet exists, /// `StreamModeConflict` if the sheet is in stream-writer mode, if `row` is less @@ -3914,6 +3961,8 @@ pub fn Workbook::remove_rows( sheet_name : StringView, row : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) // Stage the defined-name adjustment before mutating the sheet, then commit it @@ -3924,7 +3973,7 @@ pub fn Workbook::remove_rows( ) -> String? raise XlsxError { adjust_ref_after_row_remove(ref_text, row, count) }) - sheet.remove_rows(row, count) + sheet.remove_rows(row, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -3934,8 +3983,10 @@ pub fn Workbook::remove_row( self : Workbook, sheet_name : StringView, row : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { - self.remove_rows(sheet_name, row, 1) + self.remove_rows(sheet_name, row, 1, formula_limits~, cancelled~) } ///| @@ -3943,6 +3994,8 @@ pub fn Workbook::remove_row( /// shifting existing columns right along with their cells and associated /// features. Defined names referring to the sheet are adjusted to follow the /// shift. +/// Shared-formula materialization is bounded by `formula_limits` and polls +/// `cancelled` before and during fan-out. /// /// Raises `XlsxError` if the sheet name is invalid or no matching sheet exists, /// `StreamModeConflict` if the sheet is in stream-writer mode, if `col` is less @@ -3952,6 +4005,8 @@ pub fn Workbook::insert_cols( sheet_name : StringView, col : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) if count > cell_ref_max_cols { @@ -3967,7 +4022,7 @@ pub fn Workbook::insert_cols( ) -> String? raise XlsxError { adjust_ref_after_col_insert(ref_text, col, count) }) - sheet.insert_cols(col, count) + sheet.insert_cols(col, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -3975,6 +4030,8 @@ pub fn Workbook::insert_cols( /// Removes `count` columns starting at column `col` (1-based) on `sheet_name`, /// shifting the columns to its right leftward and adjusting associated features /// and defined names accordingly. +/// Shared formulas that survive the removal are bounded by `formula_limits`; +/// removed cells are never translated, and `cancelled` is polled during work. /// /// Raises `XlsxError` if the sheet name is invalid or no matching sheet exists, /// `StreamModeConflict` if the sheet is in stream-writer mode, if `col` is less @@ -3984,6 +4041,8 @@ pub fn Workbook::remove_cols( sheet_name : StringView, col : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) // Stage names before mutating the sheet (see remove_rows). @@ -3992,7 +4051,7 @@ pub fn Workbook::remove_cols( ) -> String? raise XlsxError { adjust_ref_after_col_remove(ref_text, col, count) }) - sheet.remove_cols(col, count) + sheet.remove_cols(col, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -4002,8 +4061,10 @@ pub fn Workbook::remove_col( self : Workbook, sheet_name : StringView, col : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { - self.remove_cols(sheet_name, col, 1) + self.remove_cols(sheet_name, col, 1, formula_limits~, cancelled~) } ///| @@ -4018,13 +4079,15 @@ pub fn Workbook::duplicate_row( self : Workbook, sheet_name : StringView, row : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { // Bound `row` before forming `row + 1`, which would otherwise wrap Int for // `row == Int::max_value` (duplicate_row_to re-validates both rows too). if row < 1 || row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{row}") } - self.duplicate_row_to(sheet_name, row, row + 1) + self.duplicate_row_to(sheet_name, row, row + 1, formula_limits~, cancelled~) } ///| @@ -4035,6 +4098,8 @@ pub fn Workbook::duplicate_row( /// ranges scoped to the source row are also duplicated onto `target_row`, though /// merged-range duplication is skipped if `target_row` falls inside an existing /// merge. Defined names are adjusted for the insert. +/// Source-row copying and the nested insertion share one `formula_limits` +/// budget and cancellation callback. /// /// Raises `XlsxError` if the sheet name is invalid or no matching sheet exists, /// `StreamModeConflict` if the sheet is in stream-writer mode, if `row` is less @@ -4044,6 +4109,8 @@ pub fn Workbook::duplicate_row_to( sheet_name : StringView, row : Int, target_row : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) // Stage the defined-name shift (which raises if a name would leave the grid) @@ -4054,6 +4121,6 @@ pub fn Workbook::duplicate_row_to( ) -> String? raise XlsxError { adjust_ref_after_row_insert(ref_text, target_row, 1) }) - sheet.duplicate_row_to(row, target_row) + sheet.duplicate_row_to(row, target_row, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 65ae3a29..817bb078 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -395,25 +395,32 @@ pub fn Worksheet::formula_refs(self : Worksheet) -> Array[String] { /// Indexes and validates one worksheet's shared-formula groups. Followers are /// omitted from the returned map, so master formula text and its copy anchor /// are retained only once even for large shared ranges. +fn shared_formula_validation_checkpoint( + budget : ReadBudget?, + cancelled : () -> Bool, +) -> Unit raise XlsxError { + match budget { + Some(value) => value.checkpoint() + None => if cancelled() { raise ReadCancelled } + } +} + +///| fn validated_shared_formula_masters( cells : ArrayView[Cell], budget? : ReadBudget, + cancelled? : () -> Bool = () => false, ) -> Map[UInt, SharedFormulaMaster] raise XlsxError { let masters : Map[UInt, SharedFormulaMaster] = Map([]) + shared_formula_validation_checkpoint(budget, cancelled) match budget { - Some(value) => { - value.checkpoint() - value.charge_work(cells.length()) - } + Some(value) => value.charge_work(cells.length()) None => () } let mut cell_index = 0 for cell in cells { if (cell_index & 4095) == 0 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_validation_checkpoint(budget, cancelled) } match ( @@ -456,21 +463,16 @@ fn validated_shared_formula_masters( } cell_index += 1 } - validate_shared_formula_master_ranges(masters, budget?) + validate_shared_formula_master_ranges(masters, budget?, cancelled~) + shared_formula_validation_checkpoint(budget, cancelled) match budget { - Some(value) => { - value.checkpoint() - value.charge_work(cells.length()) - } + Some(value) => value.charge_work(cells.length()) None => () } cell_index = 0 for cell in cells { if (cell_index & 4095) == 0 { - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_validation_checkpoint(budget, cancelled) } match (cell.formula_type, cell.formula_shared_index, cell.formula_ref) { (Some(Shared), Some(shared_index), None) => @@ -487,10 +489,7 @@ fn validated_shared_formula_masters( } cell_index += 1 } - match budget { - Some(value) => value.checkpoint() - None => () - } + shared_formula_validation_checkpoint(budget, cancelled) masters } @@ -505,6 +504,46 @@ fn clone_shared_formula_masters_index( cloned } +///| +/// Counts only the shared cells an edit will retain or copy, then performs one +/// aggregate fan-out check before the first translated string is allocated. +fn preflight_shared_formula_materialization( + cells : ArrayView[Cell], + masters : Map[UInt, SharedFormulaMaster], + budget : SharedFormulaMaterializationBudget, + retain : (Cell) -> Bool, +) -> Unit raise XlsxError { + let formula_counts : Map[UInt, Int] = Map([]) + budget.checkpoint() + let mut cell_index = 0 + for cell in cells { + if (cell_index & 4095) == 0 { + budget.checkpoint() + } + if retain(cell) { + match (cell.formula_type, cell.formula_shared_index) { + (Some(Shared), Some(shared_index)) => { + let current = formula_counts.get(shared_index).unwrap_or(0) + let (next, exceeded) = read_budget_actual(current, 1, 0x7fffffff) + if exceeded { + raise ResourceLimitExceeded( + kind="shared_formula_followers", + limit=0x7fffffff, + actual=next, + ) + } + formula_counts[shared_index] = next + } + (Some(Shared), None) => + raise InvalidXml(msg="shared formula index missing") + _ => () + } + } + cell_index += 1 + } + budget.preflight(formula_counts, masters) +} + ///| /// Converts one shared-formula cell to an independent normal formula before a /// structural edit moves or duplicates it. Shared ranges are metadata over @@ -515,6 +554,7 @@ fn clone_shared_formula_masters_index( fn normal_formula_cell_for_structural_edit( cell : Cell, masters : Map[UInt, SharedFormulaMaster], + budget : SharedFormulaMaterializationBudget, ) -> Cell raise XlsxError { match (cell.formula_type, cell.formula_shared_index) { (Some(Shared), Some(shared_index)) => { @@ -529,7 +569,7 @@ fn normal_formula_cell_for_structural_edit( value: cell.value, value_type: cell.value_type, rich_text: cell.rich_text, - formula: Some(master.translate_to(cell.row, cell.col)), + formula: Some(budget.translate(master, cell.row, cell.col)), formula_type: None, formula_ref: None, formula_shared_index: None, @@ -550,14 +590,28 @@ fn Worksheet::invalidate_shared_formula_masters_index(self : Worksheet) -> Unit ///| fn Worksheet::ensure_shared_formula_masters_index( self : Worksheet, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { + if cancelled() { + raise ReadCancelled + } if self.shared_formula_masters_index_valid { return } - let rebuilt = validated_shared_formula_masters(self.cells) + let rebuilt = validated_shared_formula_masters(self.cells, cancelled~) self.shared_formula_masters_index.clear() + let mut master_index = 0 for shared_index, master in rebuilt { + if (master_index & 4095) == 0 && cancelled() { + self.shared_formula_masters_index.clear() + raise ReadCancelled + } self.shared_formula_masters_index[shared_index] = master + master_index += 1 + } + if cancelled() { + self.shared_formula_masters_index.clear() + raise ReadCancelled } self.shared_formula_masters_index_valid = true } @@ -577,11 +631,13 @@ pub fn Worksheet::shared_formula_masters( /// Looks up one validated shared-formula master without cloning the complete /// master index. Parsed worksheets already carry an eagerly validated index; /// programmatically edited worksheets rebuild it lazily before this lookup. +/// `cancelled` is polled while rebuilding a stale index. pub fn Worksheet::shared_formula_master( self : Worksheet, shared_index : UInt, + cancelled? : () -> Bool = () => false, ) -> SharedFormulaMaster? raise XlsxError { - self.ensure_shared_formula_masters_index() + self.ensure_shared_formula_masters_index(cancelled~) self.shared_formula_masters_index.get(shared_index) } @@ -3465,9 +3521,11 @@ fn Worksheet::insert_rows( self : Worksheet, row : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, + formula_budget? : SharedFormulaMaterializationBudget, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.invalidate_shared_formula_masters_index() if row <= 0 || row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{row}") } @@ -3481,7 +3539,20 @@ fn Worksheet::insert_rows( msg="row count \{count} exceeds the sheet's \{cell_ref_max_rows} rows", ) } - let shared_formula_masters = validated_shared_formula_masters(self.cells) + let translation_budget = match formula_budget { + Some(value) => value + None => SharedFormulaMaterializationBudget::new(formula_limits, cancelled~) + } + let shared_formula_masters = validated_shared_formula_masters( + self.cells, + cancelled=translation_budget.cancelled, + ) + preflight_shared_formula_materialization( + self.cells, + shared_formula_masters, + translation_budget, + fn(_cell : Cell) -> Bool { true }, + ) // Transactional: stage every shifted collection into locals first — cell, // merge, hyperlink, filter, table, sparkline, image, and chart references // shift through `cell_ref_from`/range adjusters that raise once a coordinate @@ -3492,7 +3563,7 @@ fn Worksheet::insert_rows( let updated_cells : Array[Cell] = [] for stored_cell in self.cells { let cell = normal_formula_cell_for_structural_edit( - stored_cell, shared_formula_masters, + stored_cell, shared_formula_masters, translation_budget, ) if cell.row >= row { let new_row = cell.row + count @@ -3591,6 +3662,7 @@ fn Worksheet::insert_rows( ) // Commit: every stage above succeeded, so applying the staged state cannot // fail partway. + self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() self.cells.clear() self.cells.append(updated_cells) @@ -3620,10 +3692,10 @@ fn Worksheet::remove_rows( self : Worksheet, row : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.invalidate_shared_formula_masters_index() - self.invalidate_cell_index() if row <= 0 { raise InvalidCellRef(value="\{0}:\{row}") } @@ -3640,20 +3712,33 @@ fn Worksheet::remove_rows( msg="row count \{count} exceeds the sheet's \{cell_ref_max_rows} rows", ) } - let shared_formula_masters = validated_shared_formula_masters(self.cells) + let translation_budget = SharedFormulaMaterializationBudget::new( + formula_limits, + cancelled~, + ) + let shared_formula_masters = validated_shared_formula_masters( + self.cells, + cancelled=translation_budget.cancelled, + ) let end_row = row + count - 1 + preflight_shared_formula_materialization( + self.cells, + shared_formula_masters, + translation_budget, + fn(cell : Cell) -> Bool { cell.row < row || cell.row > end_row }, + ) let updated_cells : Array[Cell] = [] for stored_cell in self.cells { + if stored_cell.row >= row && stored_cell.row <= end_row { + continue + } let cell = normal_formula_cell_for_structural_edit( - stored_cell, shared_formula_masters, + stored_cell, shared_formula_masters, translation_budget, ) if cell.row < row { updated_cells.push(cell) continue } - if cell.row <= end_row { - continue - } let new_row = cell.row - count let reference = cell_ref_from(new_row, cell.col) updated_cells.push({ @@ -3671,6 +3756,8 @@ fn Worksheet::remove_rows( style_id: cell.style_id, }) } + self.invalidate_shared_formula_masters_index() + self.invalidate_cell_index() self.cells.clear() self.cells.append(updated_cells) let updated_merges : Array[String] = [] @@ -3937,10 +4024,10 @@ fn Worksheet::duplicate_row_to( self : Worksheet, row : Int, target_row : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.invalidate_shared_formula_masters_index() - self.invalidate_cell_index() if row <= 0 || row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{row}") } @@ -3952,18 +4039,38 @@ fn Worksheet::duplicate_row_to( if target_row > cell_ref_max_rows { raise InvalidCellRef(value="\{0}:\{target_row}") } - let shared_formula_masters = validated_shared_formula_masters(self.cells) + let translation_budget = SharedFormulaMaterializationBudget::new( + formula_limits, + cancelled~, + ) + let shared_formula_masters = validated_shared_formula_masters( + self.cells, + cancelled=translation_budget.cancelled, + ) + preflight_shared_formula_materialization( + self.cells, + shared_formula_masters, + translation_budget, + fn(cell : Cell) -> Bool { cell.row == row }, + ) let row_cells : Array[Cell] = [] for stored_cell in self.cells { + if stored_cell.row != row { + continue + } let cell = normal_formula_cell_for_structural_edit( - stored_cell, shared_formula_masters, + stored_cell, shared_formula_masters, translation_budget, ) - if cell.row == row { - row_cells.push(cell) - } + row_cells.push(cell) } let row_dim = self.row_dimensions.get(row) - self.insert_rows(target_row, 1) + self.insert_rows( + target_row, + 1, + formula_limits~, + cancelled~, + formula_budget=translation_budget, + ) for cell in row_cells { let reference = cell_ref_from(target_row, cell.col) self.cells.push({ @@ -3995,9 +4102,10 @@ fn Worksheet::insert_cols( self : Worksheet, col : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.invalidate_shared_formula_masters_index() if col <= 0 || col > cell_ref_max_cols { raise InvalidCellRef(value="\{col}:\{0}") } @@ -4010,7 +4118,20 @@ fn Worksheet::insert_cols( msg="column count \{count} exceeds the sheet's \{cell_ref_max_cols} columns", ) } - let shared_formula_masters = validated_shared_formula_masters(self.cells) + let translation_budget = SharedFormulaMaterializationBudget::new( + formula_limits, + cancelled~, + ) + let shared_formula_masters = validated_shared_formula_masters( + self.cells, + cancelled=translation_budget.cancelled, + ) + preflight_shared_formula_materialization( + self.cells, + shared_formula_masters, + translation_budget, + fn(_cell : Cell) -> Bool { true }, + ) // Transactional (see insert_rows): stage everything into locals — reference // shifts raise once a coordinate leaves the grid, column dimensions and page // breaks are bound-checked explicitly — and commit only once every stage has @@ -4018,7 +4139,7 @@ fn Worksheet::insert_cols( let updated_cells : Array[Cell] = [] for stored_cell in self.cells { let cell = normal_formula_cell_for_structural_edit( - stored_cell, shared_formula_masters, + stored_cell, shared_formula_masters, translation_budget, ) if cell.col >= col { let new_col = cell.col + count @@ -4114,6 +4235,7 @@ fn Worksheet::insert_cols( count, ) // Commit. + self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() self.cells.clear() self.cells.append(updated_cells) @@ -4143,10 +4265,10 @@ fn Worksheet::remove_cols( self : Worksheet, col : Int, count : Int, + formula_limits? : SharedFormulaLimits = SharedFormulaLimits::new(), + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.invalidate_shared_formula_masters_index() - self.invalidate_cell_index() if col <= 0 { raise InvalidCellRef(value="\{col}:\{0}") } @@ -4162,20 +4284,33 @@ fn Worksheet::remove_cols( msg="column count \{count} exceeds the sheet's \{cell_ref_max_cols} columns", ) } - let shared_formula_masters = validated_shared_formula_masters(self.cells) + let translation_budget = SharedFormulaMaterializationBudget::new( + formula_limits, + cancelled~, + ) + let shared_formula_masters = validated_shared_formula_masters( + self.cells, + cancelled=translation_budget.cancelled, + ) let end_col = col + count - 1 + preflight_shared_formula_materialization( + self.cells, + shared_formula_masters, + translation_budget, + fn(cell : Cell) -> Bool { cell.col < col || cell.col > end_col }, + ) let updated_cells : Array[Cell] = [] for stored_cell in self.cells { + if stored_cell.col >= col && stored_cell.col <= end_col { + continue + } let cell = normal_formula_cell_for_structural_edit( - stored_cell, shared_formula_masters, + stored_cell, shared_formula_masters, translation_budget, ) if cell.col < col { updated_cells.push(cell) continue } - if cell.col <= end_col { - continue - } let new_col = cell.col - count let reference = cell_ref_from(cell.row, new_col) updated_cells.push({ @@ -4193,6 +4328,8 @@ fn Worksheet::remove_cols( style_id: cell.style_id, }) } + self.invalidate_shared_formula_masters_index() + self.invalidate_cell_index() self.cells.clear() self.cells.append(updated_cells) let updated_merges : Array[String] = [] From db270a7327f8689b1018d52acde9851a241c56ba Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 00:14:41 +0800 Subject: [PATCH 106/150] test(office): align percent overflow contract --- office/cmd/office/xlsx_read_wbtest.mbt | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index e7275e76..763e8646 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -891,7 +891,7 @@ async test "XLSX formatting rejects text amplification before materialization" { } ///| -async test "XLSX structured formatting rejects non-finite percent scaling" { +async test "XLSX structured formatting contains percent scaling overflow" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("Data")) workbook.set_cell_value("Data", "A1", Numeric(0.0)) @@ -911,8 +911,23 @@ async test "XLSX structured formatting rejects non-finite percent scaling" { let resolved = resolve_xlsx_selector( projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]", ) + assert_true(xlsx_get_data(projection, resolved).stringify().contains("%")) + + workbook.set_cell_value("Data", "A1", Numeric(1.0)) + let nonzero_source = xlsx_bounded_test_source( + "non-finite-nonzero-format.xlsx", + @xlsx.write(workbook), + ) + let nonzero_projection = make_xlsx_projection( + nonzero_source.file, + open_xlsx_read_package(nonzero_source), + 10, + ) + let nonzero_resolved = resolve_xlsx_selector( + nonzero_projection, "/xlsx/sheet[name=\"Data\"]/cell[A1]", + ) let error = xlsx_cli_error_async(() => { - ignore(xlsx_get_data(projection, resolved)) + ignore(xlsx_get_data(nonzero_projection, nonzero_resolved)) }) assert_eq(error.code, "office.invalid_package") } From 7ee43b4a20d2e696ea7ca77ae913252e722f813d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 01:20:12 +0800 Subject: [PATCH 107/150] fix(xlsx): scale percent formats incrementally --- xlsx/value_format.mbt | 35 ++++++++++++++++++++++++----------- xlsx/value_format_test.mbt | 15 +++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index e24b37ff..78d2c7e4 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -590,17 +590,7 @@ fn format_number_pattern( let negative = value < 0.0 let abs_value = if negative { -value } else { value } let (prefix, suffix, info) = parse_number_pattern(pattern) - // IEEE-754 defines 0 * infinity as NaN. A long but syntactically valid run - // of percent markers can overflow the scale factor, yet formatting numeric - // zero must remain exactly zero rather than manufacturing a formatter error. - let scaled = if abs_value == 0.0 { - 0.0 - } else { - abs_value * @math.pow(100.0, Double::from_int(info.percent_count)) - } - if scaled.is_nan() || scaled.is_inf() { - raise InvalidXml(msg="number format scale must remain finite") - } + let scaled = scale_number_for_percent_format(abs_value, info.percent_count) let formatted = if info.has_fraction { format_fraction_number(scaled, info) } else if info.has_scientific { @@ -619,6 +609,29 @@ fn format_number_pattern( output.to_string() } +///| +fn scale_number_for_percent_format( + value : Double, + percent_count : Int, +) -> Double raise XlsxError { + // Applying the entire power first can overflow even when the final product + // is representable (for example, 1e-300 * 100^155). Scaling one marker at a + // time preserves that finite result and lets us fail at the first genuinely + // non-finite intermediate. Keeping zero on an explicit fast path also avoids + // manufacturing NaN from IEEE-754's 0 * infinity rule. + if value == 0.0 || percent_count == 0 { + return value + } + let mut scaled = value + for _ in 0.. Date: Sun, 19 Jul 2026 01:41:11 +0800 Subject: [PATCH 108/150] fix(xlsx): propagate spill calculation failures --- xlsx/formula_builtins_stats.mbt | 23 +++++++++--- xlsx/formula_eval_wbtest.mbt | 47 +++++++++++++++++++++++++ xlsx/shared_formula_translate.mbt | 9 +++-- xlsx/shared_formula_validation_test.mbt | 42 ++++++++++++++++++++++ 4 files changed, 114 insertions(+), 7 deletions(-) diff --git a/xlsx/formula_builtins_stats.mbt b/xlsx/formula_builtins_stats.mbt index 5d9162b4..fca64561 100644 --- a/xlsx/formula_builtins_stats.mbt +++ b/xlsx/formula_builtins_stats.mbt @@ -1843,7 +1843,13 @@ fn resolve_cell_value( } ctx.visiting[key] = true ctx.depth = ctx.depth + 1 - let value = calc_cell_value_internal(workbook, sheet_name, reference, ctx) + let value = calc_cell_value_internal(workbook, sheet_name, reference, ctx) catch { + error => { + ignore(ctx.visiting.remove(key)) + ctx.depth = ctx.depth - 1 + raise error + } + } ignore(ctx.visiting.remove(key)) ctx.depth = ctx.depth - 1 value @@ -1868,9 +1874,10 @@ fn spill_value_for_cell( None => continue } let expr = parse_formula_expr(formula) catch { _ => continue } - let shape = array_shape_from_expr(workbook, sheet_name, expr, ctx) catch { - _ => continue - } + // Shape evaluation returns ordinary formula failures as values. Raised + // errors are structural, resource, or cancellation failures and must not + // be downgraded to "not a spill" while probing an otherwise empty cell. + let shape = array_shape_from_expr(workbook, sheet_name, expr, ctx) let (rows, cols) = match shape { Some(value) => value None => continue @@ -1944,7 +1951,13 @@ fn calc_cell_value_internal( let prev_ref = ctx.current_ref ctx.current_sheet = sheet_name ctx.current_ref = reference - let result = eval_expr(workbook, sheet_name, expr, ctx) + let result = eval_expr(workbook, sheet_name, expr, ctx) catch { + error => { + ctx.current_sheet = prev_sheet + ctx.current_ref = prev_ref + raise error + } + } ctx.current_sheet = prev_sheet ctx.current_ref = prev_ref return result diff --git a/xlsx/formula_eval_wbtest.mbt b/xlsx/formula_eval_wbtest.mbt index a888ba59..d64b9733 100644 --- a/xlsx/formula_eval_wbtest.mbt +++ b/xlsx/formula_eval_wbtest.mbt @@ -1398,6 +1398,53 @@ test "formula eval wb: array shape dispatcher guards part 3" { ) } +///| +test "formula eval wb: raised spill probes restore calculation state" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts("B1", "2", opts=FormulaOpts::shared("B1:B2")) + sheet.set_cell_formula("D1", "SEQUENCE(B2)") + let ctx = CalcContext::new(cancelled=() => true) + try resolve_cell_value(workbook, "Data", "D2", ctx) catch { + ReadCancelled => () + _ => fail("unexpected spill cleanup error") + } noraise { + _ => fail("expected spill cancellation") + } + inspect(ctx.depth, content="0") + inspect(ctx.visiting.length(), content="0") + inspect(ctx.current_sheet, content="") + inspect(ctx.current_ref, content="") +} + +///| +test "formula eval wb: failed translations consume aggregate work" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts("B1", "A1+1", opts=FormulaOpts::shared("B1:B2")) + guard sheet.shared_formula_masters().get(0) is Some(master) else { + fail("missing shared formula master") + } + let checks = [0] + let limits = SharedFormulaLimits::with_values( + max_input_chars=16, + max_output_chars=16, + max_total_output_chars=16, + max_work_units=32, + ) + let budget = SharedFormulaMaterializationBudget::new(limits, cancelled=() => { + checks[0] += 1 + checks[0] >= 3 + }) + try budget.translate(master, 2, 2) catch { + ReadCancelled => () + _ => fail("unexpected failed-translation error") + } noraise { + value => fail("expected translation cancellation, got \{value}") + } + inspect(budget.work_units, content="4") +} + ///| test "formula eval wb: array shape advanced function branches part 3" { let workbook = Workbook::new() diff --git a/xlsx/shared_formula_translate.mbt b/xlsx/shared_formula_translate.mbt index 76b27036..764818cc 100644 --- a/xlsx/shared_formula_translate.mbt +++ b/xlsx/shared_formula_translate.mbt @@ -245,8 +245,13 @@ fn SharedFormulaMaterializationBudget::translate( actual=projected_work, ) } + // Reserve the minimum translation work before entering the scanner. If a + // caller catches a cancellation or resource failure and reuses this budget, + // the failed attempt still consumes aggregate work instead of becoming free. + let previous_work = self.work_units + self.work_units = projected_work let remaining_output = self.limits.max_total_output_chars - self.output_chars - let remaining_work = self.limits.max_work_units - self.work_units + let remaining_work = self.limits.max_work_units - previous_work let (translated, work) = master.translate_to_limited( row, column, @@ -268,7 +273,7 @@ fn SharedFormulaMaterializationBudget::translate( ) } let (work_units, work_exceeded) = read_budget_actual( - self.work_units, + previous_work, work, self.limits.max_work_units, ) diff --git a/xlsx/shared_formula_validation_test.mbt b/xlsx/shared_formula_validation_test.mbt index 73e4dc12..d83bc5f7 100644 --- a/xlsx/shared_formula_validation_test.mbt +++ b/xlsx/shared_formula_validation_test.mbt @@ -400,6 +400,48 @@ test "calculation shares formula output and cancellation budgets" { } } +///| +fn shared_formula_spill_probe_fixture() -> @xlsx.Workbook raise { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.add_sheet("Data") + sheet.set_cell_formula_opts( + "B1", + "2", + opts=@xlsx.FormulaOpts::shared("B1:B2"), + ) + sheet.set_cell_formula("D1", "SEQUENCE(B2)") + workbook +} + +///| +test "spill probing propagates shared formula cancellation and limits" { + let cancelled = shared_formula_spill_probe_fixture() + try cancelled.calc_cell_value("Data", "D2", cancelled=() => true) catch { + ReadCancelled => () + _ => fail("unexpected spill cancellation error") + } noraise { + value => fail("expected spill cancellation, got \{value}") + } + + let limited = shared_formula_spill_probe_fixture() + let limits = @xlsx.SharedFormulaLimits::with_values( + max_input_chars=16, + max_output_chars=16, + max_total_output_chars=16, + max_work_units=1, + ) + try limited.calc_cell_value("Data", "D2", formula_limits=limits) catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "shared_formula_work_units") + assert_eq(limit, 1) + assert_true(actual > limit) + } + _ => fail("unexpected spill resource error") + } noraise { + value => fail("expected spill resource rejection, got \{value}") + } +} + ///| test "XLSX read rejects overlapping shared-formula master ranges" { let workbook = @xlsx.Workbook::new() From 44a7f1694bfa41d32b5be3285fda0e490fceb50a Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 01:48:43 +0800 Subject: [PATCH 109/150] fix(xlsx): project MCE conservatively --- ooxml/mce_projection.mbt | 50 ++++++++++++++++- ooxml/mce_projection_test.mbt | 10 ++++ ooxml/pkg.generated.mbti | 2 +- xlsx/chart_sheet_test.mbt | 55 +++++++++++++++++++ ...integration_unknown_ext_roundtrip_test.mbt | 3 +- xlsx/read.mbt | 18 +++++- xlsx/read_namespace_test.mbt | 35 ++++++++++++ xlsx/read_xml_namespaces.mbt | 10 +++- 8 files changed, 176 insertions(+), 7 deletions(-) diff --git a/ooxml/mce_projection.mbt b/ooxml/mce_projection.mbt index 9fc36648..a1a7c7b1 100644 --- a/ooxml/mce_projection.mbt +++ b/ooxml/mce_projection.mbt @@ -10,6 +10,12 @@ priv struct XmlMceName { local_name : String } derive(Eq, Hash) +///| +priv struct XmlMceExtensionName { + namespace_uri : String + local_name : String +} derive(Eq, Hash) + ///| priv struct XmlMceScopeDelta { ignorable_namespace_ids : Array[Int] @@ -25,6 +31,7 @@ priv struct XmlMceScopeEntry { ///| priv struct XmlMceContext { understood_namespaces : Set[String] + extension_elements : Set[XmlMceExtensionName] ignorable_namespace_counts : Map[Int, Int] process_content_name_counts : Map[XmlMceName, Int] max_work_chars : Int @@ -34,14 +41,21 @@ priv struct XmlMceContext { ///| fn XmlMceContext::new( understood_namespaces : ArrayView[String], + extension_elements : ArrayView[(String, String)], max_work_chars : Int, ) -> XmlMceContext { let understood : Set[String] = Set([]) for namespace_uri in understood_namespaces { understood.add(namespace_uri) } + let extensions : Set[XmlMceExtensionName] = Set([]) + for extension in extension_elements { + let (namespace_uri, local_name) = extension + extensions.add({ namespace_uri, local_name }) + } { understood_namespaces: understood, + extension_elements: extensions, ignorable_namespace_counts: Map([]), process_content_name_counts: Map([]), max_work_chars, @@ -49,6 +63,15 @@ fn XmlMceContext::new( } } +///| +fn XmlMceContext::is_extension_element( + self : XmlMceContext, + namespace_uri : String, + local_name : String, +) -> Bool { + self.extension_elements.contains({ namespace_uri, local_name }) +} + ///| fn XmlMceContext::charge( self : XmlMceContext, @@ -170,6 +193,7 @@ priv struct XmlMceProjectionState { mut alternate_choice_seen : Bool mut alternate_fallback_seen : Bool mut alternate_selected : Bool + opaque_descendants : Bool } ///| @@ -447,6 +471,21 @@ fn xml_mce_state_for_current_element( alternate_choice_seen: false, alternate_fallback_seen: false, alternate_selected: false, + opaque_descendants: false, + } + _ => () + } + match parent { + Some(parent_state) if parent_state.opaque_descendants => + return { + active: true, + scope_delta: None, + pending_bindings: None, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + opaque_descendants: true, } _ => () } @@ -465,6 +504,7 @@ fn xml_mce_state_for_current_element( alternate_choice_seen: false, alternate_fallback_seen: false, alternate_selected: false, + opaque_descendants: false, } } let parent_state = parent.unwrap() @@ -572,6 +612,7 @@ fn xml_mce_state_for_current_element( alternate_choice_seen: false, alternate_fallback_seen: false, alternate_selected: false, + opaque_descendants: context.is_extension_element(namespace_uri, local_name), } } @@ -581,12 +622,15 @@ fn xml_mce_state_for_current_element( /// subtrees are removed, selected `Choice`/`Fallback` and `ProcessContent` /// wrappers are unwrapped, and namespace declarations from removed wrappers /// are retained on promoted children. Processing is streaming, cancellable, -/// and bounded by `max_output_chars`. +/// and bounded by `max_output_chars`. Descendants of application-defined +/// `extension_elements` remain opaque: they are namespace-validated as XML but +/// MCE directives inside their payload are preserved rather than interpreted. pub fn project_xml_markup_compatibility( xml : StringView, understood_namespaces : ArrayView[String], max_output_chars? : Int = 16 * 1024 * 1024, cancelled? : () -> Bool = () => false, + extension_elements? : ArrayView[(String, String)] = [], ) -> String raise ParseXmlError { check_xml_cancelled(cancelled) if max_output_chars < 0 { @@ -602,7 +646,9 @@ pub fn project_xml_markup_compatibility( max_output_chars, cancelled~, ) - let context = XmlMceContext::new(understood_namespaces, max_output_chars) + let context = XmlMceContext::new( + understood_namespaces, extension_elements, max_output_chars, + ) let states : Array[XmlMceProjectionState] = [] while scanner.next() { states.push(xml_mce_state_for_current_element(scanner, states, context)) diff --git a/ooxml/mce_projection_test.mbt b/ooxml/mce_projection_test.mbt index d5933573..3ce22d49 100644 --- a/ooxml/mce_projection_test.mbt +++ b/ooxml/mce_projection_test.mbt @@ -34,6 +34,16 @@ test "MCE projection unwraps ProcessContent and ignores opaque extensions" { assert_true(projected.contains("a:Kept")) } +///| +test "MCE projection preserves configured extension payloads verbatim" { + let source = + #| + let projected = @ooxml.project_xml_markup_compatibility(source, ["urn:app"], extension_elements=[ + ("urn:app", "ext"), + ]) + inspect(projected, content=source) +} + ///| test "MCE projection does not enforce MustUnderstand on an unselected Choice" { let xml = diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 93369095..dbb1cbcc 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -40,7 +40,7 @@ pub fn parse_internal_relationship_targets(StringView, StringView, cancelled? : pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError -pub fn project_xml_markup_compatibility(StringView, ArrayView[String], max_output_chars? : Int, cancelled? : () -> Bool) -> String raise ParseXmlError +pub fn project_xml_markup_compatibility(StringView, ArrayView[String], max_output_chars? : Int, cancelled? : () -> Bool, extension_elements? : ArrayView[(String, String)]) -> String raise ParseXmlError pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError diff --git a/xlsx/chart_sheet_test.mbt b/xlsx/chart_sheet_test.mbt index a1c095a3..48ccabb8 100644 --- a/xlsx/chart_sheet_test.mbt +++ b/xlsx/chart_sheet_test.mbt @@ -49,6 +49,61 @@ test "chart sheet write and read" { inspect(parsed_chart.chart_xml().contains("GOOD", + ), + ) + ignore( + workbook.add_chart_sheet( + "Bad", "BAD", + ), + ) + let archive = @zip.read(@xlsx.write(workbook)) + guard archive.get("xl/chartsheets/sheet1.xml") is Some(sheet_bytes) else { + fail("missing first chart sheet") + } + let sheet_xml = @encoding/utf8.decode(sheet_bytes) catch { + _ => fail("chart sheet is not UTF-8") + } + let with_namespaces = sheet_xml.replace( + old="", + new="", + ) + assert_true(projected != sheet_xml) + assert_true( + archive.replace( + "xl/chartsheets/sheet1.xml", + @encoding/utf8.encode(projected), + ), + ) + let rels_path = "xl/chartsheets/_rels/sheet1.xml.rels" + guard archive.get(rels_path) is Some(rels_bytes) else { + fail("missing first chart-sheet relationships") + } + let rels = @encoding/utf8.decode(rels_bytes) catch { + _ => fail("chart-sheet relationships are not UTF-8") + } + let with_bad = rels.replace( + old="", + new="", + ) + assert_true(with_bad != rels) + assert_true(archive.replace(rels_path, @encoding/utf8.encode(with_bad))) + let parsed = @xlsx.read(@zip.write(archive)) + assert_eq(parsed.chart_sheets().length(), 2) + assert_true(parsed.chart_sheets()[0].chart_xml().contains("GOOD")) + assert_false(parsed.chart_sheets()[0].chart_xml().contains(">BAD<")) +} + ///| test "chart sheet with options (excelize-like)" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/integration_unknown_ext_roundtrip_test.mbt b/xlsx/integration_unknown_ext_roundtrip_test.mbt index fe8aa703..2e782577 100644 --- a/xlsx/integration_unknown_ext_roundtrip_test.mbt +++ b/xlsx/integration_unknown_ext_roundtrip_test.mbt @@ -62,7 +62,7 @@ test "integration unknown worksheet ext roundtrip with mixed known ext" { Some(value) => @encoding/utf8.decode(value) None => fail("missing sheet2.xml") } - let unknown_ext = " keep-me\n" + let unknown_ext = " keep-me\n" let updated_sheet = if sheet_xml.contains(" \n") { sheet_xml.replace_all( old=" \n", @@ -125,6 +125,7 @@ test "integration unknown worksheet ext roundtrip with mixed known ext" { roundtrip_sheet_xml.contains("keep-me"), content="true", ) + inspect(roundtrip_sheet_xml.contains(""), content="true") inspect(roundtrip_sheet_xml.contains("x14:sparklineGroups"), content="true") inspect( roundtrip_sheet_xml.contains("x14:conditionalFormattings"), diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 313807f4..fbd23887 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4641,7 +4641,23 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise MissingPart(path=chartsheet_path) } - let chartsheet_xml = decode(chartsheet_bytes) + let chartsheet_raw_xml = decode_source(chartsheet_bytes) + read_budget.charge_work(chartsheet_raw_xml.length()) + let chartsheet_source_xml = project_xlsx_markup_compatibility( + chartsheet_raw_xml, + limits.max_xml_part_bytes, + cancelled~, + ) + if chartsheet_source_xml != chartsheet_raw_xml { + read_budget.charge_work(chartsheet_source_xml.length()) + } + let chartsheet_xml = canonicalize_xlsx_xml( + chartsheet_source_xml, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(chartsheet_xml.length()) let drawing_rel_id = parse_legacy_drawing_rel_id( chartsheet_xml, "drawing", ) diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 9e774463..57d013ec 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -158,6 +158,41 @@ test "workbook and worksheet MCE fallbacks read end to end" { ) } +///| +test "x14 Choice uses core auto-filter fallback" { + let archive = @zip.read(mce_fallback_xlsx_fixture()) + guard archive.get("xl/worksheets/sheet1.xml") is Some(sheet_bytes) else { + fail("missing MCE worksheet fixture") + } + let source = @encoding/utf8.decode(sheet_bytes) catch { + _ => fail("MCE worksheet fixture is not UTF-8") + } + let with_namespace = source.replace( + old="xmlns:x=\"urn:vendor\" mc:Ignorable=\"x\"", + new="xmlns:x=\"urn:vendor\" xmlns:x14=\"http://schemas.microsoft.com/office/spreadsheetml/2009/9/main\" mc:Ignorable=\"x x14\"", + ) + let updated = with_namespace.replace( + old="", + new=( + #| + ), + ) + assert_true(updated != source) + assert_true( + archive.replace("xl/worksheets/sheet1.xml", @encoding/utf8.encode(updated)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + guard parsed.get_auto_filter("Data") is Some(filter) else { + fail("missing fallback auto filter") + } + assert_eq(filter.columns.length(), 1) + guard filter.columns[0].custom_filters is Some(custom) else { + fail("missing fallback custom filter") + } + assert_eq(custom.filters.length(), 1) + assert_eq(custom.filters[0].value, "fallback") +} + ///| test "physical ZIP names follow Unicode logical OPC part names end to end" { let workbook = @xlsx.read(unicode_part_name_xlsx_fixture()) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index ffaca78a..f442d9c7 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -34,8 +34,13 @@ let markup_compatibility_namespace = "http://schemas.openxmlformats.org/markup-c ///| let xlsx_mce_understood_namespaces : Array[String] = [ transitional_spreadsheet_namespace, strict_spreadsheet_namespace, transitional_relationship_attribute_namespace, - strict_relationship_attribute_namespace, xlsx_extension_namespace_x14, xlsx_extension_namespace_x15, - xlsx_extension_namespace_xm, + strict_relationship_attribute_namespace, +] + +///| +let xlsx_mce_extension_elements : Array[(String, String)] = [ + (transitional_spreadsheet_namespace, "ext"), + (strict_spreadsheet_namespace, "ext"), ] ///| @@ -52,6 +57,7 @@ fn project_xlsx_markup_compatibility( xlsx_mce_understood_namespaces, max_output_chars~, cancelled~, + extension_elements=xlsx_mce_extension_elements, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled From 5c884943ebc40432b540d663ca51e66993b9d769 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 01:58:59 +0800 Subject: [PATCH 110/150] fix(ooxml): validate package declarations strictly --- ooxml/read_parse.mbt | 404 +++++++++++++++++++++++++++++++++++- ooxml/read_parse_test.mbt | 247 ++++++++++++++++------ ooxml/start_tag_scanner.mbt | 90 ++++++++ xlsx/ooxml_rels.mbt | 16 +- 4 files changed, 678 insertions(+), 79 deletions(-) diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 85f06803..69066df2 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -98,7 +98,7 @@ let max_relationship_id_chars = 1024 let max_relationship_type_chars : Int = 16 * 1024 ///| -let max_relationship_target_chars : Int = 16 * 1024 * 1024 +let max_relationship_target_chars : Int = 64 * 1024 ///| let max_relationship_retained_chars : Int = 16 * 1024 * 1024 @@ -144,6 +144,91 @@ fn relationship_attribute( value } +///| +fn relationship_uri_ascii_character(character : UInt16) -> Bool { + (character >= 'A' && character <= 'Z') || + (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character == '-' || + character == '.' || + character == '_' || + character == '~' || + character == ':' || + character == '/' || + character == '?' || + character == '#' || + character == '[' || + character == ']' || + character == '@' || + character == '!' || + character == '$' || + character == '&' || + character == ('\'' : UInt16) || + character == '(' || + character == ')' || + character == '*' || + character == '+' || + character == ',' || + character == ';' || + character == '=' || + character == '%' +} + +///| +fn relationship_uri_hex_digit(character : UInt16) -> Bool { + (character >= '0' && character <= '9') || + (character >= 'A' && character <= 'F') || + (character >= 'a' && character <= 'f') +} + +///| +fn relationship_uri_is_valid(value : StringView, absolute : Bool) -> Bool { + if value == "" { + return false + } + let mut index = 0 + while index < value.length() { + let character = value[index] + if character == ('%' : UInt16) { + if index + 2 >= value.length() || + !relationship_uri_hex_digit(value[index + 1]) || + !relationship_uri_hex_digit(value[index + 2]) { + return false + } + index = index + 3 + continue + } + if character < 0x80 && !relationship_uri_ascii_character(character) { + return false + } + index = index + 1 + } + if !absolute { + return true + } + let mut scheme_end = 0 + while scheme_end < value.length() && value[scheme_end] != (':' : UInt16) { + scheme_end = scheme_end + 1 + } + if scheme_end == 0 || + scheme_end >= value.length() || + !value[0].to_char().unwrap().is_ascii_alphabetic() { + return false + } + for index in 1..= 'A' && character <= 'Z') || + (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character == '+' || + character == '-' || + character == '.') { + return false + } + } + true +} + ///| priv enum ParsedRelationshipTargetMode { InternalTarget @@ -157,7 +242,7 @@ fn relationship_target_mode( match scanner.attribute("", "TargetMode") { None => InternalTarget Some(value) => - match collapse_schema_whitespace(value) { + match value { "Internal" => InternalTarget "External" => ExternalTarget _ => raise InvalidXml(msg="relationship target mode invalid") @@ -502,6 +587,49 @@ fn relationship_mce_require_must_understand( } } +///| +fn relationship_mce_directive_attribute(local_name : StringView) -> Bool { + local_name == "Ignorable" || + local_name == "ProcessContent" || + local_name == "MustUnderstand" || + local_name == "PreserveElements" || + local_name == "PreserveAttributes" +} + +///| +fn relationship_schema_attribute_allowed( + scanner : XmlStartTagScanner, + context : RelationshipMceContext, + allowed_unqualified : ArrayView[String], + element_label : StringView, +) -> Unit raise ParseXmlError { + for attribute in scanner.expanded_attribute_names() { + if attribute.namespace_uri == "" { + let mut allowed = false + for name in allowed_unqualified { + if attribute.local_name == name { + allowed = true + break + } + } + if allowed { + continue + } + } else if attribute.namespace_uri == markup_compatibility_namespace && + relationship_mce_directive_attribute(attribute.local_name) { + continue + } else if context.is_effectively_ignorable( + attribute.namespace_id, + attribute.namespace_uri, + ) { + continue + } + raise InvalidXml( + msg="relationships \{element_label.to_owned()} has a disallowed attribute", + ) + } +} + ///| fn relationship_mce_validate_container_attributes( scanner : XmlStartTagScanner, @@ -515,10 +643,13 @@ fn relationship_mce_validate_container_attributes( attribute.local_name == "Requires" { continue } - if attribute.namespace_uri != xml_namespace_uri && - ( - attribute.namespace_uri == markup_compatibility_namespace || - context.is_ignorable(attribute.namespace_id) + if attribute.namespace_uri == markup_compatibility_namespace && + relationship_mce_directive_attribute(attribute.local_name) { + continue + } + if context.is_effectively_ignorable( + attribute.namespace_id, + attribute.namespace_uri, ) { continue } @@ -612,6 +743,12 @@ fn relationship_mce_state_for_current_element( raise InvalidXml(msg="relationships document element invalid") } relationship_mce_require_must_understand(scope_entry) + relationship_schema_attribute_allowed( + scanner, + context, + [], + "document element", + ) return { active: true, included_in_effective_tree: true, @@ -761,6 +898,12 @@ fn parse_relationship_targets_by_types_selected( mce_state.element_effective_depth != 2 { raise InvalidXml(msg="relationships effective content is invalid") } + relationship_schema_attribute_allowed( + scanner, + mce_context, + ["Id", "Type", "Target", "TargetMode"], + "record", + ) records = records + 1 if records > max_relationship_records { raise InvalidXml(msg="relationship record limit exceeded") @@ -772,12 +915,21 @@ fn parse_relationship_targets_by_types_selected( if rel.length() > max_relationship_type_chars { raise InvalidXml(msg="relationship type is too long") } + if !relationship_uri_is_valid(rel, true) { + raise InvalidXml(msg="relationship type is invalid") + } if id.length() > max_relationship_id_chars { raise InvalidXml(msg="relationship id is too long") } + if !is_xml_ncname(id) { + raise InvalidXml(msg="relationship id is invalid") + } if target.length() > max_relationship_target_chars { raise InvalidXml(msg="relationship target is too long") } + if !relationship_uri_is_valid(target, false) { + raise InvalidXml(msg="relationship target is invalid") + } if ids.contains(id) { raise InvalidXml(msg="duplicate relationship id") } @@ -879,6 +1031,187 @@ priv struct PackageContentTypes { defaults : Map[String, String] } +///| +let max_content_type_records = 1_000_000 + +///| +let max_content_type_attribute_chars : Int = 16 * 1024 + +///| +let max_content_type_retained_chars : Int = 16 * 1024 * 1024 + +///| +fn content_type_token_character(character : UInt16) -> Bool { + character > 0x20 && + character < 0x7f && + character != '(' && + character != ')' && + character != '<' && + character != '>' && + character != '@' && + character != ',' && + character != ';' && + character != ':' && + character != ('\\' : UInt16) && + character != '"' && + character != '/' && + character != '[' && + character != ']' && + character != '?' && + character != '=' && + character != '{' && + character != '}' +} + +///| +fn content_type_skip_optional_whitespace( + value : StringView, + start : Int, +) -> Int { + let mut index = start + while index < value.length() && + (value[index] == (' ' : UInt16) || value[index] == ('\t' : UInt16)) { + index = index + 1 + } + index +} + +///| +fn content_type_media_type_is_valid(value : StringView) -> Bool { + let mut index = 0 + let type_start = index + while index < value.length() && content_type_token_character(value[index]) { + index = index + 1 + } + if index == type_start || + index >= value.length() || + value[index] != ('/' : UInt16) { + return false + } + index = index + 1 + let subtype_start = index + while index < value.length() && content_type_token_character(value[index]) { + index = index + 1 + } + if index == subtype_start { + return false + } + while true { + index = content_type_skip_optional_whitespace(value, index) + if index == value.length() { + return true + } + if value[index] != (';' : UInt16) { + return false + } + index = content_type_skip_optional_whitespace(value, index + 1) + let name_start = index + while index < value.length() && content_type_token_character(value[index]) { + index = index + 1 + } + if index == name_start { + return false + } + index = content_type_skip_optional_whitespace(value, index) + if index >= value.length() || value[index] != ('=' : UInt16) { + return false + } + index = content_type_skip_optional_whitespace(value, index + 1) + if index >= value.length() { + return false + } + if value[index] == ('"' : UInt16) { + index = index + 1 + let mut closed = false + while index < value.length() { + let character = value[index] + if character == ('"' : UInt16) { + index = index + 1 + closed = true + break + } + if character == ('\\' : UInt16) { + index = index + 1 + if index >= value.length() || + value[index] < 0x20 || + value[index] >= 0x7f { + return false + } + } else if character < 0x20 || character >= 0x7f { + return false + } + index = index + 1 + } + if !closed { + return false + } + } else { + let value_start = index + while index < value.length() && content_type_token_character(value[index]) { + index = index + 1 + } + if index == value_start { + return false + } + } + } + false +} + +///| +fn content_type_part_name_is_valid( + value : StringView, + cancelled : () -> Bool, +) -> Bool raise ParseXmlError { + if value.length() <= 1 || + !value.has_prefix("/") || + value.has_prefix("//") || + value.has_suffix("/") { + return false + } + for segment in value[1:].split("/") { + if !is_valid_opc_part_segment_cancellable_impl(segment, cancelled) { + return false + } + } + true +} + +///| +fn content_type_extension_is_valid(value : StringView) -> Bool { + value != "" && + value.length() <= 1024 && + !value.contains(".") && + !value.contains("/") && + !value.contains("\\") && + is_valid_opc_part_segment(value) +} + +///| +fn content_type_validate_attributes( + scanner : XmlStartTagScanner, + allowed : ArrayView[String], + label : StringView, +) -> Unit raise ParseXmlError { + for attribute in scanner.expanded_attribute_names() { + if attribute.namespace_uri == "" { + let mut matched = false + for name in allowed { + if attribute.local_name == name { + matched = true + break + } + } + if matched { + continue + } + } + raise InvalidXml( + msg="content types \{label.to_owned()} has a disallowed attribute", + ) + } +} + ///| /// Returns the ASCII-case-insensitive identity key used for OPC part names. /// The leading slash, when present, remains part of the key. @@ -924,15 +1257,30 @@ fn parse_content_types( scanner.local_name() != "Types" { raise InvalidXml(msg="content types document element invalid") } + content_type_validate_attributes(scanner, [], "document element") + let mut records = 0 + let mut retained_chars = 0 while scanner.next() { if scanner.namespace_uri() != package_content_types_namespace { - continue + raise InvalidXml(msg="content types element is invalid") } if scanner.depth() != 2 { raise InvalidXml(msg="content type record is not a direct child") } + records = records + 1 + if records > max_content_type_records { + raise InvalidXml(msg="content type record limit exceeded") + } + if !scanner.current_element_content_is_empty() { + raise InvalidXml(msg="content type declaration is not empty") + } match scanner.local_name() { "Override" => { + content_type_validate_attributes( + scanner, + ["PartName", "ContentType"], + "override", + ) let part_name = match scanner.attribute("", "PartName") { Some(value) => collapse_schema_whitespace(value) None => @@ -946,7 +1294,25 @@ fn parse_content_types( if part_name == "" || content_type == "" { raise InvalidXml(msg="content type override value is empty") } - let part_name_key = package_part_name_key(part_name) + if part_name.length() > max_content_type_attribute_chars || + content_type.length() > max_content_type_attribute_chars { + raise InvalidXml(msg="content type override value is too long") + } + if !content_type_part_name_is_valid(part_name, cancelled) { + raise InvalidXml(msg="content type override part name is invalid") + } + if !content_type_media_type_is_valid(content_type) { + raise InvalidXml(msg="content type override media type is invalid") + } + let retained = part_name.length() + content_type.length() + if retained > max_content_type_retained_chars - retained_chars { + raise InvalidXml(msg="content type retained text limit exceeded") + } + retained_chars = retained_chars + retained + let part_name_key = package_part_name_key_cancellable( + part_name, + cancelled~, + ) if overrides.contains(part_name_key) { raise InvalidXml(msg="duplicate content type override") } @@ -954,6 +1320,11 @@ fn parse_content_types( override_part_names[part_name_key] = part_name } "Default" => { + content_type_validate_attributes( + scanner, + ["Extension", "ContentType"], + "default", + ) let extension = match scanner.attribute("", "Extension") { Some(value) => collapse_schema_whitespace(value).to_lower() None => raise InvalidXml(msg="content type default extension missing") @@ -966,6 +1337,18 @@ fn parse_content_types( if extension == "" || content_type == "" { raise InvalidXml(msg="content type default value is empty") } + if !content_type_extension_is_valid(extension) { + raise InvalidXml(msg="content type default extension is invalid") + } + if content_type.length() > max_content_type_attribute_chars || + !content_type_media_type_is_valid(content_type) { + raise InvalidXml(msg="content type default media type is invalid") + } + let retained = extension.length() + content_type.length() + if retained > max_content_type_retained_chars - retained_chars { + raise InvalidXml(msg="content type retained text limit exceeded") + } + retained_chars = retained_chars + retained if defaults.contains(extension) { raise InvalidXml(msg="duplicate content type default") } @@ -974,6 +1357,9 @@ fn parse_content_types( _ => raise InvalidXml(msg="content types element is invalid") } } + if records == 0 { + raise InvalidXml(msg="content types document has no declarations") + } { overrides, override_part_names, defaults } } @@ -1055,7 +1441,7 @@ test "ooxml read_parse wb: every relationship record is validated" { #| #| #| - try parse_internal_relationship_targets(xml, "t1") catch { + try parse_internal_relationship_targets(xml, "urn:t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship type missing") _ => fail("unexpected relationship parse cancellation") } noraise { diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 890b03ce..305585ac 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -3,12 +3,12 @@ test "ooxml read_parse: internal relationships support self-closing and expanded let xml = #| #| - #| - #| - #| + #| + #| + #| #| #| - let targets = @ooxml.parse_internal_relationship_targets(xml, "t1") + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:t1") debug_inspect( targets, content=( @@ -22,10 +22,10 @@ test "ooxml read_parse: internal relationships report missing id/target and malf let missing_id = #| #| - #| + #| #| #| - try @ooxml.parse_internal_relationship_targets(missing_id, "t1") catch { + try @ooxml.parse_internal_relationship_targets(missing_id, "urn:t1") catch { InvalidXml(msg~) => inspect(msg, content="relationship id missing") _ => fail("unexpected relationship parse cancellation") } noraise { @@ -35,10 +35,12 @@ test "ooxml read_parse: internal relationships report missing id/target and malf let missing_target = #| #| - #| + #| #| #| - try @ooxml.parse_internal_relationship_targets(missing_target, "t1") catch { + try + @ooxml.parse_internal_relationship_targets(missing_target, "urn:t1") + catch { InvalidXml(msg~) => inspect(msg, content="relationship target missing") _ => fail("unexpected relationship parse cancellation") } noraise { @@ -51,7 +53,7 @@ test "ooxml read_parse: internal relationships report missing id/target and malf #| #| - try @ooxml.parse_internal_relationship_targets(malformed, "t1") catch { + try @ooxml.parse_internal_relationship_targets(malformed, "urn:t1") catch { InvalidXml(msg~) => inspect(msg, content="XML start tag is not closed") _ => fail("unexpected relationship parse cancellation") } noraise { @@ -63,13 +65,13 @@ test "ooxml read_parse: internal relationships report missing id/target and malf test "ooxml read_parse: prefixed relationships decode entities exactly once" { let xml = #| - #| + #| #| let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:type1") debug_inspect( targets, content=( - #|{ "r&1": "custom/a b.xml" } + #|{ "r1": "custom/a b.xml" } ), ) } @@ -79,13 +81,13 @@ test "ooxml read_parse: uppercase-X character references never change relationsh for xml in [ ( - #| + #| ), ( - #| + #| ), ] { - try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { InvalidXml(msg~) => inspect(msg, content="XML entity is invalid") _ => fail("unexpected uppercase-X entity error") } noraise { @@ -95,31 +97,62 @@ test "ooxml read_parse: uppercase-X character references never change relationsh } ///| -test "ooxml read_parse: relationship schema whitespace is collapsed" { - let xml = - #| - #| - #| - let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:type one") - debug_inspect(targets, content="{ \"r 1\": \"custom/a b.xml\" }") +test "ooxml read_parse: relationship identifiers and URIs fail closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "relationship id is invalid", + ), + ( + ( + #| + ), + "relationship type is invalid", + ), + ( + ( + #| + ), + "relationship target is invalid", + ), + ] + for case in cases { + let (xml, expected) = case + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected relationship grammar error") + } noraise { + _ => fail("expected invalid relationship grammar") + } + } } ///| test "ooxml read_parse: relationship target modes are explicit and filtered" { let xml = #| - #| - #| - #| + #| + #| + #| #| debug_inspect( - @ooxml.parse_internal_relationship_targets(xml, "wanted"), + @ooxml.parse_internal_relationship_targets(xml, "urn:wanted"), content=( #|{ "default": "inside.xml", "internal": "also-inside.xml" } ), ) + let padded = + #| + try @ooxml.parse_internal_relationship_targets(padded, "urn:wanted") catch { + InvalidXml(msg~) => inspect(msg, content="relationship target mode invalid") + _ => fail("unexpected padded relationship target mode error") + } noraise { + _ => fail("expected padded relationship target mode rejection") + } debug_inspect( - @ooxml.parse_external_relationship_targets(xml, "wanted"), + @ooxml.parse_external_relationship_targets(xml, "urn:wanted"), content=( #|{ "external": "https://example.invalid" } ), @@ -129,44 +162,44 @@ test "ooxml read_parse: relationship target modes are explicit and filtered" { ///| test "ooxml read_parse: relationships apply OPC Markup Compatibility" { let process_content = - #| + #| let selected_choice = - #| + #| let selected_fallback = - #| + #| let wildcard_process_content = - #| + #| let empty_directives = - #| + #| let understood_ignorable = - #| + #| let processed = @ooxml.parse_internal_relationship_targets( - process_content, "wanted", + process_content, "urn:wanted", ) assert_eq(processed.get("processed"), Some("processed.xml")) assert_eq(processed.length(), 1) let choice = @ooxml.parse_internal_relationship_targets( - selected_choice, "wanted", + selected_choice, "urn:wanted", ) assert_eq(choice.get("choice"), Some("choice.xml")) assert_eq(choice.length(), 1) let fallback = @ooxml.parse_internal_relationship_targets( - selected_fallback, "wanted", + selected_fallback, "urn:wanted", ) assert_eq(fallback.get("fallback"), Some("fallback.xml")) assert_eq(fallback.length(), 1) let wildcard = @ooxml.parse_internal_relationship_targets( - wildcard_process_content, "wanted", + wildcard_process_content, "urn:wanted", ) assert_eq(wildcard.get("wildcard"), Some("wildcard.xml")) assert_eq(wildcard.length(), 1) let empty = @ooxml.parse_internal_relationship_targets( - empty_directives, "wanted", + empty_directives, "urn:wanted", ) assert_eq(empty.get("empty"), Some("empty.xml")) assert_eq(empty.length(), 1) let understood = @ooxml.parse_internal_relationship_targets( - understood_ignorable, "wanted", + understood_ignorable, "urn:wanted", ) assert_eq(understood.get("understood"), Some("understood.xml")) assert_eq(understood.length(), 1) @@ -191,7 +224,7 @@ test "ooxml read_parse: MCE names retain one copy of a long namespace URI" { "\" mc:Ignorable=\"v\" mc:ProcessContent=\"" + names.to_string() + "\"/>" - let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") assert_eq(targets.length(), 0) } @@ -260,7 +293,7 @@ test "ooxml read_parse: invalid relationship MCE fails closed" { ), ( ( - #| + #| ), "MCE ProcessContent element has an unsafe xml attribute", ), @@ -273,7 +306,7 @@ test "ooxml read_parse: invalid relationship MCE fails closed" { ] for case in cases { let (xml, expected) = case - try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { InvalidXml(msg~) => assert_eq(msg, expected) _ => fail("unexpected relationship parse cancellation") } noraise { @@ -285,8 +318,8 @@ test "ooxml read_parse: invalid relationship MCE fails closed" { ///| test "ooxml read_parse: unselected MCE choices remain opaque" { let xml = - #| - let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + #| + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") assert_eq(targets.get("selected"), Some("selected.xml")) assert_eq(targets.length(), 1) } @@ -294,8 +327,8 @@ test "ooxml read_parse: unselected MCE choices remain opaque" { ///| test "ooxml read_parse: unselected Choice does not enforce MustUnderstand" { let xml = - #| - let targets = @ooxml.parse_internal_relationship_targets(xml, "wanted") + #| + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") assert_eq(targets.get("ok"), Some("ok.xml")) assert_eq(targets.length(), 1) } @@ -305,28 +338,34 @@ test "ooxml read_parse: MCE container grammar fails closed" { let cases : Array[(String, String)] = [ ( ( - #| + #| ), "MCE ProcessContent name is invalid", ), ( ( - #| + #| ), "MCE ProcessContent name is invalid", ), ( ( - #| + #| ), "MCE AlternateContent has a disallowed attribute", ), ( ( - #| + #| ), "MCE Choice has a disallowed attribute", ), + ( + ( + #| + ), + "MCE AlternateContent has a disallowed attribute", + ), ( ( #| @@ -335,14 +374,14 @@ test "ooxml read_parse: MCE container grammar fails closed" { ), ( ( - #| + #| ), "MCE AlternateContent child is invalid", ), ] for case in cases { let (xml, expected) = case - try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { InvalidXml(msg~) => assert_eq(msg, expected) _ => fail("unexpected relationship parse cancellation") } noraise { @@ -351,16 +390,43 @@ test "ooxml read_parse: MCE container grammar fails closed" { } } +///| +test "ooxml read_parse: relationship schema attributes are closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "relationships document element has a disallowed attribute", + ), + ( + ( + #| + ), + "relationships record has a disallowed attribute", + ), + ] + for case in cases { + let (xml, expected) = case + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected relationship attribute error") + } noraise { + _ => fail("expected relationship attribute rejection") + } + } +} + ///| test "ooxml read_parse: invalid target modes fail closed" { let xml = - #| + #| for external in [false, true] { try { if external { - ignore(@ooxml.parse_external_relationship_targets(xml, "wanted")) + ignore(@ooxml.parse_external_relationship_targets(xml, "urn:wanted")) } else { - ignore(@ooxml.parse_internal_relationship_targets(xml, "wanted")) + ignore(@ooxml.parse_internal_relationship_targets(xml, "urn:wanted")) } } catch { InvalidXml(msg~) => @@ -377,13 +443,13 @@ test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay for xml in [ ( - #| + #| ), ( - #| + #| ), ] { - try @ooxml.parse_internal_relationship_targets(xml, "wanted") catch { + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { InvalidXml(_) => () _ => fail("unexpected relationship parse cancellation") } noraise { @@ -400,17 +466,17 @@ test "ooxml read_parse: relationship limits accept writer boundaries" { ) for index in 0..<16_385 { xml.write_string( - "", + "", ) } let long_target = "x".repeat(4097) xml.write_string( - "", + "", ) xml.write_string("") let targets = @ooxml.parse_internal_relationship_targets( xml.to_string(), - "wanted", + "urn:wanted", ) assert_eq(targets.get("wanted"), Some(long_target)) } @@ -423,13 +489,13 @@ test "ooxml read_parse: relationship scans poll cancellation internally" { ) for index in 0..<2048 { xml.write_string( - "", + "", ) } xml.write_string("") let checks = [0] try - @ooxml.parse_internal_relationship_targets(xml.to_string(), "wanted", cancelled=() => { + @ooxml.parse_internal_relationship_targets(xml.to_string(), "urn:wanted", cancelled=() => { checks[0] += 1 checks[0] >= 4 }) @@ -552,3 +618,60 @@ test "ooxml read_parse: content type overrides reject case-variant duplicates" { _ => fail("expected case-variant duplicate override rejection") } } + +///| +test "ooxml read_parse: content type grammar fails closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "content types document element has a disallowed attribute", + ), + ( + ( + #| + ), + "content types default has a disallowed attribute", + ), + ( + ( + #| + ), + "content type default extension is invalid", + ), + ( + ( + #| + ), + "content type default media type is invalid", + ), + ( + ( + #| + ), + "content type override part name is invalid", + ), + ( + ( + #|payload + ), + "content type declaration is not empty", + ), + ( + ( + #| + ), + "content types document has no declarations", + ), + ] + for case in cases { + let (xml, expected) = case + try @ooxml.find_part_content_type(xml, "xl/workbook.xml") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected content type grammar error") + } noraise { + _ => fail("expected content type grammar rejection") + } + } +} diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 4403edc1..9aabeb7d 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -1966,6 +1966,96 @@ fn XmlStartTagScanner::expanded_attribute_names( output } +///| +/// Reports whether the current element has no element or non-whitespace text +/// content. The scanner still performs the authoritative closing-tag and XML +/// validation as iteration continues; this helper only enforces schema +/// declarations whose content model is empty. +fn XmlStartTagScanner::current_element_content_is_empty( + self : XmlStartTagScanner, +) -> Bool raise ParseXmlError { + if self.current_self_closing { + return true + } + let mut index = self.current_tag_end + 1 + let mut next_checkpoint = index + while index < self.xml.length() { + if index >= next_checkpoint { + check_xml_cancelled(self.cancelled) + next_checkpoint = index + 4096 + } + let unit = self.xml[index] + if is_attr_space(unit) { + index = index + 1 + continue + } + if unit == ('&' : UInt16) { + let entity_end = match + find_xml_sequence(self.xml, index + 1, ";", cancelled=self.cancelled) { + Some(value) => value + None => raise InvalidXml(msg="XML entity is not closed") + } + let character = match xml_entity_value(self.xml[index + 1:entity_end]) { + Some(value) => value + None => raise InvalidXml(msg="XML entity is invalid") + } + if character != ' ' && + character != '\t' && + character != '\n' && + character != '\r' { + return false + } + index = entity_end + 1 + continue + } + if unit != ('<' : UInt16) || index + 1 >= self.xml.length() { + return false + } + if self.xml[index + 1] == ('/' : UInt16) { + return true + } + if index + 4 <= self.xml.length() && + xml_span_equals(self.xml, index, index + 4, "", cancelled=self.cancelled) { + Some(value) => value + None => raise InvalidXml(msg="XML comment is not closed") + } + index = comment_end + 3 + continue + } + if self.xml[index + 1] == ('?' : UInt16) { + let instruction_end = match + find_xml_sequence(self.xml, index + 2, "?>", cancelled=self.cancelled) { + Some(value) => value + None => raise InvalidXml(msg="XML processing instruction is not closed") + } + index = instruction_end + 2 + continue + } + if index + 9 <= self.xml.length() && + xml_span_equals(self.xml, index, index + 9, "", cancelled=self.cancelled) { + Some(value) => value + None => raise InvalidXml(msg="XML CDATA section is not closed") + } + for character in self.xml[index + 9:cdata_end] { + if character != ' ' && + character != '\t' && + character != '\n' && + character != '\r' { + return false + } + } + index = cdata_end + 3 + continue + } + return false + } + false +} + ///| fn XmlStartTagScanner::current_namespace_declarations( self : XmlStartTagScanner, diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 5b5a448c..1339a23a 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -299,12 +299,12 @@ test "ooxml_rels: internal targets filter by Type and map Id->Target" { let xml = #| #| - #| - #| - #| + #| + #| + #| #| #| - let targets = parse_internal_relationship_targets(xml, "t1") + let targets = parse_internal_relationship_targets(xml, "urn:t1") debug_inspect( targets, content=( @@ -317,17 +317,17 @@ test "ooxml_rels: internal targets filter by Type and map Id->Target" { test "ooxml_rels: target modes separate package parts from external resources" { let xml = #| - #| - #| + #| + #| #| debug_inspect( - parse_internal_relationship_targets(xml, "t1"), + parse_internal_relationship_targets(xml, "urn:t1"), content=( #|{ "inside": "worksheets/sheet1.xml" } ), ) debug_inspect( - parse_external_relationship_targets(xml, "t1"), + parse_external_relationship_targets(xml, "urn:t1"), content=( #|{ "outside": "https://example.invalid/sheet.xml" } ), From 495cd95da2d19b8f74d3d70a80372102f53be14a Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 02:08:38 +0800 Subject: [PATCH 111/150] fix(xlsx): resolve relationships with bounded OPC paths --- ooxml/opc_part_name.mbt | 9 +++ ooxml/pkg.generated.mbti | 2 + xlsx/cell_images.mbt | 52 +++++++++---- xlsx/header_footer_image_read.mbt | 5 +- xlsx/ooxml_rels.mbt | 109 +++++++++++++++++++++----- xlsx/read.mbt | 125 +++++++++++++++++++++++------- xlsx/read_drawing_xml.mbt | 10 ++- xlsx/read_package_parts.mbt | 4 +- xlsx/read_sheet_rel_parts.mbt | 6 +- xlsx/rich_value_images.mbt | 53 ++++++++++++- xlsx/rich_value_images_test.mbt | 10 +-- 11 files changed, 307 insertions(+), 78 deletions(-) diff --git a/ooxml/opc_part_name.mbt b/ooxml/opc_part_name.mbt index dab164c6..51854a1e 100644 --- a/ooxml/opc_part_name.mbt +++ b/ooxml/opc_part_name.mbt @@ -186,6 +186,15 @@ pub fn is_valid_opc_part_segment(segment : StringView) -> Bool { try! is_valid_opc_part_segment_cancellable_impl(segment, () => false) } +///| +/// Cancellation-aware form of `is_valid_opc_part_segment`. +pub fn is_valid_opc_part_segment_cancellable( + segment : StringView, + cancelled? : () -> Bool = () => false, +) -> Bool raise ParseXmlError { + is_valid_opc_part_segment_cancellable_impl(segment, cancelled) +} + ///| /// Maps an ASCII physical ZIP item name to its logical OPC PartName. Literal /// non-ASCII IRI scalars are restored from their canonical UTF-8 `%HH` diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index dbb1cbcc..3794a4ee 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -22,6 +22,8 @@ pub fn find_part_content_type(StringView, StringView, cancelled? : () -> Bool) - pub fn is_valid_opc_part_segment(StringView) -> Bool +pub fn is_valid_opc_part_segment_cancellable(StringView, cancelled? : () -> Bool) -> Bool raise ParseXmlError + pub fn logical_opc_part_name_from_zip_item_name(StringView) -> String? pub fn logical_opc_part_name_from_zip_item_name_cancellable(StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index d2ccb0fc..fb23d3df 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -20,6 +20,8 @@ struct CellImage { fn parse_cell_images( cell_images_xml : StringView, rels_xml : StringView, + source_part : StringView, + part_names : Map[String, String], archive : @zip.Archive, budget? : ReadBudget, cancelled? : () -> Bool = () => false, @@ -103,7 +105,12 @@ fn parse_cell_images( Some(value) => value None => continue } - let media_path = normalize_cell_image_target(target) + let media_path = actual_relationship_target_path( + source_part, + target, + part_names, + cancelled~, + ) match archive.get(media_path) { Some(data) => { let extension = cell_image_extension(media_path) @@ -125,21 +132,6 @@ fn parse_cell_images( images } -///| -/// Resolves a cellimages relationship target to an archive path. WPS -/// targets are workbook-relative (e.g. "media/image1.png"); a leading -/// "/" is absolute and a "../" prefix is rewritten to "xl", matching -/// Excelize's `"xl/" + r.Target` / package path handling. -fn normalize_cell_image_target(target : String) -> String { - if target.has_prefix("/") { - target[1:].to_owned() - } else if target.has_prefix("../") { - "xl/" + target[3:].to_owned() - } else { - "xl/" + target - } -} - ///| /// Returns the file extension (with leading dot) of a media path, or the /// empty string when there is none. @@ -244,8 +236,34 @@ test "parse_cell_images handles unprefixed blip without matching blipFill" { #| let archive = @zip.Archive::new() archive.add("xl/media/u.png", b"\x89PNGunprefixed") - let images = parse_cell_images(cellimages_xml, rels_xml, archive) + let images = parse_cell_images( + cellimages_xml, + rels_xml, + "xl/cellimages.xml", + archive_part_name_index(archive), + archive, + ) inspect(images.length(), content="1") inspect(images[0].name, content="ID_U") inspect(images[0].extension, content=".png") } + +///| +test "parse_cell_images resolves targets relative to its source part" { + let cellimages_xml = + #| + let rels_xml = + #| + let archive = @zip.Archive::new() + archive.add("decoy.png", b"root image") + archive.add("xl/decoy.png", b"wrong image") + let images = parse_cell_images( + cellimages_xml, + rels_xml, + "xl/cellimages.xml", + archive_part_name_index(archive), + archive, + ) + assert_eq(images.length(), 1) + assert_eq(images[0].data, b"root image") +} diff --git a/xlsx/header_footer_image_read.mbt b/xlsx/header_footer_image_read.mbt index d50d625d..596680bc 100644 --- a/xlsx/header_footer_image_read.mbt +++ b/xlsx/header_footer_image_read.mbt @@ -137,7 +137,10 @@ fn parse_header_footer_images_from_vml( None => raise InvalidXml(msg="header/footer image relationship missing") } let image_path = actual_relationship_target_path( - vml_part, target, part_names, + vml_part, + target, + part_names, + cancelled~, ) let image_bytes = match archive.get(image_path) { Some(value) => value diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 1339a23a..0b92e1bf 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -4,6 +4,12 @@ let transitional_office_relationship_prefix = "http://schemas.openxmlformats.org ///| let strict_office_relationship_prefix = "http://purl.oclc.org/ooxml/officeDocument/relationships/" +///| +let max_relationship_part_path_chars : Int = 64 * 1024 + +///| +let max_relationship_part_path_segments = 4096 + ///| fn parse_internal_relationship_targets_exact( xml : StringView, @@ -188,8 +194,11 @@ fn relationship_target_has_ambiguous_first_segment_colon( ///| fn validate_relationship_target_uri( target : StringView, + cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { + check_read_cancelled(cancelled) if target == "" || + target.length() > max_relationship_part_path_chars || target.has_prefix("//") || target.contains("\\") || target.contains("?") || @@ -200,11 +209,32 @@ fn validate_relationship_target_uri( } ///| -fn normalize_rel_part_path(path : StringView) -> String raise XlsxError { +fn normalize_rel_part_path( + path : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + check_read_cancelled(cancelled) + if path == "" || path.length() > max_relationship_part_path_chars { + raise InvalidXml(msg="relationship target invalid") + } let segments : Array[String] = [] - let chunks = path.split("/").collect() - for index, chunk in chunks { - let segment = chunk.to_owned() + let mut segment_start = 0 + let mut segment_count = 0 + let mut index = 0 + while index <= path.length() { + if index < path.length() && path[index] != ('/' : UInt16) { + index = index + 1 + continue + } + segment_count = segment_count + 1 + if segment_count > max_relationship_part_path_segments { + raise InvalidXml(msg="relationship target segment limit exceeded") + } + if (segment_count & 127) == 1 { + check_read_cancelled(cancelled) + } + let segment = path[segment_start:index] + let terminal = index == path.length() if segment == "" { // Empty URI path segments are significant. Silently removing them lets // malformed targets such as `xl//workbook.xml` alias valid package parts. @@ -212,22 +242,29 @@ fn normalize_rel_part_path(path : StringView) -> String raise XlsxError { } if segment == "." { // A terminal dot segment resolves to a directory URI, not an OPC part. - if index + 1 == chunks.length() { + if terminal { raise InvalidXml(msg="relationship target invalid") } - continue - } - if segment == ".." { - if segments.length() == 0 || index + 1 == chunks.length() { + } else if segment == ".." { + if segments.length() == 0 || terminal { raise InvalidXml(msg="relationship target invalid") } ignore(segments.pop()) - continue - } - if !@ooxml.is_valid_opc_part_segment(segment) { - raise InvalidXml(msg="relationship target invalid") + } else { + let valid = @ooxml.is_valid_opc_part_segment_cancellable( + segment, + cancelled~, + ) catch { + ReadCancelled => raise ReadCancelled + InvalidXml(msg~) => raise InvalidXml(msg~) + } + if !valid { + raise InvalidXml(msg="relationship target invalid") + } + segments.push(segment.to_owned()) } - segments.push(segment) + segment_start = index + 1 + index = index + 1 } if segments.length() == 0 { raise InvalidXml(msg="relationship target invalid") @@ -239,14 +276,15 @@ fn normalize_rel_part_path(path : StringView) -> String raise XlsxError { fn resolve_part_rel_target( source_part : StringView, target : StringView, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let source = source_part.to_owned() let target_str = target.to_owned() if source == "" || source.has_prefix("/") || target_str == "" { raise InvalidXml(msg="relationship target invalid") } - validate_relationship_target_uri(target_str) - let normalized_source = normalize_rel_part_path(source) + validate_relationship_target_uri(target_str, cancelled~) + let normalized_source = normalize_rel_part_path(source, cancelled~) if normalized_source != source { raise InvalidXml(msg="relationship target invalid") } @@ -258,16 +296,17 @@ fn resolve_part_rel_target( None => target_str } } - normalize_rel_part_path(raw) + normalize_rel_part_path(raw, cancelled~) } ///| fn resolve_rel_target( target : StringView, base_folder : StringView, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let target_str = target.to_owned() - validate_relationship_target_uri(target_str) + validate_relationship_target_uri(target_str, cancelled~) let raw = if target_str.has_prefix("/") { drop_first_path_segment(target_str) } else if target_str.has_prefix("../") { @@ -282,7 +321,7 @@ fn resolve_rel_target( "xl/" + base + "/" + target_str } } - let normalized = normalize_rel_part_path(raw) + let normalized = normalize_rel_part_path(raw, cancelled~) if !normalized.has_prefix("xl/") { raise InvalidXml(msg="relationship target invalid") } @@ -290,8 +329,11 @@ fn resolve_rel_target( } ///| -fn resolve_workbook_rel_target(target : StringView) -> String raise XlsxError { - resolve_part_rel_target(workbook_part_path, target) +fn resolve_workbook_rel_target( + target : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + resolve_part_rel_target(workbook_part_path, target, cancelled~) } ///| @@ -524,3 +566,28 @@ test "ooxml_rels: invalid empty and terminal path segments never alias parts" { } } } + +///| +test "ooxml_rels: normalization bounds segments and polls cancellation" { + let many_segments = "a/".repeat(max_relationship_part_path_segments) + "z" + try normalize_rel_part_path(many_segments) catch { + InvalidXml(msg~) => + inspect(msg, content="relationship target segment limit exceeded") + _ => fail("unexpected relationship segment-limit error") + } noraise { + _ => fail("expected relationship segment-limit rejection") + } + + let checks = [0] + try + normalize_rel_part_path("a".repeat(60 * 1024), cancelled=() => { + checks[0] = checks[0] + 1 + checks[0] >= 4 + }) + catch { + ReadCancelled => assert_true(checks[0] >= 4) + _ => fail("unexpected relationship normalization cancellation error") + } noraise { + _ => fail("expected relationship normalization cancellation") + } +} diff --git a/xlsx/read.mbt b/xlsx/read.mbt index fbd23887..e91a8d79 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3086,9 +3086,15 @@ fn first_existing_rel_target_path( targets : Map[String, String], source_part : StringView, part_names : Map[String, String], + cancelled? : () -> Bool = () => false, ) -> String? raise XlsxError { for _, target in targets { - let path = actual_relationship_target_path(source_part, target, part_names) + let path = actual_relationship_target_path( + source_part, + target, + part_names, + cancelled~, + ) // `path` is the physical ZIP spelling when the target exists. The archive // index is keyed in logical OPC PartName space, so restore percent-encoded // Unicode before testing identity. @@ -3105,11 +3111,13 @@ fn first_existing_workbook_rel_target_path( targets : Map[String, String], workbook_part : StringView, part_names : Map[String, String], + cancelled? : () -> Bool = () => false, ) -> String? raise XlsxError { for _, target in targets { let path = resolve_part_rel_target( logical_archive_part_path(workbook_part), target, + cancelled~, ) match part_names.get(@ooxml.package_part_name_key(path)) { Some(actual) => return Some(actual) @@ -3409,10 +3417,12 @@ fn actual_relationship_target_path( source_part : StringView, target : StringView, part_names : Map[String, String], + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let source_relative = resolve_part_rel_target( logical_archive_part_path(source_part), target, + cancelled~, ) part_names .get(@ooxml.package_part_name_key(source_relative)) @@ -3456,7 +3466,12 @@ fn workbook_related_part_path( cancelled~, ) match - first_existing_workbook_rel_target_path(targets, workbook_part, part_names) { + first_existing_workbook_rel_target_path( + targets, + workbook_part, + part_names, + cancelled~, + ) { Some(path) => Some(path) None => match first_relationship_target(targets) { @@ -3467,6 +3482,7 @@ fn workbook_related_part_path( resolve_part_rel_target( logical_archive_part_path(workbook_part), target, + cancelled~, ), ), ) @@ -3756,15 +3772,21 @@ fn read_zip_archive_core_unchecked( let calc_props = parse_calc_props(workbook_core_xml) let workbook_protection = parse_workbook_protection(workbook_core_xml) let style_count = styles.length() - let cell_images = match archive.get("xl/cellimages.xml") { + let cell_images_part = actual_archive_part_path( + part_names, "xl/cellimages.xml", + ) + let cell_images = match archive.get(cell_images_part) { Some(value) => { - let rels_xml = match archive.get("xl/_rels/cellimages.xml.rels") { + let rels_xml = match + archive.get(actual_relationship_part_path(cell_images_part, part_names)) { Some(rels) => decode(rels) None => "" } parse_cell_images( decode(value), rels_xml, + cell_images_part, + part_names, archive, budget=read_budget, cancelled~, @@ -3774,6 +3796,7 @@ fn read_zip_archive_core_unchecked( } let rich_value_images = parse_rich_value_images( archive, + part_names, decode, budget=read_budget, cancelled~, @@ -3782,16 +3805,14 @@ fn read_zip_archive_core_unchecked( match rich_value_images { Some(data) => { for _, target in data.rel_targets { - let path = normalize_cell_image_target(target) - match archive.get(path) { - Some(bytes) => rich_value_media[path] = bytes.to_owned() + match archive.get(target) { + Some(bytes) => rich_value_media[target] = bytes.to_owned() None => () } } for _, target in data.web_rel_targets { - let path = normalize_cell_image_target(target) - match archive.get(path) { - Some(bytes) => rich_value_media[path] = bytes.to_owned() + match archive.get(target) { + Some(bytes) => rich_value_media[target] = bytes.to_owned() None => () } } @@ -3847,7 +3868,10 @@ fn read_zip_archive_core_unchecked( ) let vba_path = match first_existing_workbook_rel_target_path( - vba_targets, workbook_xml_part_path, part_names, + vba_targets, + workbook_xml_part_path, + part_names, + cancelled~, ) { Some(path) => Some(path) None => @@ -3859,6 +3883,7 @@ fn read_zip_archive_core_unchecked( resolve_part_rel_target( logical_archive_part_path(workbook_xml_part_path), target, + cancelled~, ), ), ) @@ -3912,6 +3937,7 @@ fn read_zip_archive_core_unchecked( resolve_part_rel_target( logical_archive_part_path(workbook_xml_part_path), target, + cancelled~, ), ) let part_key = @ooxml.package_part_name_key(path) @@ -4077,7 +4103,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="slicer relationship missing") } let slicer_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) if seen_paths.contains(slicer_path) { continue @@ -4108,7 +4137,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="picture target missing") } let image_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let image_bytes = match archive.get(image_path) { Some(value) => value @@ -4197,7 +4229,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="table relationship missing") } let table_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let table_bytes = match archive.get(table_path) { Some(value) => value @@ -4239,7 +4274,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="pivot table relationship missing") } let pivot_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let pivot_bytes = match archive.get(pivot_path) { Some(value) => value @@ -4262,7 +4300,10 @@ fn read_zip_archive_core_unchecked( ) let cache_path = match first_existing_rel_target_path( - cache_targets, pivot_path, part_names, + cache_targets, + pivot_path, + part_names, + cancelled~, ) { Some(path) => path None => { @@ -4273,7 +4314,10 @@ fn read_zip_archive_core_unchecked( raise InvalidXml(msg="pivot cache relationship missing") } actual_relationship_target_path( - pivot_path, cache_target, part_names, + pivot_path, + cache_target, + part_names, + cancelled~, ) } } @@ -4300,7 +4344,10 @@ fn read_zip_archive_core_unchecked( ) let record_path = match first_existing_rel_target_path( - record_targets, cache_path, part_names, + record_targets, + cache_path, + part_names, + cancelled~, ) { Some(path) => Some(path) None => @@ -4308,7 +4355,10 @@ fn read_zip_archive_core_unchecked( Some(record_target) => Some( actual_relationship_target_path( - cache_path, record_target, part_names, + cache_path, + record_target, + part_names, + cancelled~, ), ) None => None @@ -4357,7 +4407,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="vml drawing relationship missing") } let vml_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let vml_bytes = match archive.get(vml_path) { Some(value) => value @@ -4374,7 +4427,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="vml drawing hf relationship missing") } let vml_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let vml_bytes = match archive.get(vml_path) { Some(value) => value @@ -4420,7 +4476,10 @@ fn read_zip_archive_core_unchecked( let parsed_comments : Array[Comment] = [] for _rel_id, target in comment_targets { let comment_path = actual_relationship_target_path( - sheet_path, target, part_names, + sheet_path, + target, + part_names, + cancelled~, ) let comment_bytes = match archive.get(comment_path) { Some(value) => value @@ -4502,7 +4561,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="drawing target missing") } let drawing_path = actual_relationship_target_path( - sheet_path, drawing_target, part_names, + sheet_path, + drawing_target, + part_names, + cancelled~, ) let drawing_bytes = match archive.get(drawing_path) { Some(value) => value @@ -4684,7 +4746,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="drawing target missing") } let drawing_path = actual_relationship_target_path( - chartsheet_path, drawing_target, part_names, + chartsheet_path, + drawing_target, + part_names, + cancelled~, ) let drawing_rels_path = actual_relationship_part_path( drawing_path, part_names, @@ -4701,7 +4766,12 @@ fn read_zip_archive_core_unchecked( cancelled~, ) let chart_path = match - first_existing_rel_target_path(chart_targets, drawing_path, part_names) { + first_existing_rel_target_path( + chart_targets, + drawing_path, + part_names, + cancelled~, + ) { Some(path) => path None => { let chart_target = match first_relationship_target(chart_targets) { @@ -4709,7 +4779,10 @@ fn read_zip_archive_core_unchecked( None => raise InvalidXml(msg="chart target missing") } actual_relationship_target_path( - drawing_path, chart_target, part_names, + drawing_path, + chart_target, + part_names, + cancelled~, ) } } diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 89490c27..470842f5 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -355,7 +355,10 @@ fn parse_drawing_images( None => raise InvalidXml(msg="drawing image target missing") } let media_path = actual_relationship_target_path( - drawing_part, target, part_names, + drawing_part, + target, + part_names, + cancelled~, ) let data = match archive.get(media_path) { Some(value) => value @@ -455,7 +458,10 @@ fn parse_drawing_charts( None => raise InvalidXml(msg="drawing chart target missing") } let chart_path = actual_relationship_target_path( - drawing_part, target, part_names, + drawing_part, + target, + part_names, + cancelled~, ) let chart_bytes = match archive.get(chart_path) { Some(value) => value diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index b94174a2..d73db49b 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -32,9 +32,9 @@ fn workbook_part_from_root_relationships( } match first_relationship_target(targets) { Some(target) => { - validate_relationship_target_uri(target) + validate_relationship_target_uri(target, cancelled~) let part_name = archive_path_from_part_name(target) - normalize_rel_part_path(part_name) + normalize_rel_part_path(part_name, cancelled~) } None => raise InvalidXml(msg="root workbook relationship missing") } diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index c36b5b9a..fc6bb5f4 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -357,7 +357,11 @@ fn parse_slicer_cache_definitions( for _rel_id, target in targets { let cache_path = actual_archive_part_path( part_names, - resolve_part_rel_target(logical_archive_part_path(workbook_part), target), + resolve_part_rel_target( + logical_archive_part_path(workbook_part), + target, + cancelled~, + ), ) let cache_bytes = match archive.get(cache_path) { Some(value) => value diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index 62d38b55..11262b1e 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -30,6 +30,7 @@ struct RichValueImages { /// when the workbook has no rich value metadata. fn parse_rich_value_images( archive : @zip.Archive, + part_names : Map[String, String], decode : (BytesView) -> String raise XlsxError, budget? : ReadBudget, cancelled? : () -> Bool = () => false, @@ -49,7 +50,7 @@ fn parse_rich_value_images( Some(value) => parse_rich_value_rel_ids(decode(value)) None => [] } - let rel_targets = match + let raw_rel_targets = match archive.get("xl/richData/_rels/richValueRel.xml.rels") { Some(value) => parse_internal_relationship_targets( @@ -60,11 +61,20 @@ fn parse_rich_value_images( ) None => Map([]) } + let rel_targets : Map[String, String] = Map([]) + for rel_id, target in raw_rel_targets { + rel_targets[rel_id] = actual_relationship_target_path( + "xl/richData/richValueRel.xml", + target, + part_names, + cancelled~, + ) + } let web_blip_ids = match archive.get("xl/richData/rdRichValueWebImage.xml") { Some(value) => parse_web_image_blip_ids(decode(value)) None => [] } - let web_rel_targets = match + let raw_web_rel_targets = match archive.get("xl/richData/_rels/rdRichValueWebImage.xml.rels") { Some(value) => parse_internal_relationship_targets( @@ -75,6 +85,15 @@ fn parse_rich_value_images( ) None => Map([]) } + let web_rel_targets : Map[String, String] = Map([]) + for rel_id, target in raw_web_rel_targets { + web_rel_targets[rel_id] = actual_relationship_target_path( + "xl/richData/rdRichValueWebImage.xml", + target, + part_names, + cancelled~, + ) + } Some({ value_metadata, rich_values, @@ -86,6 +105,34 @@ fn parse_rich_value_images( }) } +///| +test "rich value image relationships use source-relative OPC resolution" { + let archive = @zip.Archive::new() + archive.add("xl/metadata.xml", b"") + archive.add("xl/richData/rdrichvalue.xml", b"") + archive.add( + "xl/richData/_rels/richValueRel.xml.rels", + @encoding/utf8.encode( + ( + #| + ), + ), + ) + archive.add("decoy.png", b"root image") + archive.add("xl/decoy.png", b"wrong image") + let parsed = parse_rich_value_images( + archive, + archive_part_name_index(archive), + value => { + @encoding/utf8.decode(value) catch { + _ => raise InvalidXml(msg="invalid utf8") + } + }, + ) + guard parsed is Some(data) else { fail("expected rich value image data") } + assert_eq(data.rel_targets.get("rId1"), Some("decoy.png")) +} + ///| /// Parses the ordered blip relationship ids from `` /// elements of `xl/richData/rdRichValueWebImage.xml`. Each web image's @@ -431,7 +478,7 @@ fn Workbook::rich_value_image_for_cell( rich_value_web_target(data, keys, values) } let media_path = match target { - Some(value) => normalize_cell_image_target(value) + Some(value) => value None => return None } match self.rich_value_media.get(media_path) { diff --git a/xlsx/rich_value_images_test.mbt b/xlsx/rich_value_images_test.mbt index fef85710..72179784 100644 --- a/xlsx/rich_value_images_test.mbt +++ b/xlsx/rich_value_images_test.mbt @@ -70,7 +70,7 @@ fn inject_rich_value_image(base : Bytes, media : Bytes) -> Bytes raise { ( #| #| - #| + #| #| ), ), @@ -168,7 +168,7 @@ fn inject_mismatched_rich_value(base : Bytes) -> Bytes raise { "xl/richData/_rels/richValueRel.xml.rels", @encoding/utf8.encode( ( - #| + #| ), ), ) @@ -243,7 +243,7 @@ fn inject_web_image(base : Bytes, media : Bytes) -> Bytes raise { "xl/richData/_rels/rdRichValueWebImage.xml.rels", @encoding/utf8.encode( ( - #| + #| ), ), ) @@ -325,7 +325,7 @@ fn inject_local_key_wins(base : Bytes, media : Bytes) -> Bytes raise { "xl/richData/_rels/richValueRel.xml.rels", @encoding/utf8.encode( ( - #| + #| ), ), ) @@ -341,7 +341,7 @@ fn inject_local_key_wins(base : Bytes, media : Bytes) -> Bytes raise { "xl/richData/_rels/rdRichValueWebImage.xml.rels", @encoding/utf8.encode( ( - #| + #| ), ), ) From 59bc9759cb4566e8cc3bda1df90ad9b454fd720a Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 02:21:27 +0800 Subject: [PATCH 112/150] fix(xlsx): enforce relationship content identities --- ooxml/pkg.generated.mbti | 7 + ooxml/read_parse.mbt | 78 +++++++----- xlsx/ooxml_rels_error_test.mbt | 31 +++++ xlsx/read.mbt | 225 ++++++++++++++++++++++++--------- xlsx/read_drawing_xml.mbt | 8 ++ xlsx/read_namespace_test.mbt | 3 +- xlsx/read_package_parts.mbt | 73 +++++++++-- xlsx/read_sheet_rel_parts.mbt | 8 ++ xlsx/write_ooxml_constants.mbt | 12 ++ 9 files changed, 346 insertions(+), 99 deletions(-) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 3794a4ee..6d380d68 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -42,6 +42,8 @@ pub fn parse_internal_relationship_targets(StringView, StringView, cancelled? : pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String], cancelled? : () -> Bool) -> Map[String, String] raise ParseXmlError +pub fn parse_package_content_types(StringView, cancelled? : () -> Bool) -> PackageContentTypes raise ParseXmlError + pub fn project_xml_markup_compatibility(StringView, ArrayView[String], max_output_chars? : Int, cancelled? : () -> Bool, extension_elements? : ArrayView[(String, String)]) -> String raise ParseXmlError pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError @@ -71,6 +73,11 @@ pub fn OpcPartNameRegistry::new(maximum_identity_nodes? : Int) -> Self pub fn OpcPartNameRegistry::register(Self, StringView, String) -> OpcPartNameConflict? raise ParseXmlError pub fn OpcPartNameRegistry::register_cancellable(Self, StringView, String, cancelled? : () -> Bool) -> OpcPartNameConflict? raise ParseXmlError +pub struct PackageContentTypes { + // private fields +} +pub fn PackageContentTypes::content_type_for(Self, StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError + type WorkbookManifest pub fn WorkbookManifest::add_content_default(Self, String, String) -> Unit pub fn WorkbookManifest::add_content_override(Self, String, String) -> Unit diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 69066df2..da1f620e 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -1025,10 +1025,11 @@ pub fn parse_external_relationship_targets( } ///| -priv struct PackageContentTypes { - overrides : Map[String, String] - override_part_names : Map[String, String] - defaults : Map[String, String] +/// Parsed, reusable OPC content-type manifest index. +pub struct PackageContentTypes { + priv overrides : Map[String, String] + priv override_part_names : Map[String, String] + priv defaults : Map[String, String] } ///| @@ -1243,7 +1244,9 @@ pub fn package_part_name_key_cancellable( } ///| -fn parse_content_types( +/// Parses and validates an OPC `[Content_Types].xml` manifest once so callers +/// can perform multiple role checks without rescanning the XML document. +pub fn parse_package_content_types( xml : StringView, cancelled? : () -> Bool = () => false, ) -> PackageContentTypes raise ParseXmlError { @@ -1363,39 +1366,24 @@ fn parse_content_types( { overrides, override_part_names, defaults } } -///| -pub fn find_override_part_by_content_types( - xml : StringView, - content_types : ArrayView[String], - cancelled? : () -> Bool = () => false, -) -> String? raise ParseXmlError { - let content_types_value = parse_content_types(xml, cancelled~) - for wanted in content_types { - for part_name_key, content_type in content_types_value.overrides { - if content_type == wanted { - return content_types_value.override_part_names.get(part_name_key) - } - } - } - None -} - ///| /// Returns the declared content type for `part_name`. An ASCII-case-insensitive /// OPC part-name override wins; otherwise the case-insensitive extension /// default is used. -pub fn find_part_content_type( - xml : StringView, +pub fn PackageContentTypes::content_type_for( + self : PackageContentTypes, part_name : StringView, cancelled? : () -> Bool = () => false, ) -> String? raise ParseXmlError { - let content_types_value = parse_content_types(xml, cancelled~) let normalized = if part_name.has_prefix("/") { part_name.to_owned() } else { "/" + part_name.to_owned() } - match content_types_value.overrides.get(package_part_name_key(normalized)) { + match + self.overrides.get( + package_part_name_key_cancellable(normalized, cancelled~), + ) { Some(value) => Some(value) None => { let filename = match normalized.rev_find("/") { @@ -1407,9 +1395,41 @@ pub fn find_part_content_type( filename[index + 1:].to_owned().to_lower() _ => return None } - content_types_value.defaults.get(extension) + self.defaults.get(extension) + } + } +} + +///| +pub fn find_override_part_by_content_types( + xml : StringView, + content_types : ArrayView[String], + cancelled? : () -> Bool = () => false, +) -> String? raise ParseXmlError { + let content_types_value = parse_package_content_types(xml, cancelled~) + for wanted in content_types { + for part_name_key, content_type in content_types_value.overrides { + if content_type == wanted { + return content_types_value.override_part_names.get(part_name_key) + } } } + None +} + +///| +/// Returns the declared content type for `part_name`. An ASCII-case-insensitive +/// OPC part-name override wins; otherwise the case-insensitive extension +/// default is used. +pub fn find_part_content_type( + xml : StringView, + part_name : StringView, + cancelled? : () -> Bool = () => false, +) -> String? raise ParseXmlError { + parse_package_content_types(xml, cancelled~).content_type_for( + part_name, + cancelled~, + ) } ///| @@ -1457,7 +1477,7 @@ test "ooxml read_parse wb: content-type override required attrs" { #| #| #| - try parse_content_types(missing_part) catch { + try parse_package_content_types(missing_part) catch { InvalidXml(msg~) => inspect(msg, content="content type override part name missing") _ => fail("unexpected content type parse cancellation") @@ -1471,7 +1491,7 @@ test "ooxml read_parse wb: content-type override required attrs" { #| #| #| - try parse_content_types(missing_content_type) catch { + try parse_package_content_types(missing_content_type) catch { InvalidXml(msg~) => inspect(msg, content="content type override content type missing") _ => fail("unexpected content type parse cancellation") diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index 20b8aa64..4078980f 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -286,6 +286,37 @@ test "ooxml rels: relocated workbook path requires root relationship and content debug_inspect(parsed.get_sheet_list(), content="[\"Sheet1\"]") } +///| +test "ooxml rels: worksheet relationship target requires worksheet content type" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + let archive = @zip.read(@xlsx.write(workbook)) catch { + err => fail("zip read failed: \{repr(err)}") + } + let content_types = decode_zip_entry_utf8(archive, "[Content_Types].xml") + let worksheet_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml" + let patched_content_types = content_types.replace( + old=worksheet_type, + new="application/xml", + ) + if patched_content_types == content_types { + fail("worksheet content type marker missing") + } + let updated = rewrite_zip_entry( + archive, + "[Content_Types].xml", + @encoding/utf8.encode(patched_content_types), + ) catch { + err => fail("zip rewrite failed: \{repr(err)}") + } + let result = Ok(@xlsx.read(updated)) catch { error => Err(error) } + match result { + Err(InvalidXml(msg~)) => + inspect(msg, content="worksheet content type is invalid") + _ => fail("expected InvalidXml for worksheet content type mismatch") + } +} + ///| test "ooxml rels: pivot cache rels tolerate stale first entries and mixed targets" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read.mbt b/xlsx/read.mbt index e91a8d79..cba5bfae 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3525,15 +3525,15 @@ fn read_zip_archive_core_unchecked( read_budget.charge_work(canonical.length()) canonical } - let workbook_xml_part_path = actual_archive_part_path( + let (workbook_logical_part_path, content_types) = resolve_workbook_xml_part_path( + archive, part_names, - resolve_workbook_xml_part_path( - archive, - part_names, - decode, - budget=read_budget, - cancelled~, - ), + decode, + budget=read_budget, + cancelled~, + ) + let workbook_xml_part_path = actual_archive_part_path( + part_names, workbook_logical_part_path, ) let workbook_bytes = match archive.get(workbook_xml_part_path) { Some(value) => value @@ -3611,38 +3611,42 @@ fn read_zip_archive_core_unchecked( cancelled~, ) let shared_strings = match shared_strings_path { - Some(path) => - match archive.get(path) { - Some(value) => { - let raw_source = decode_source(value) - read_budget.charge_work(raw_source.length()) - let source = project_xlsx_markup_compatibility( - raw_source, - limits.max_xml_part_bytes, - cancelled~, - ) - if source != raw_source { - read_budget.charge_work(source.length()) - } - let core_source = match - xlsx_core_source_without_foreign(source, "sst", cancelled~) { - Some(filtered) => filtered - None => source - } - if core_source != source { - read_budget.charge_work(core_source.length()) - } - let canonical = canonicalize_xlsx_xml( - core_source, - max_output_chars=limits.max_xml_part_bytes, - cancelled~, - ) - read_budget.checkpoint() - read_budget.charge_work(canonical.length()) - parse_shared_strings(canonical, budget=read_budget) - } - None => raise MissingPart(path~) + Some(path) => { + let value = load_typed_part( + archive, + content_types, + path, + [ct_shared_strings], + "shared strings", + cancelled~, + ) + let raw_source = decode_source(value) + read_budget.charge_work(raw_source.length()) + let source = project_xlsx_markup_compatibility( + raw_source, + limits.max_xml_part_bytes, + cancelled~, + ) + if source != raw_source { + read_budget.charge_work(source.length()) + } + let core_source = match + xlsx_core_source_without_foreign(source, "sst", cancelled~) { + Some(filtered) => filtered + None => source + } + if core_source != source { + read_budget.charge_work(core_source.length()) } + let canonical = canonicalize_xlsx_xml( + core_source, + max_output_chars=limits.max_xml_part_bytes, + cancelled~, + ) + read_budget.checkpoint() + read_budget.charge_work(canonical.length()) + parse_shared_strings(canonical, budget=read_budget) + } None => [] } let styles_path = workbook_related_part_path( @@ -3665,10 +3669,14 @@ fn read_zip_archive_core_unchecked( styles_ext_lst_xml, ) = match styles_path { Some(path) => { - let value = match archive.get(path) { - Some(value) => value - None => raise MissingPart(path~) - } + let value = load_typed_part( + archive, + content_types, + path, + [ct_styles], + "styles", + cancelled~, + ) let styles_raw_xml = decode_source(value) read_budget.charge_work(styles_raw_xml.length()) let styles_source_xml = project_xlsx_markup_compatibility( @@ -3745,11 +3753,17 @@ fn read_zip_archive_core_unchecked( cancelled~, ) let theme_xml = match theme_path { - Some(path) => - match archive.get(path) { - Some(value) => Some(decode(value)) - None => raise MissingPart(path~) - } + Some(path) => { + let value = load_typed_part( + archive, + content_types, + path, + [ct_theme], + "theme", + cancelled~, + ) + Some(decode(value)) + } None => None } let theme_colors = match theme_xml { @@ -3855,6 +3869,7 @@ fn read_zip_archive_core_unchecked( workbook_rels, workbook_xml_part_path, part_names, + content_types, archive, decode, budget=read_budget, @@ -3892,10 +3907,14 @@ fn read_zip_archive_core_unchecked( } match vba_path { Some(path) => { - let vba_bytes = match archive.get(path) { - Some(value) => value - None => raise MissingPart(path~) - } + let vba_bytes = load_typed_part( + archive, + content_types, + path, + [ct_vba_project], + "VBA project", + cancelled~, + ) workbook.vba_project = Some(vba_bytes.to_owned()) } None => () @@ -3957,10 +3976,14 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="worksheet target missing") } - let sheet_bytes = match archive.get(sheet_path) { - Some(value) => value - None => raise MissingPart(path=sheet_path) - } + let sheet_bytes = load_typed_part( + archive, + content_types, + sheet_path, + [ct_worksheet], + "worksheet", + cancelled~, + ) let sheet_raw_xml = decode_source(sheet_bytes) read_budget.charge_work(sheet_raw_xml.length()) let sheet_source_xml = project_xlsx_markup_compatibility( @@ -4108,6 +4131,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(slicer_path), + [ct_slicer], + "slicer", + cancelled~, + ) if seen_paths.contains(slicer_path) { continue } @@ -4234,6 +4264,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(table_path), + [ct_table], + "table", + cancelled~, + ) let table_bytes = match archive.get(table_path) { Some(value) => value None => raise MissingPart(path=table_path) @@ -4279,6 +4316,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(pivot_path), + [ct_pivot_table], + "pivot table", + cancelled~, + ) let pivot_bytes = match archive.get(pivot_path) { Some(value) => value None => raise MissingPart(path=pivot_path) @@ -4321,6 +4365,13 @@ fn read_zip_archive_core_unchecked( ) } } + require_part_content_type( + content_types, + logical_archive_part_path(cache_path), + [ct_pivot_cache_def], + "pivot cache definition", + cancelled~, + ) let cache_bytes = match archive.get(cache_path) { Some(value) => value None => raise MissingPart(path=cache_path) @@ -4366,6 +4417,13 @@ fn read_zip_archive_core_unchecked( } match record_path { Some(path) => { + require_part_content_type( + content_types, + logical_archive_part_path(path), + [ct_pivot_cache_records], + "pivot cache records", + cancelled~, + ) let record_bytes = match archive.get(path) { Some(value) => value None => raise MissingPart(path~) @@ -4412,6 +4470,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(vml_path), + [ct_vml], + "VML drawing", + cancelled~, + ) let vml_bytes = match archive.get(vml_path) { Some(value) => value None => raise MissingPart(path=vml_path) @@ -4432,6 +4497,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(vml_path), + [ct_vml], + "VML drawing", + cancelled~, + ) let vml_bytes = match archive.get(vml_path) { Some(value) => value None => raise MissingPart(path=vml_path) @@ -4481,6 +4553,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(comment_path), + [ct_comments], + "comments", + cancelled~, + ) let comment_bytes = match archive.get(comment_path) { Some(value) => value None => raise MissingPart(path=comment_path) @@ -4566,6 +4645,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(drawing_path), + [ct_drawing], + "drawing", + cancelled~, + ) let drawing_bytes = match archive.get(drawing_path) { Some(value) => value None => raise MissingPart(path=drawing_path) @@ -4612,6 +4698,7 @@ fn read_zip_archive_core_unchecked( drawing_rels_xml, drawing_path, part_names, + content_types, drawing_metrics, archive, decode, @@ -4699,10 +4786,14 @@ fn read_zip_archive_core_unchecked( Some(value) => value None => raise InvalidXml(msg="chartsheet target missing") } - let chartsheet_bytes = match archive.get(chartsheet_path) { - Some(value) => value - None => raise MissingPart(path=chartsheet_path) - } + let chartsheet_bytes = load_typed_part( + archive, + content_types, + chartsheet_path, + [ct_chartsheet], + "chartsheet", + cancelled~, + ) let chartsheet_raw_xml = decode_source(chartsheet_bytes) read_budget.charge_work(chartsheet_raw_xml.length()) let chartsheet_source_xml = project_xlsx_markup_compatibility( @@ -4751,6 +4842,13 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(drawing_path), + [ct_drawing], + "drawing", + cancelled~, + ) let drawing_rels_path = actual_relationship_part_path( drawing_path, part_names, ) @@ -4786,6 +4884,13 @@ fn read_zip_archive_core_unchecked( ) } } + require_part_content_type( + content_types, + logical_archive_part_path(chart_path), + [ct_chart], + "chart", + cancelled~, + ) let chart_bytes = match archive.get(chart_path) { Some(value) => value None => raise MissingPart(path=chart_path) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 470842f5..cbbac380 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -425,6 +425,7 @@ fn parse_drawing_charts( drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, metrics : DrawingMetrics, archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, @@ -463,6 +464,13 @@ fn parse_drawing_charts( part_names, cancelled~, ) + require_part_content_type( + content_types, + logical_archive_part_path(chart_path), + [ct_chart], + "chart", + cancelled~, + ) let chart_bytes = match archive.get(chart_path) { Some(value) => value None => raise MissingPart(path=chart_path) diff --git a/xlsx/read_namespace_test.mbt b/xlsx/read_namespace_test.mbt index 57d013ec..c1a440f8 100644 --- a/xlsx/read_namespace_test.mbt +++ b/xlsx/read_namespace_test.mbt @@ -48,7 +48,8 @@ fn namespace_qualified_xlsx_fixture( } else { "worksheets/sheet.xml" } - let relationship_id = if entity_paths { "sheet&1" } else { "sheet" } + // Exercise entity decoding without producing an invalid XML Schema `ID`. + let relationship_id = if entity_paths { "sheet1" } else { "sheet" } let root_target = if entity_paths { "custom/a&b.xml" } else { diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index d73db49b..30431109 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -51,6 +51,58 @@ fn workbook_content_type_is_supported(value : StringView) -> Bool { false } +///| +fn require_part_content_type( + content_types : @ooxml.PackageContentTypes, + part_name : StringView, + expected : ArrayView[String], + role : StringView, + cancelled? : () -> Bool = () => false, +) -> Unit raise XlsxError { + let declared = content_types.content_type_for(part_name, cancelled~) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + match declared { + Some(value) => { + let normalized = value.to_lower() + for candidate in expected { + if normalized == candidate.to_lower() { + return + } + } + raise InvalidXml(msg="\{role.to_owned()} content type is invalid") + } + None => raise InvalidXml(msg="\{role.to_owned()} content type is missing") + } +} + +///| +/// Loads a relationship-selected package part before validating its semantic +/// media type. Missing targets remain `MissingPart`; an existing part whose +/// manifest identity does not match `expected` is rejected as invalid XML. +fn load_typed_part( + archive : @zip.Archive, + content_types : @ooxml.PackageContentTypes, + path : StringView, + expected : ArrayView[String], + role : StringView, + cancelled? : () -> Bool = () => false, +) -> BytesView raise XlsxError { + let bytes = match archive.get(path) { + Some(value) => value + None => raise MissingPart(path=path.to_owned()) + } + require_part_content_type( + content_types, + logical_archive_part_path(path), + expected, + role, + cancelled~, + ) + bytes +} + ///| test "read package parts: workbook media types are case-insensitive" { assert_true( @@ -72,7 +124,7 @@ fn resolve_workbook_xml_part_path( decode : (BytesView) -> String raise XlsxError, budget? : ReadBudget, cancelled? : () -> Bool = () => false, -) -> String raise XlsxError { +) -> (String, @ooxml.PackageContentTypes) raise XlsxError { let workbook_part = workbook_part_from_root_relationships( archive, part_names, @@ -94,19 +146,21 @@ fn resolve_workbook_xml_part_path( } None => check_read_cancelled(cancelled) } - let content_type = @ooxml.find_part_content_type( + let content_types = @ooxml.parse_package_content_types( content_types_xml, - workbook_part, cancelled~, ) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } - match content_type { - Some(value) if workbook_content_type_is_supported(value) => workbook_part - Some(_) => raise InvalidXml(msg="root workbook content type is invalid") - None => raise InvalidXml(msg="root workbook content type is missing") - } + require_part_content_type( + content_types, + workbook_part, + workbook_content_type_candidates(), + "root workbook", + cancelled~, + ) + (workbook_part, content_types) } ///| @@ -120,11 +174,12 @@ fn decode_utf8_or_invalid_xml(bytes : BytesView) -> String raise XlsxError { fn resolve_workbook_xml_part_path_for_test( archive : @zip.Archive, ) -> String raise XlsxError { - resolve_workbook_xml_part_path( + let (path, _) = resolve_workbook_xml_part_path( archive, archive_part_name_index(archive), decode_utf8_or_invalid_xml, ) + path } ///| diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index fc6bb5f4..53fd867a 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -342,6 +342,7 @@ fn parse_slicer_cache_definitions( workbook_rels_xml : StringView, workbook_part : StringView, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, archive : @zip.Archive, decode : (BytesView) -> String raise XlsxError, budget? : ReadBudget, @@ -363,6 +364,13 @@ fn parse_slicer_cache_definitions( cancelled~, ), ) + require_part_content_type( + content_types, + logical_archive_part_path(cache_path), + [ct_slicer_cache], + "slicer cache", + cancelled~, + ) let cache_bytes = match archive.get(cache_path) { Some(value) => value None => raise MissingPart(path=cache_path) diff --git a/xlsx/write_ooxml_constants.mbt b/xlsx/write_ooxml_constants.mbt index 103e0f58..b0457e8f 100644 --- a/xlsx/write_ooxml_constants.mbt +++ b/xlsx/write_ooxml_constants.mbt @@ -102,3 +102,15 @@ let ct_vml = "application/vnd.openxmlformats-officedocument.vmlDrawing" ///| let ct_comments = "application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml" + +///| +let ct_worksheet = "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml" + +///| +let ct_styles = "application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml" + +///| +let ct_shared_strings = "application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml" + +///| +let ct_theme = "application/vnd.openxmlformats-officedocument.theme+xml" From 858fa80115ef74b42a8888fbf48436bee1f6c259 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 02:24:45 +0800 Subject: [PATCH 113/150] fix(office): aggregate XLSX used-range scans --- office/cmd/office/pkg.generated.mbti | 2 ++ office/cmd/office/xlsx_read_model.mbt | 35 +++++++++++++++++++++++--- office/cmd/office/xlsx_read_output.mbt | 17 ++++++++++--- office/cmd/office/xlsx_read_wbtest.mbt | 27 ++++++++++++++++++++ xlsx/pkg.generated.mbti | 1 + xlsx/worksheet.mbt | 10 +++++++- 6 files changed, 84 insertions(+), 8 deletions(-) diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 381bf495..50957eac 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -95,6 +95,8 @@ type XlsxSheetContent type XlsxSheetTarget +type XlsxStoredCellBudget + type XlsxStringBudget // Type aliases diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index e33ac283..2bd09ec0 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -116,6 +116,15 @@ struct XlsxMetadataBudget { mut string_chars : Int } +///| +/// Command-local budget for the stored-cell records inspected while deriving +/// worksheet used ranges. One instance must be shared across every worksheet +/// participating in a single command result. +struct XlsxStoredCellBudget { + maximum : Int + mut used : Int +} + ///| fn xlsx_cli_failure( code : String, @@ -341,10 +350,12 @@ fn xlsx_rect_from_coordinate(coordinate : @lib.SelectorCoordinate) -> XlsxRect { fn xlsx_used_rect( projection : XlsxProjection, worksheet : @xlsx.Worksheet, + stored_cells : XlsxStoredCellBudget, ) -> XlsxRect? raise CliFailure { projection.checkpoint() + stored_cells.charge(worksheet.stored_cell_count()) let (max_row, max_col) = worksheet.used_bounds_limited( - maximum_stored_cells=projection.max_scan_cells, + maximum_stored_cells=stored_cells.maximum, cancelled=projection.cancelled, ) catch { ReadCancelled => @@ -359,6 +370,23 @@ fn xlsx_used_rect( } } +///| +fn XlsxStoredCellBudget::new(maximum : Int) -> XlsxStoredCellBudget { + { maximum, used: 0 } +} + +///| +fn XlsxStoredCellBudget::charge( + self : XlsxStoredCellBudget, + count : Int, +) -> Unit raise CliFailure { + let actual = self.used.to_int64() + count.to_int64() + if actual > self.maximum.to_int64() { + raise xlsx_scan_resource_failure("stored_cells", self.maximum, actual) + } + self.used = actual.to_int() +} + ///| fn XlsxProjection::check_scan_rect( self : XlsxProjection, @@ -529,13 +557,14 @@ fn xlsx_scan_regions( ) -> Array[XlsxScanRegion] raise CliFailure { projection.checkpoint() let regions : Array[XlsxScanRegion] = [] + let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) match under { Some(Cell(sheet, rect, ..)) | Some(Range(sheet, rect, ..)) => regions.push({ sheet, rect }) Some(Sheet(sheet)) => match sheet.content { Worksheet(worksheet) => - match xlsx_used_rect(projection, worksheet) { + match xlsx_used_rect(projection, worksheet, stored_cells) { Some(rect) => regions.push({ sheet, rect }) None => () } @@ -546,7 +575,7 @@ fn xlsx_scan_regions( projection.checkpoint() match sheet.content { Worksheet(worksheet) => - match xlsx_used_rect(projection, worksheet) { + match xlsx_used_rect(projection, worksheet, stored_cells) { Some(rect) => regions.push({ sheet, rect }) None => () } diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index c2b4e3f2..7a3cbe8a 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -97,6 +97,7 @@ fn xlsx_sheet_summary_json( projection : XlsxProjection, sheet : XlsxSheetTarget, metadata : XlsxMetadataBudget, + stored_cells : XlsxStoredCellBudget, ) -> Json raise CliFailure { projection.checkpoint() metadata.charge_item([sheet.name, sheet.path]) @@ -114,7 +115,7 @@ fn xlsx_sheet_summary_json( Worksheet(worksheet) => { fields["kind"] = Json::string("worksheet") fields["state"] = Json::string(xlsx_sheet_state_name(worksheet.state())) - match xlsx_used_rect(projection, worksheet) { + match xlsx_used_rect(projection, worksheet, stored_cells) { Some(rect) => { fields["max_row"] = Json::number(rect.row_hi.to_double()) fields["max_column"] = Json::number(rect.col_hi.to_double()) @@ -170,10 +171,13 @@ fn xlsx_sheet_summary_json( fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { projection.checkpoint() let metadata = XlsxMetadataBudget::new() + let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) let sheets : Array[Json] = [] for sheet in projection.sheets { projection.checkpoint() - sheets.push(xlsx_sheet_summary_json(projection, sheet, metadata)) + sheets.push( + xlsx_sheet_summary_json(projection, sheet, metadata, stored_cells), + ) } let defined_names : Array[Json] = [] for defined in projection.workbook.defined_names() { @@ -524,9 +528,12 @@ fn retain_xlsx_outline_records( budget : OfficeOutputBudget, ) -> Unit raise CliFailure { let metadata = XlsxMetadataBudget::new() + let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) for sheet in projection.sheets { projection.checkpoint() - let record = xlsx_sheet_summary_json(projection, sheet, metadata) + let record = xlsx_sheet_summary_json( + projection, sheet, metadata, stored_cells, + ) budget.reserve_array_item(record, !sheets.is_empty()) sheets.push(record) } @@ -617,6 +624,7 @@ async fn xlsx_get_data( projection, sheet, XlsxMetadataBudget::new(), + XlsxStoredCellBudget::new(projection.max_scan_cells), ), }) Cell(sheet, rect, path~) => { @@ -890,6 +898,7 @@ fn xlsx_outline_human( output.begin_line() output.write_plain("workbook\t/xlsx/workbook\t") output.write_plain("\{projection.sheets.length()} sheets") + let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) for sheet in projection.sheets { projection.checkpoint() output.begin_line() @@ -900,7 +909,7 @@ fn xlsx_outline_human( ChartSheet(_) => output.write_plain("\tchart-sheet") Worksheet(worksheet) => { output.write_plain("\tworksheet") - match xlsx_used_rect(projection, worksheet) { + match xlsx_used_rect(projection, worksheet, stored_cells) { Some(rect) => { output.write_char('\t') output.write_terminal_safe(rect.range_reference(projection.file)) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 763e8646..1340dddf 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -447,6 +447,33 @@ test "XLSX used-range preprocessing rejects excess stored cells before scanning" assert_eq(details.get("actual"), Some(Json::number(2))) } +///| +test "XLSX used-range preprocessing shares one stored-cell budget across sheets" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("First")) + ignore(workbook.new_sheet("Second")) + workbook.set_cell_str("First", "A1", "one") + workbook.set_cell_str("Second", "A1", "two") + let projection = make_xlsx_projection( + "aggregate-stored-cells.xlsx", workbook, 1, + ) + for + action in [ + () => ignore(xlsx_scan_regions(projection, None)), + () => ignore(xlsx_outline_data(projection)), + () => ignore(xlsx_outline_human(projection, 10_000)), + ] { + let limited = xlsx_cli_error(() => action()) + assert_eq(limited.code, "office.xlsx.resource_limit") + guard limited.details is Some(Object(details)) else { + fail("expected aggregate stored-cell limit details") + } + assert_eq(details.get("resource"), Some(Json::string("stored_cells"))) + assert_eq(details.get("limit"), Some(Json::number(1))) + assert_eq(details.get("actual"), Some(Json::number(2))) + } +} + ///| test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" { let projection = xlsx_read_test_projection() diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 59d9a59c..ab01162d 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -2203,6 +2203,7 @@ pub fn Worksheet::sheet_protection(Self) -> SheetProtection? pub fn Worksheet::slicers(Self) -> ArrayView[Slicer] pub fn Worksheet::sparkline_groups(Self) -> ArrayView[SparklineGroup] pub fn Worksheet::state(Self) -> SheetState +pub fn Worksheet::stored_cell_count(Self) -> Int pub fn Worksheet::tables(Self) -> ArrayView[Table] pub fn Worksheet::unmerge_cells(Self, String) -> Unit raise XlsxError pub fn Worksheet::unprotect_sheet(Self, password? : String) -> Unit raise XlsxError diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 817bb078..18abcaee 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2987,6 +2987,14 @@ pub fn Worksheet::max_col(self : Worksheet) -> Int { max_col } +///| +/// Returns the number of stored cell records in this worksheet in O(1) time. +/// This is a representation count, not the area of the logical used range; +/// callers can use it to preflight aggregate scan work before iteration. +pub fn Worksheet::stored_cell_count(self : Worksheet) -> Int { + self.cells.length() +} + ///| /// Returns the used `(max_row, max_column)` bounds after rejecting a worksheet /// whose stored-cell representation exceeds `maximum_stored_cells`. The O(1) @@ -3000,7 +3008,7 @@ pub fn Worksheet::used_bounds_limited( if maximum_stored_cells < 1 { raise InvalidOptions(msg="maximum stored cells must be positive") } - let stored = self.cells.length() + let stored = self.stored_cell_count() if stored > maximum_stored_cells { raise ResourceLimitExceeded( kind="stored_cells", From 9e16ed85e62ac9c61bab031dada162232b2f9d47 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 02:26:34 +0800 Subject: [PATCH 114/150] fix(xlsx): validate parsed sheet names --- xlsx/read_workbook_xml.mbt | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index e4386e64..4a2184ae 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -46,6 +46,7 @@ fn parse_workbook_sheets( cancelled? : () -> Bool = () => false, ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] + let sheet_names : Map[String, Bool] = Map([]) let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) let mut workbook_namespace : String? = None while workbook_scanner_next(scanner) { @@ -88,6 +89,15 @@ fn parse_workbook_sheets( Some(value) => value None => raise InvalidXml(msg="sheet name missing") } + check_sheet_name(name) catch { + InvalidSheetName(msg~) => raise InvalidXml(msg~) + error => raise error + } + let normalized_name = normalize_sheet_name(name) + if sheet_names.contains(normalized_name) { + raise InvalidXml(msg="duplicate sheet name") + } + sheet_names[normalized_name] = true let relationship_namespace = if namespace_uri == strict_spreadsheet_namespace { strict_relationship_attribute_namespace @@ -113,6 +123,33 @@ fn parse_workbook_sheets( sheets } +///| +test "read workbook sheets rejects invalid and case-duplicate names" { + let prefix = "" + let suffix = "" + let invalid = prefix + + "" + + suffix + try parse_workbook_sheets(invalid, 10) catch { + InvalidXml(msg~) => + inspect(msg, content="sheet name contains invalid characters") + _ => fail("unexpected invalid sheet-name error") + } noraise { + _ => fail("expected invalid sheet-name rejection") + } + + let duplicate = prefix + + "" + + "" + + suffix + try parse_workbook_sheets(duplicate, 10) catch { + InvalidXml(msg~) => inspect(msg, content="duplicate sheet name") + _ => fail("unexpected duplicate sheet-name error") + } noraise { + _ => fail("expected duplicate sheet-name rejection") + } +} + ///| fn parse_active_sheet_index(xml : StringView) -> Int? raise XlsxError { let xml_str = xml.to_owned() From 3fa46e164b6177f7ee6a7c1d0a99e767a510d3ec Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 02:34:23 +0800 Subject: [PATCH 115/150] fix(xlsx): make row duplication transactional --- xlsx/read_coordinate_hardening_test.mbt | 6 ++ xlsx/sqref.mbt | 7 +- xlsx/worksheet.mbt | 128 +++++++++++++++++------- 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/xlsx/read_coordinate_hardening_test.mbt b/xlsx/read_coordinate_hardening_test.mbt index 35fa7816..c33e2913 100644 --- a/xlsx/read_coordinate_hardening_test.mbt +++ b/xlsx/read_coordinate_hardening_test.mbt @@ -427,6 +427,12 @@ test "coordinate attributes accept quoted delimiters and numeric XML spaces" { let validations = workbook.get_data_validations("Sheet1") inspect(validations.length(), content="1") inspect(validations[0].sqref, content="A1 B1 C1") + guard workbook.sheet("Sheet1") is Some(sheet) else { + fail("missing entity-sqref worksheet") + } + assert_false(sheet.data_validations()[0].contains("&#")) + workbook.duplicate_row_to("Sheet1", 1, 2) + ignore(@xlsx.read(@xlsx.write(workbook))) } ///| diff --git a/xlsx/sqref.mbt b/xlsx/sqref.mbt index 32c47cf3..8cfffeef 100644 --- a/xlsx/sqref.mbt +++ b/xlsx/sqref.mbt @@ -128,12 +128,13 @@ fn normalize_xml_sqref_attr_for_read( Some(value) => value None => raise InvalidXml(msg="\{field.to_owned()} tag missing") } - let raw = match attr_value(tag, "sqref") { - Some(value) => unescape_xml_text(value) + let encoded = match attr_value(tag, "sqref") { + Some(value) => value None => raise InvalidXml(msg="\{field.to_owned()} sqref missing") } + let raw = unescape_xml_text(encoded) let normalized = normalize_sqref_for_read(raw, field, budget?) - if normalized == raw { + if normalized == encoded { return xml.to_owned() } match replace_attr_value_in_open_tag(xml, "sqref", normalized) { diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 18abcaee..0c7c5748 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -3966,21 +3966,19 @@ fn duplicate_xml_sqref( } ///| -fn Worksheet::duplicate_conditional_formats( - self : Worksheet, +fn duplicate_xml_sqref_copies( + entries : ArrayView[String], row : Int, target_row : Int, -) -> Unit raise XlsxError { +) -> Array[String] raise XlsxError { let copies : Array[String] = [] - for xml in self.conditional_formats { + for xml in entries { match duplicate_xml_sqref(xml, row, target_row) { Some(value) => copies.push(value) None => () } } - if copies.length() > 0 { - self.conditional_formats.append(copies) - } + copies } ///| @@ -3989,42 +3987,55 @@ fn Worksheet::duplicate_data_validations( row : Int, target_row : Int, ) -> Unit raise XlsxError { - let copies : Array[String] = [] - for xml in self.data_validations { - match duplicate_xml_sqref(xml, row, target_row) { - Some(value) => copies.push(value) - None => () - } - } + let copies = duplicate_xml_sqref_copies( + self.data_validations, + row, + target_row, + ) if copies.length() > 0 { self.data_validations.append(copies) } } ///| -fn Worksheet::duplicate_merge_cells( - self : Worksheet, +fn duplicate_merge_cell_copies( + merged_cells : ArrayView[String], row : Int, target_row : Int, -) -> Unit raise XlsxError { +) -> Array[String] raise XlsxError { + let copies : Array[String] = [] let mut source_row = row if row > target_row { source_row = row + 1 } - for range_ref in self.merged_cells { + for range_ref in merged_cells { let (min_row, _min_col, max_row, _max_col) = parse_range_ref(range_ref) if min_row < target_row && target_row < max_row { - return + return copies } } - for range_ref in self.merged_cells { + for range_ref in merged_cells { let (min_row, min_col, max_row, max_col) = parse_range_ref(range_ref) if min_row == max_row && min_row == source_row { let from_ref = cell_ref_from(target_row, min_col) let to_ref = cell_ref_from(target_row, max_col) - self.merge_cells("\{from_ref}:\{to_ref}") + let duplicate = normalize_range_ref("\{from_ref}:\{to_ref}") + if !merged_cells.contains(duplicate) && !copies.contains(duplicate) { + copies.push(duplicate) + } } } + copies +} + +///| +fn Worksheet::duplicate_merge_cells( + self : Worksheet, + row : Int, + target_row : Int, +) -> Unit raise XlsxError { + let copies = duplicate_merge_cell_copies(self.merged_cells, row, target_row) + self.merged_cells.append(copies) } ///| @@ -4061,7 +4072,10 @@ fn Worksheet::duplicate_row_to( translation_budget, fn(cell : Cell) -> Bool { cell.row == row }, ) - let row_cells : Array[Cell] = [] + // Stage every fallible derivative before `insert_rows` commits its own + // transactional shift. Once that insert succeeds, the remaining appends and + // map assignments below are infallible state installation. + let duplicated_cells : Array[Cell] = [] for stored_cell in self.cells { if stored_cell.row != row { continue @@ -4069,19 +4083,8 @@ fn Worksheet::duplicate_row_to( let cell = normal_formula_cell_for_structural_edit( stored_cell, shared_formula_masters, translation_budget, ) - row_cells.push(cell) - } - let row_dim = self.row_dimensions.get(row) - self.insert_rows( - target_row, - 1, - formula_limits~, - cancelled~, - formula_budget=translation_budget, - ) - for cell in row_cells { let reference = cell_ref_from(target_row, cell.col) - self.cells.push({ + duplicated_cells.push({ reference, row: target_row, col: cell.col, @@ -4096,13 +4099,44 @@ fn Worksheet::duplicate_row_to( style_id: cell.style_id, }) } + let row_dim = self.row_dimensions.get(row) + let conditional_format_copies = duplicate_xml_sqref_copies( + self.conditional_formats, + row, + target_row, + ) + let data_validation_copies = duplicate_xml_sqref_copies( + self.data_validations, + row, + target_row, + ) + let shifted_merges : Array[String] = [] + for range_ref in self.merged_cells { + shifted_merges.push(adjust_range_after_row_insert(range_ref, target_row, 1)) + } + let merge_copies = duplicate_merge_cell_copies( + shifted_merges, row, target_row, + ) + self.insert_rows( + target_row, + 1, + formula_limits~, + cancelled~, + formula_budget=translation_budget, + ) + self.cells.append(duplicated_cells) match row_dim { - Some(dim) => self.set_row_dimension(target_row, dim) + Some(dim) => + if row_dimension_is_default(dim) { + self.row_dimensions.remove(target_row) + } else { + self.row_dimensions[target_row] = dim + } None => self.row_dimensions.remove(target_row) } - self.duplicate_conditional_formats(row, target_row) - self.duplicate_data_validations(row, target_row) - self.duplicate_merge_cells(row, target_row) + self.conditional_formats.append(conditional_format_copies) + self.data_validations.append(data_validation_copies) + self.merged_cells.append(merge_copies) } ///| @@ -5944,6 +5978,24 @@ test "worksheet wb: duplicate-row helper edge branches" { ) } +///| +test "worksheet wb: duplicate row stages malformed sqref before mutation" { + let sheet = Worksheet::new("Atomic") + sheet.set_cell("A1", "source") + sheet.set_cell("A2", "tail") + sheet.add_data_validation_xml( + "", + ) + let result : Result[Unit, Error] = Ok(sheet.duplicate_row_to(1, 2)) catch { + error => Err(error) + } + assert_true(result is Err(_)) + debug_inspect(sheet.get_cell("A1"), content="Some(\"source\")") + debug_inspect(sheet.get_cell("A2"), content="Some(\"tail\")") + debug_inspect(sheet.get_cell("A3"), content="None") + inspect(sheet.data_validations().length(), content="1") +} + ///| test "worksheet wb: col insert/remove structural edge branches" { let guard_sheet = Worksheet::new("Guard") From 07ae177a3688f336ea7499d37593b04af5ed51ee Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 03:42:37 +0800 Subject: [PATCH 116/150] fix(xlsx): bound cooperative formatting metadata --- office/cmd/office/pkg.generated.mbti | 2 + office/cmd/office/xlsx_read_model.mbt | 10 ++ office/cmd/office/xlsx_read_output.mbt | 157 ++++++++++++++++--- office/cmd/office/xlsx_read_wbtest.mbt | 161 +++++++++++++++++-- xlsx/pkg.generated.mbti | 2 + xlsx/value_format.mbt | 206 +++++++++++++++++++++---- xlsx/value_format_test.mbt | 31 ++++ xlsx/worksheet.mbt | 115 ++++++++++---- 8 files changed, 588 insertions(+), 96 deletions(-) diff --git a/office/cmd/office/pkg.generated.mbti b/office/cmd/office/pkg.generated.mbti index 50957eac..219e1a96 100644 --- a/office/cmd/office/pkg.generated.mbti +++ b/office/cmd/office/pkg.generated.mbti @@ -67,6 +67,8 @@ type XlsxCellPredicate type XlsxCellSnapshot +type XlsxConditionalFormatBudget + type XlsxFormatBudget type XlsxFormulaBudget diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 2bd09ec0..67efd4d1 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -125,6 +125,16 @@ struct XlsxStoredCellBudget { mut used : Int } +///| +/// Command-wide budget for conditional-format metadata inspected while +/// producing workbook and sheet summaries. +struct XlsxConditionalFormatBudget { + maximum_ranges : Int + maximum_work_units : Int + mut ranges : Int + mut work_units : Int +} + ///| fn xlsx_cli_failure( code : String, diff --git a/office/cmd/office/xlsx_read_output.mbt b/office/cmd/office/xlsx_read_output.mbt index 7a3cbe8a..31b2d1e7 100644 --- a/office/cmd/office/xlsx_read_output.mbt +++ b/office/cmd/office/xlsx_read_output.mbt @@ -74,31 +74,132 @@ fn xlsx_sheet_state_name(state : @xlsx.SheetState) -> String { } ///| -fn xlsx_conditional_format_range_count( +let xlsx_cli_max_conditional_format_open_tag_chars : Int = 32 * 1024 + +///| +fn XlsxConditionalFormatBudget::new( + maximum_ranges? : Int = office_read_max_parser_items, + maximum_work_units? : Int = office_read_max_xml_part_bytes, +) -> XlsxConditionalFormatBudget { + { maximum_ranges, maximum_work_units, ranges: 0, work_units: 0 } +} + +///| +fn XlsxConditionalFormatBudget::reserve_work( + self : XlsxConditionalFormatBudget, + amount : Int, +) -> Unit raise CliFailure { + if amount < 0 || amount > self.maximum_work_units - self.work_units { + let actual = if amount < 0 || amount > 0x7fffffff - self.work_units { + 0x7fffffff + } else { + self.work_units + amount + } + raise xlsx_resource_failure( + "conditional-format count work units", + self.maximum_work_units, + actual~, + ) + } + self.work_units += amount +} + +///| +fn XlsxConditionalFormatBudget::reserve_ranges( + self : XlsxConditionalFormatBudget, + amount : Int, +) -> Unit raise CliFailure { + if amount < 0 || amount > self.maximum_ranges - self.ranges { + let actual = if amount < 0 || amount > 0x7fffffff - self.ranges { + 0x7fffffff + } else { + self.ranges + amount + } + raise xlsx_resource_failure( + "conditional-format ranges", + self.maximum_ranges, + actual~, + ) + } + self.ranges += amount +} + +///| +// Returns only the opening tag. Conditional-format bodies can be large, but +// range counting needs just `sqref`; bounding this tag keeps the subsequent +// synchronous XML scanner below one cooperative scheduling quantum. +fn bounded_conditional_format_open_tag( + xml : StringView, +) -> StringView raise CliFailure { + let mut quote : UInt16? = None + let mut index = 0 + while index < xml.length() { + if index >= xlsx_cli_max_conditional_format_open_tag_chars { + raise xlsx_resource_failure( + "conditional-format opening-tag work units", + xlsx_cli_max_conditional_format_open_tag_chars, + actual=index + 1, + ) + } + let unit = xml[index] + match quote { + Some(expected) => if unit == expected { quote = None } + None => + if unit == ('"' : UInt16) || unit == ('\'' : UInt16) { + quote = Some(unit) + } else if unit == ('>' : UInt16) { + return xml[:index + 1] + } + } + index += 1 + } + xml +} + +///| +async fn xlsx_conditional_format_range_count_with_budget( projection : XlsxProjection, worksheet : @xlsx.Worksheet, -) -> Int raise CliFailure { + budget : XlsxConditionalFormatBudget, +) -> Int { projection.checkpoint() - let count = worksheet.conditional_format_range_count_limited( - office_read_max_parser_items, - office_read_max_xml_part_bytes, - cancelled=projection.cancelled, - ) catch { - ReadCancelled => - raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") - error => raise xlsx_read_failure(error, projection.file) + let mut count = 0 + let mut work_since_yield = 0 + for xml in worksheet.conditional_formats() { + projection.checkpoint() + budget.reserve_work(xml.length()) + let opening_tag = bounded_conditional_format_open_tag(xml) + let fragment_count = @xlsx.conditional_format_xml_range_count_limited( + opening_tag, + budget.maximum_ranges, + opening_tag.length(), + cancelled=projection.cancelled, + ) catch { + ReadCancelled => + raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") + error => raise xlsx_read_failure(error, projection.file) + } + budget.reserve_ranges(fragment_count) + count += fragment_count + work_since_yield += xml.length() + if work_since_yield >= xlsx_cli_scan_yield_work_units { + @async.pause() + projection.checkpoint() + work_since_yield = 0 + } } projection.checkpoint() count } ///| -fn xlsx_sheet_summary_json( +async fn xlsx_sheet_summary_json( projection : XlsxProjection, sheet : XlsxSheetTarget, metadata : XlsxMetadataBudget, stored_cells : XlsxStoredCellBudget, -) -> Json raise CliFailure { + conditional_formats : XlsxConditionalFormatBudget, +) -> Json { projection.checkpoint() metadata.charge_item([sheet.name, sheet.path]) let fields : Map[String, Json] = { @@ -157,7 +258,9 @@ fn xlsx_sheet_summary_json( "hyperlinks": Json::number(hyperlink_count.to_double()), "data_validations": Json::number(validations.length().to_double()), "conditional_format_ranges": Json::number( - xlsx_conditional_format_range_count(projection, worksheet).to_double(), + xlsx_conditional_format_range_count_with_budget( + projection, worksheet, conditional_formats, + ).to_double(), ), "slicers": Json::number(slicers.length().to_double()), }) @@ -168,16 +271,20 @@ fn xlsx_sheet_summary_json( } ///| -fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { +async fn xlsx_outline_data(projection : XlsxProjection) -> Json { projection.checkpoint() let metadata = XlsxMetadataBudget::new() let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) + let conditional_formats = XlsxConditionalFormatBudget::new() let sheets : Array[Json] = [] for sheet in projection.sheets { projection.checkpoint() sheets.push( - xlsx_sheet_summary_json(projection, sheet, metadata, stored_cells), + xlsx_sheet_summary_json( + projection, sheet, metadata, stored_cells, conditional_formats, + ), ) + @async.pause() } let defined_names : Array[Json] = [] for defined in projection.workbook.defined_names() { @@ -240,10 +347,10 @@ fn xlsx_outline_data(projection : XlsxProjection) -> Json raise CliFailure { } ///| -fn xlsx_outline_payload( +async fn xlsx_outline_payload( projection : XlsxProjection, max_output_chars : Int, -) -> BoundedOfficePayload raise CliFailure { +) -> BoundedOfficePayload { projection.checkpoint() let sheets : Array[Json] = [] let defined_names : Array[Json] = [] @@ -521,21 +628,24 @@ async fn collect_xlsx_cell_page( } ///| -fn retain_xlsx_outline_records( +async fn retain_xlsx_outline_records( projection : XlsxProjection, sheets : Array[Json], defined_names : Array[Json], budget : OfficeOutputBudget, -) -> Unit raise CliFailure { +) -> Unit { let metadata = XlsxMetadataBudget::new() let stored_cells = XlsxStoredCellBudget::new(projection.max_scan_cells) + let conditional_formats = XlsxConditionalFormatBudget::new() for sheet in projection.sheets { projection.checkpoint() let record = xlsx_sheet_summary_json( - projection, sheet, metadata, stored_cells, + projection, sheet, metadata, stored_cells, conditional_formats, ) budget.reserve_array_item(record, !sheets.is_empty()) sheets.push(record) + @async.pause() + projection.checkpoint() } for defined in projection.workbook.defined_names() { projection.checkpoint() @@ -625,6 +735,7 @@ async fn xlsx_get_data( sheet, XlsxMetadataBudget::new(), XlsxStoredCellBudget::new(projection.max_scan_cells), + XlsxConditionalFormatBudget::new(), ), }) Cell(sheet, rect, path~) => { @@ -886,10 +997,10 @@ async fn xlsx_query_payload( } ///| -fn xlsx_outline_human( +async fn xlsx_outline_human( projection : XlsxProjection, maximum : Int, -) -> String raise CliFailure { +) -> String { projection.checkpoint() let output = DocxHumanWriter::new(maximum, format="xlsx") output.begin_line() @@ -918,6 +1029,8 @@ fn xlsx_outline_human( } } } + @async.pause() + projection.checkpoint() } projection.checkpoint() output.finish() diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 1340dddf..c9bd5b6a 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -122,17 +122,14 @@ async test "XLSX command cell scans yield before their bounded maximum" { } ///| -test "XLSX post-projection metadata, styles, and human output observe cancellation" { +async test "XLSX post-projection metadata, styles, and human output observe cancellation" { let projection = { ..xlsx_read_test_projection(), cancelled: () => true } guard projection.sheets[0].content is Worksheet(worksheet) else { fail("expected worksheet fixture") } for action in [ - () => ignore(xlsx_outline_data(projection)), - () => ignore(xlsx_conditional_format_range_count(projection, worksheet)), () => ignore(xlsx_styles_json(projection, [])), - () => ignore(xlsx_outline_human(projection, 10_000)), () => { ignore( xlsx_page_human( @@ -147,6 +144,22 @@ test "XLSX post-projection metadata, styles, and human output observe cancellati let error = xlsx_cli_error(() => action()) assert_eq(error.code, "office.cancelled") } + for + error in [ + xlsx_cli_error_async(() => ignore(xlsx_outline_data(projection))), + xlsx_cli_error_async(() => { + ignore( + xlsx_conditional_format_range_count_with_budget( + projection, + worksheet, + XlsxConditionalFormatBudget::new(), + ), + ) + }), + xlsx_cli_error_async(() => ignore(xlsx_outline_human(projection, 10_000))), + ] { + assert_eq(error.code, "office.cancelled") + } } ///| @@ -448,7 +461,7 @@ test "XLSX used-range preprocessing rejects excess stored cells before scanning" } ///| -test "XLSX used-range preprocessing shares one stored-cell budget across sheets" { +async test "XLSX used-range preprocessing shares one stored-cell budget across sheets" { let workbook = @xlsx.Workbook::new() ignore(workbook.new_sheet("First")) ignore(workbook.new_sheet("Second")) @@ -457,13 +470,16 @@ test "XLSX used-range preprocessing shares one stored-cell budget across sheets" let projection = make_xlsx_projection( "aggregate-stored-cells.xlsx", workbook, 1, ) - for - action in [ - () => ignore(xlsx_scan_regions(projection, None)), - () => ignore(xlsx_outline_data(projection)), - () => ignore(xlsx_outline_human(projection, 10_000)), - ] { - let limited = xlsx_cli_error(() => action()) + let limited_scan = xlsx_cli_error(() => { + ignore(xlsx_scan_regions(projection, None)) + }) + let limited_outline = xlsx_cli_error_async(() => { + ignore(xlsx_outline_data(projection)) + }) + let limited_human = xlsx_cli_error_async(() => { + ignore(xlsx_outline_human(projection, 10_000)) + }) + for limited in [limited_scan, limited_outline, limited_human] { assert_eq(limited.code, "office.xlsx.resource_limit") guard limited.details is Some(Object(details)) else { fail("expected aggregate stored-cell limit details") @@ -474,6 +490,96 @@ test "XLSX used-range preprocessing shares one stored-cell budget across sheets" } } +///| +async test "XLSX outline shares conditional-format budgets across sheets" { + let workbook = @xlsx.Workbook::new() + let first = workbook.new_sheet("First") + let second = workbook.new_sheet("Second") + let fragment = "" + first.add_conditional_format_xml(fragment) + second.add_conditional_format_xml(fragment) + let projection = make_xlsx_projection( + "aggregate-conditional-formats.xlsx", workbook, 10, + ) + let range_budget = XlsxConditionalFormatBudget::new( + maximum_ranges=1, + maximum_work_units=4096, + ) + assert_eq( + xlsx_conditional_format_range_count_with_budget( + projection, first, range_budget, + ), + 1, + ) + let range_error = xlsx_cli_error_async(() => { + ignore( + xlsx_conditional_format_range_count_with_budget( + projection, second, range_budget, + ), + ) + }) + assert_eq(range_error.code, "office.xlsx.resource_limit") + guard range_error.details is Some(Object(range_details)) else { + fail("expected aggregate conditional-format range details") + } + assert_eq( + range_details.get("resource"), + Some(Json::string("conditional-format ranges")), + ) + + let work_budget = XlsxConditionalFormatBudget::new( + maximum_ranges=10, + maximum_work_units=fragment.length() * 2 - 1, + ) + assert_eq( + xlsx_conditional_format_range_count_with_budget( + projection, first, work_budget, + ), + 1, + ) + let work_error = xlsx_cli_error_async(() => { + ignore( + xlsx_conditional_format_range_count_with_budget( + projection, second, work_budget, + ), + ) + }) + assert_eq(work_error.code, "office.xlsx.resource_limit") + guard work_error.details is Some(Object(work_details)) else { + fail("expected aggregate conditional-format work details") + } + assert_eq( + work_details.get("resource"), + Some(Json::string("conditional-format count work units")), + ) +} + +///| +async test "XLSX outline yields between worksheet summaries" { + let workbook = @xlsx.Workbook::new() + for index in 0..<8 { + ignore(workbook.new_sheet("Sheet\{index}")) + } + let projection = make_xlsx_projection( + "cooperative-outline.xlsx", + workbook, + 10, + cancelled=() => @async.is_being_cancelled(), + ) + @async.with_task_group() <| group => { + let task = group.spawn(() => ignore(xlsx_outline_data(projection))) + @async.pause() + task.cancel() + let cancelled = try { + ignore(task.wait()) + false + } catch { + error => @async.is_cancellation_error(error) + } + assert_true(cancelled) + } +} + ///| test "XLSX cell snapshots preserve typed values, formulas, and canonical paths" { let projection = xlsx_read_test_projection() @@ -896,7 +1002,7 @@ async test "XLSX formatting rejects text amplification before materialization" { ignore(workbook.new_sheet("Data")) workbook.set_cell_str("Data", "A1", "x".repeat(64 * 1024)) let style_id = workbook.new_style( - @xlsx.Style::number_format("@".repeat(64 * 1024)), + @xlsx.Style::number_format("@".repeat(16 * 1024)), ) workbook.set_cell_style("Data", "A1", style_id) let projection = make_xlsx_projection("amplified-format.xlsx", workbook, 10) @@ -917,6 +1023,33 @@ async test "XLSX formatting rejects text amplification before materialization" { ) } +///| +async test "XLSX formatting rejects scheduler-sized format programs" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_value("Data", "A1", Numeric(1.0)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("0".repeat(32 * 1024 + 1)), + ) + workbook.set_cell_style("Data", "A1", style_id) + let projection = make_xlsx_projection("format-program.xlsx", workbook, 10) + let regions = xlsx_scan_regions( + projection, + Some(resolve_xlsx_selector(projection, "/xlsx/sheet[name=\"Data\"]")), + ) + let limited = xlsx_cli_error_async(() => { + ignore(collect_xlsx_cell_page(projection, regions, TextCells, 0, 1)) + }) + assert_eq(limited.code, "office.xlsx.resource_limit") + guard limited.details is Some(Object(details)) else { + fail("expected format-program limit details") + } + assert_eq( + details.get("resource"), + Some(Json::string("formatted_number_format_work_units")), + ) +} + ///| async test "XLSX structured formatting contains percent scaling overflow" { let workbook = @xlsx.Workbook::new() @@ -1378,7 +1511,7 @@ async test "XLSX JSON and human output enforce format-specific stdout ceilings" } assert_eq(characters + docx_cli_output_framing_chars, payload.budget.used) } - let too_small = xlsx_cli_error(() => { + let too_small = xlsx_cli_error_async(() => { ignore(xlsx_outline_payload(projection, 8)) }) assert_eq(too_small.code, "office.xlsx.resource_limit") diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index ab01162d..f2825603 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -15,6 +15,8 @@ pub fn column_name_to_number(StringView) -> Int raise XlsxError pub fn column_number_to_name(Int) -> String raise XlsxError +pub fn conditional_format_xml_range_count_limited(StringView, Int, Int, cancelled? : () -> Bool) -> Int raise XlsxError + pub fn coordinates_to_cell_name(Int, Int, abs? : Bool) -> String raise XlsxError pub fn decrypt(BytesView, options? : Options, limits? : ReadLimits, cancelled? : () -> Bool) -> Bytes raise XlsxError diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index 78d2c7e4..734c9b03 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -39,7 +39,13 @@ fn format_cell_value_limited( Some(style) => match style.number_format { Some(format) => - format_number_with_format(raw, format, options, use_1904_format) + format_number_with_format( + raw, + format, + options, + use_1904_format, + max_output_chars?, + ) None => raw } None => raw @@ -49,6 +55,66 @@ fn format_cell_value_limited( output } +///| +// Bounded formatting is used from cooperative async commands. Keeping every +// synchronous format program below half of the command's 64-KiB work quantum +// ensures one cell cannot monopolize the scheduler before the caller reaches +// its next suspension point. +let max_bounded_format_program_chars : Int = 32 * 1024 + +///| +fn check_bounded_format_program( + program : StringView, + maximum : Int?, +) -> Unit raise XlsxError { + match maximum { + None => () + Some(_) => + if program.length() > max_bounded_format_program_chars { + raise ResourceLimitExceeded( + kind="formatted_number_format_work_units", + limit=max_bounded_format_program_chars, + actual=program.length(), + ) + } + } +} + +///| +// Proves a conservative upper bound before any formatted result is allocated. +// `program_multiplier` covers literal prefixes/suffixes, grouping separators, +// fraction placeholders, and date-token expansion. Callers select a tighter +// multiplier for the format family they are about to execute. +fn check_bounded_numeric_output( + fallback_chars : Int, + program_chars : Int, + program_multiplier : Int, + fixed_overhead : Int, + maximum : Int?, +) -> Unit raise XlsxError { + match maximum { + None => () + Some(limit) => { + if limit < 0 { + raise InvalidOptions(msg="formatted output limit must be non-negative") + } + if fallback_chars > limit || + program_chars < 0 || + program_multiplier < 0 || + fixed_overhead < 0 || + fixed_overhead > limit || + program_multiplier == 0 || + program_chars > (limit - fixed_overhead) / program_multiplier { + raise ResourceLimitExceeded( + kind="formatted_cell_value_chars", + limit~, + actual=if limit < 0x7fffffff { limit + 1 } else { limit }, + ) + } + } + } +} + ///| fn format_text_value( raw : String, @@ -61,6 +127,7 @@ fn format_text_value( if code.to_lower().trim().to_owned() == "general" { raw } else { + check_bounded_format_program(code, max_output_chars) format_text_pattern(raw, code, max_output_chars?) } } @@ -104,10 +171,25 @@ fn format_number_with_format( format : NumberFormat, options : Options, use_1904_format : Bool, + max_output_chars? : Int, ) -> String raise XlsxError { match format { - Builtin(id) => format_number_builtin(raw, id, options, use_1904_format) - Custom(code) => format_number_code(raw, code, options, use_1904_format~) + Builtin(id) => + format_number_builtin( + raw, + id, + options, + use_1904_format, + max_output_chars?, + ) + Custom(code) => + format_number_code( + raw, + code, + options, + use_1904_format~, + max_output_chars?, + ) } } @@ -117,51 +199,87 @@ fn format_number_builtin( id : Int, options : Options, use_1904_format : Bool, + max_output_chars? : Int, ) -> String raise XlsxError { let short_date = options.short_date_pattern let long_date = options.long_date_pattern let long_time = options.long_time_pattern match id { 0 => raw - 1 => format_number_simple(raw, 0, false, false) - 2 => format_number_simple(raw, 2, false, false) - 3 => format_number_simple(raw, 0, true, false) - 4 => format_number_simple(raw, 2, true, false) - 9 => format_number_simple(raw, 0, false, true) - 10 => format_number_simple(raw, 2, false, true) + 1 => { + check_bounded_numeric_output(raw.length(), 0, 1, 64, max_output_chars) + format_number_simple(raw, 0, false, false) + } + 2 => { + check_bounded_numeric_output(raw.length(), 0, 1, 64, max_output_chars) + format_number_simple(raw, 2, false, false) + } + 3 => { + check_bounded_numeric_output(raw.length(), 0, 1, 64, max_output_chars) + format_number_simple(raw, 0, true, false) + } + 4 => { + check_bounded_numeric_output(raw.length(), 0, 1, 64, max_output_chars) + format_number_simple(raw, 2, true, false) + } + 9 => { + check_bounded_numeric_output(raw.length(), 0, 1, 65, max_output_chars) + format_number_simple(raw, 0, false, true) + } + 10 => { + check_bounded_numeric_output(raw.length(), 0, 1, 65, max_output_chars) + format_number_simple(raw, 2, false, true) + } 14 => if short_date != "" { - format_excel_date(raw, short_date, use_1904_format~) + format_excel_date(raw, short_date, use_1904_format~, max_output_chars?) } else { - format_excel_date(raw, "mm-dd-yy", use_1904_format~) + format_excel_date(raw, "mm-dd-yy", use_1904_format~, max_output_chars?) } 15 => if long_date != "" { - format_excel_date(raw, long_date, use_1904_format~) + format_excel_date(raw, long_date, use_1904_format~, max_output_chars?) } else { - format_excel_date(raw, "d-mmm-yy", use_1904_format~) + format_excel_date(raw, "d-mmm-yy", use_1904_format~, max_output_chars?) } - 16 => format_excel_date(raw, "d-mmm", use_1904_format~) - 17 => format_excel_date(raw, "mmm-yy", use_1904_format~) - 18 => format_excel_date(raw, "h:mm AM/PM", use_1904_format~) - 19 => format_excel_date(raw, "h:mm:ss AM/PM", use_1904_format~) + 16 => format_excel_date(raw, "d-mmm", use_1904_format~, max_output_chars?) + 17 => format_excel_date(raw, "mmm-yy", use_1904_format~, max_output_chars?) + 18 => + format_excel_date(raw, "h:mm AM/PM", use_1904_format~, max_output_chars?) + 19 => + format_excel_date( + raw, + "h:mm:ss AM/PM", + use_1904_format~, + max_output_chars?, + ) 20 => if long_time != "" { - format_excel_date(raw, long_time, use_1904_format~) + format_excel_date(raw, long_time, use_1904_format~, max_output_chars?) } else { - format_excel_date(raw, "hh:mm", use_1904_format~) + format_excel_date(raw, "hh:mm", use_1904_format~, max_output_chars?) } 21 => if long_time != "" { - format_excel_date(raw, long_time, use_1904_format~) + format_excel_date(raw, long_time, use_1904_format~, max_output_chars?) } else { - format_excel_date(raw, "hh:mm:ss", use_1904_format~) + format_excel_date(raw, "hh:mm:ss", use_1904_format~, max_output_chars?) } 22 => if short_date != "" { - format_excel_date(raw, "\{short_date} hh:mm", use_1904_format~) + format_excel_date( + raw, + "\{short_date} hh:mm", + use_1904_format~, + max_output_chars?, + ) } else { - format_excel_date(raw, "m/d/yy hh:mm", use_1904_format~) + format_excel_date( + raw, + "m/d/yy hh:mm", + use_1904_format~, + max_output_chars?, + ) } // language-glyph builtin IDs resolve through the workbook culture 27..=36 | 50..=62 | 67..=81 => @@ -170,7 +288,13 @@ fn format_number_builtin( if code == "" { raw } else { - format_number_code(raw, code, options, use_1904_format~) + format_number_code( + raw, + code, + options, + use_1904_format~, + max_output_chars?, + ) } None => raw } @@ -184,8 +308,10 @@ fn format_number_code( code : String, options : Options, use_1904_format? : Bool = false, + max_output_chars? : Int, ) -> String raise XlsxError { let _ = options + check_bounded_format_program(code, max_output_chars) let sections = split_format_sections(code) if sections.length() == 0 { return raw @@ -194,7 +320,7 @@ fn format_number_code( e => Err(e) } match parsed { - Err(_) => format_text_pattern(raw, code) + Err(_) => format_text_pattern(raw, code, max_output_chars?) Ok(value) => { let (section, add_minus) = select_format_section(sections, value) let (pattern, locale, currency_prefix) = parse_section_metadata(section) @@ -203,6 +329,7 @@ fn format_number_code( } let normalized = pattern.to_lower() if normalized == "general" { + check_bounded_numeric_output(raw.length(), 0, 1, 0, max_output_chars) return format_general_number(raw) } if has_date_tokens(pattern) { @@ -210,15 +337,24 @@ fn format_number_code( Some(tag) => if !is_supported_locale(tag) { return raw } None => () } + check_bounded_numeric_output( + raw.length(), + code.length(), + 3, + 32, + max_output_chars, + ) let formatted = format_excel_date(raw, pattern, use_1904_format~) let output = currency_prefix + formatted return if add_minus { "-" + output } else { output } } if !pattern_has_number_placeholders(pattern) { + check_bounded_numeric_output(0, code.length(), 2, 2, max_output_chars) let literal = render_literal_segment(pattern) let output = currency_prefix + literal return if add_minus { "-" + output } else { output } } + check_bounded_numeric_output(0, code.length(), 4, 64, max_output_chars) format_number_pattern(raw, pattern, add_minus, currency_prefix) } } @@ -877,6 +1013,12 @@ fn format_scientific_number(value : Double, info : NumberPatternInfo) -> String ///| fn format_fraction_number(value : Double, info : NumberPatternInfo) -> String { + // `Double::to_int64` saturates rather than reporting overflow. Fraction + // formats cannot faithfully render an integer part outside Int64, so retain + // the finite numeric value instead of manufacturing the saturation bound. + if value > 0x7fff_ffff_ffff_ffffL.to_double() { + return value.to_string() + } let int_part = Double::to_int64(Double::floor(value)) let frac = value - int_part.to_double() let integer_text = format_int64_padded( @@ -1110,6 +1252,9 @@ fn format_fixed_number( } let scale = pow10_int(places) let scale64 = Int64::from_int(scale) + if abs_value > 0x7fff_ffff_ffff_ffffL.to_double() / Double::from_int(scale) { + return abs_value.to_string() + } let scaled = Double::round(abs_value * Double::from_int(scale)) let scaled_int = Double::to_int64(scaled) let int_part = if scale == 0 { scaled_int } else { scaled_int / scale64 } @@ -1214,7 +1359,16 @@ fn format_excel_date( raw : String, pattern : String, use_1904_format? : Bool = false, -) -> String { + max_output_chars? : Int, +) -> String raise XlsxError { + check_bounded_format_program(pattern, max_output_chars) + check_bounded_numeric_output( + raw.length(), + pattern.length(), + 2, + 32, + max_output_chars, + ) let value = @string.parse_double(raw) catch { _ => return raw } let parts = if use_1904_format { excel_serial_to_parts_1904(value) diff --git a/xlsx/value_format_test.mbt b/xlsx/value_format_test.mbt index ffbc5dff..a52661d8 100644 --- a/xlsx/value_format_test.mbt +++ b/xlsx/value_format_test.mbt @@ -131,4 +131,35 @@ test "fraction, scientific, percent" { inspect(format_numeric(0.0, "# ?/?"), content="0 ") inspect(format_numeric(123.456, "0.00E+00"), content="1.23E+02") inspect(format_numeric(0.256, "0.00%"), content="25.60%") + let large_fraction = format_numeric(1.0e20, "# ?/?") + assert_false(large_fraction.contains("9223372036854775807")) +} + +///| +test "bounded numeric formatting rejects amplification before allocation" { + let workbook = @xlsx.Workbook::new() + let sheet = workbook.new_sheet("Sheet1") + workbook.set_cell_value("Sheet1", "A1", Numeric(1.0)) + let style_id = workbook.new_style( + @xlsx.Style::number_format("0".repeat(1024)), + ) + workbook.set_cell_style("Sheet1", "A1", style_id) + try + workbook.get_cell_value_styled_from_worksheet_rc( + sheet, + 1, + 1, + style_id, + max_output_chars=64, + ) + catch { + ResourceLimitExceeded(kind~, limit~, actual~) => { + assert_eq(kind, "formatted_cell_value_chars") + assert_eq(limit, 64) + assert_true(actual > limit) + } + _ => fail("expected bounded numeric formatting failure") + } noraise { + _ => fail("expected bounded numeric formatting failure") + } } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 0c7c5748..a186fe83 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2596,6 +2596,71 @@ pub fn Worksheet::get_conditional_formats( /// fragments. Unlike `get_conditional_formats`, this does not materialize rule /// objects, maps, or copied range strings. Both retained-source work and the /// result are bounded, and long scans poll `cancelled`. +pub fn conditional_format_xml_range_count_limited( + xml : StringView, + maximum_ranges : Int, + maximum_work_units : Int, + cancelled? : () -> Bool = () => false, +) -> Int raise XlsxError { + if maximum_ranges < 0 || maximum_work_units < 0 { + raise InvalidOptions( + msg="conditional-format count limits must be non-negative", + ) + } + let (work, work_exceeded) = read_budget_actual( + 0, + xml.length(), + maximum_work_units, + ) + if work_exceeded { + raise ResourceLimitExceeded( + kind="conditional_format_count_work_units", + limit=maximum_work_units, + actual=work, + ) + } + check_read_cancelled(cancelled) + let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "conditionalFormatting" { + raise InvalidXml(msg="conditionalFormatting tag missing") + } + let sqref = scanner.attribute_view("", "sqref") catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + guard sqref is Some(sqref) else { + raise InvalidXml(msg="conditional formatting sqref missing") + } + let mut count = 0 + let mut in_token = false + for index in 0.. raise InvalidXml(msg~) - ReadCancelled => raise ReadCancelled - } - guard sqref is Some(sqref) else { - raise InvalidXml(msg="conditional formatting sqref missing") - } - let mut in_token = false - for index in 0.. Date: Sun, 19 Jul 2026 05:53:20 +0800 Subject: [PATCH 117/150] fix(xlsx): make structural cancellation atomic --- xlsx/defined_name_adjust.mbt | 17 +- xlsx/structural_edit_cancellation_test.mbt | 79 ++++ xlsx/workbook.mbt | 69 ++-- xlsx/worksheet.mbt | 456 ++++++++++++++++----- 4 files changed, 488 insertions(+), 133 deletions(-) create mode 100644 xlsx/structural_edit_cancellation_test.mbt diff --git a/xlsx/defined_name_adjust.mbt b/xlsx/defined_name_adjust.mbt index 872c0ee9..e05a2887 100644 --- a/xlsx/defined_name_adjust.mbt +++ b/xlsx/defined_name_adjust.mbt @@ -342,7 +342,9 @@ fn adjust_defined_name_refers_to_for_sheet( refers_to : StringView, sheet_name : StringView, adjust_fn : (StringView) -> String? raise XlsxError, + cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { + check_read_cancelled(cancelled) let raw_chars = refers_to.to_array() let plain_prefix = "\{sheet_name.to_owned()}!" let quoted_prefix = "'\{escape_sheet_name(sheet_name)}'!" @@ -354,12 +356,15 @@ fn adjust_defined_name_refers_to_for_sheet( text : Array[Char], prefix : Array[Char], start : Int, - ) -> Int? { + ) -> Int? raise XlsxError { if prefix.length() == 0 { return None } let mut pos = start while pos + prefix.length() <= text.length() { + if (pos & 4095) == 0 { + check_read_cancelled(cancelled) + } let mut matched = true for i in 0.. { for i in idx.. { for i in idx.. @xlsx.Workbook raise { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + // Crosses the 4096-item checkpoint boundary without using shared formulas, + // so the test exercises the ordinary structural staging loops. + for row in 1..<=4097 { + workbook.set_cell_int("Data", "A\{row}", row) + } + workbook +} + +///| +fn assert_plain_structural_edit_cancellation_is_atomic( + edit : (@xlsx.Workbook, () -> Bool) -> Unit raise, +) -> Unit raise { + let workbook = large_plain_structural_edit_fixture() + let before = @xlsx.write(workbook) + let checks = [0] + let result : Result[Unit, Error] = Ok( + edit(workbook, () => { + checks[0] += 1 + // Validation and preflight consume the first checkpoints. This threshold + // delivers cancellation only after ordinary-cell staging has begun. + checks[0] >= 16 + }), + ) catch { + error => Err(error) + } + assert_true(result is Err(@xlsx.ReadCancelled)) + assert_true(checks[0] >= 16) + assert_eq(@xlsx.write(workbook), before) +} + +///| +test "plain structural edits poll late cancellation atomically" { + assert_plain_structural_edit_cancellation_is_atomic((workbook, cancelled) => { + workbook.insert_rows("Data", 2, 1, cancelled~) + }) + assert_plain_structural_edit_cancellation_is_atomic((workbook, cancelled) => { + workbook.remove_rows("Data", 2, 1, cancelled~) + }) + assert_plain_structural_edit_cancellation_is_atomic((workbook, cancelled) => { + workbook.insert_cols("Data", 2, 1, cancelled~) + }) + assert_plain_structural_edit_cancellation_is_atomic((workbook, cancelled) => { + workbook.remove_cols("Data", 2, 1, cancelled~) + }) + assert_plain_structural_edit_cancellation_is_atomic((workbook, cancelled) => { + workbook.duplicate_row_to("Data", 1, 2, cancelled~) + }) +} + +///| +test "defined-name staging polls late cancellation atomically" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.new_sheet("Data")) + workbook.set_cell_str("Data", "A1", "value") + workbook.set_defined_name( + @xlsx.DefinedName::new( + "LongName", + "Data!A1+" + "1+".repeat(6000) + "1", + scope="Data", + ), + ) + let before = @xlsx.write(workbook) + let checks = [0] + let result : Result[Unit, Error] = Ok( + workbook.insert_rows("Data", 1, 1, cancelled=() => { + checks[0] += 1 + checks[0] >= 6 + }), + ) catch { + error => Err(error) + } + assert_true(result is Err(@xlsx.ReadCancelled)) + assert_true(checks[0] >= 6) + assert_eq(@xlsx.write(workbook), before) +} diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 5d9fdfc1..53a354b5 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1014,16 +1014,22 @@ fn Workbook::stage_adjusted_defined_names_for_sheet( self : Workbook, sheet_name : String, adjust_fn : (StringView) -> String? raise XlsxError, + cancelled? : () -> Bool = () => false, ) -> Array[DefinedName]? raise XlsxError { + check_read_cancelled(cancelled) if self.defined_names.length() == 0 { return None } let updated : Array[DefinedName] = [] - for dn in self.defined_names { + for index, dn in self.defined_names { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let refers_to = adjust_defined_name_refers_to_for_sheet( dn.refers_to, sheet_name, adjust_fn, + cancelled~, ) updated.push({ name: dn.name, @@ -1032,6 +1038,7 @@ fn Workbook::stage_adjusted_defined_names_for_sheet( comment: dn.comment, }) } + check_read_cancelled(cancelled) Some(updated) } @@ -3936,11 +3943,13 @@ pub fn Workbook::insert_rows( // then commit it only after the sheet insert also succeeds. Both the sheet // insert and the defined-name adjuster raise on an out-of-grid shift; ordering // the two commits last makes the whole operation all-or-nothing. - let staged_names = self.stage_adjusted_defined_names_for_sheet(sheet.name(), fn( - ref_text : StringView, - ) -> String? raise XlsxError { - adjust_ref_after_row_insert(ref_text, row, count) - }) + let staged_names = self.stage_adjusted_defined_names_for_sheet( + sheet.name(), + fn(ref_text : StringView) -> String? raise XlsxError { + adjust_ref_after_row_insert(ref_text, row, count) + }, + cancelled~, + ) sheet.insert_rows(row, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -3968,11 +3977,13 @@ pub fn Workbook::remove_rows( // Stage the defined-name adjustment before mutating the sheet, then commit it // last — so if adjustment raises (e.g. a pre-existing out-of-grid name) the // removal is all-or-nothing, matching insert. - let staged_names = self.stage_adjusted_defined_names_for_sheet(sheet.name(), fn( - ref_text : StringView, - ) -> String? raise XlsxError { - adjust_ref_after_row_remove(ref_text, row, count) - }) + let staged_names = self.stage_adjusted_defined_names_for_sheet( + sheet.name(), + fn(ref_text : StringView) -> String? raise XlsxError { + adjust_ref_after_row_remove(ref_text, row, count) + }, + cancelled~, + ) sheet.remove_rows(row, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -4017,11 +4028,13 @@ pub fn Workbook::insert_cols( } // See insert_rows: stage the defined-name shift, mutate the sheet, then commit // the names last so an out-of-grid overflow on either side is all-or-nothing. - let staged_names = self.stage_adjusted_defined_names_for_sheet(sheet.name(), fn( - ref_text : StringView, - ) -> String? raise XlsxError { - adjust_ref_after_col_insert(ref_text, col, count) - }) + let staged_names = self.stage_adjusted_defined_names_for_sheet( + sheet.name(), + fn(ref_text : StringView) -> String? raise XlsxError { + adjust_ref_after_col_insert(ref_text, col, count) + }, + cancelled~, + ) sheet.insert_cols(col, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -4046,11 +4059,13 @@ pub fn Workbook::remove_cols( ) -> Unit raise XlsxError { let sheet = self.require_sheet(sheet_name) // Stage names before mutating the sheet (see remove_rows). - let staged_names = self.stage_adjusted_defined_names_for_sheet(sheet.name(), fn( - ref_text : StringView, - ) -> String? raise XlsxError { - adjust_ref_after_col_remove(ref_text, col, count) - }) + let staged_names = self.stage_adjusted_defined_names_for_sheet( + sheet.name(), + fn(ref_text : StringView) -> String? raise XlsxError { + adjust_ref_after_col_remove(ref_text, col, count) + }, + cancelled~, + ) sheet.remove_cols(col, count, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } @@ -4116,11 +4131,13 @@ pub fn Workbook::duplicate_row_to( // Stage the defined-name shift (which raises if a name would leave the grid) // before duplicating the row, then commit the names last — otherwise an // overflowing name would raise after the worksheet had already been mutated. - let staged_names = self.stage_adjusted_defined_names_for_sheet(sheet.name(), fn( - ref_text : StringView, - ) -> String? raise XlsxError { - adjust_ref_after_row_insert(ref_text, target_row, 1) - }) + let staged_names = self.stage_adjusted_defined_names_for_sheet( + sheet.name(), + fn(ref_text : StringView) -> String? raise XlsxError { + adjust_ref_after_row_insert(ref_text, target_row, 1) + }, + cancelled~, + ) sheet.duplicate_row_to(row, target_row, formula_limits~, cancelled~) self.commit_adjusted_defined_names(staged_names) } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index a186fe83..de37396a 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -3608,6 +3608,7 @@ fn Worksheet::insert_rows( translation_budget, fn(_cell : Cell) -> Bool { true }, ) + translation_budget.checkpoint() // Transactional: stage every shifted collection into locals first — cell, // merge, hyperlink, filter, table, sparkline, image, and chart references // shift through `cell_ref_from`/range adjusters that raise once a coordinate @@ -3616,7 +3617,10 @@ fn Worksheet::insert_rows( // `self` is mutated until every stage succeeds, so an overflowing insert // raises and leaves the worksheet unchanged instead of half-shifted. let updated_cells : Array[Cell] = [] - for stored_cell in self.cells { + for index, stored_cell in self.cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let cell = normal_formula_cell_for_structural_edit( stored_cell, shared_formula_masters, translation_budget, ) @@ -3642,11 +3646,17 @@ fn Worksheet::insert_rows( } } let updated_merges : Array[String] = [] - for range_ref in self.merged_cells { + for index, range_ref in self.merged_cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } updated_merges.push(adjust_range_after_row_insert(range_ref, row, count)) } let updated_links : Array[Hyperlink] = [] - for link in self.hyperlinks { + for index, link in self.hyperlinks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_hyperlink_ref_after_row_insert( link.reference, row, @@ -3661,34 +3671,63 @@ fn Worksheet::insert_rows( tooltip: link.tooltip, }) } + translation_budget.checkpoint() let updated_filter = match self.auto_filter { Some(filter) => Some(adjust_auto_filter_after_row_insert(filter, row, count)) None => None } let updated_tables : Array[Table] = [] - for table in self.tables { - updated_tables.push(adjust_table_after_row_insert(table, row, count)) + for index, table in self.tables { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + updated_tables.push( + adjust_table_after_row_insert( + table, + row, + count, + cancelled=translation_budget.cancelled, + ), + ) } let updated_sparklines : Array[SparklineGroup] = [] - for group in self.sparkline_groups { - let adjusted = adjust_sparkline_group_after_row_insert(group, row, count) + for index, group in self.sparkline_groups { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + let adjusted = adjust_sparkline_group_after_row_insert( + group, + row, + count, + cancelled=translation_budget.cancelled, + ) if adjusted.sparklines.length() > 0 { updated_sparklines.push(adjusted) } } let updated_images : Array[Image] = [] - for image in self.images { + for index, image in self.images { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_cell_after_row_insert(image.reference, row, count) updated_images.push(image_with_reference(image, reference)) } let updated_charts : Array[Chart] = [] - for chart in self.charts { + for index, chart in self.charts { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_cell_after_row_insert(chart.reference, row, count) updated_charts.push(chart_with_reference(chart, reference)) } let updated_rows : Map[Int, RowDimension] = Map([]) + let mut dimension_index = 0 for row_index, dim in self.row_dimensions { + if (dimension_index & 4095) == 0 { + translation_budget.checkpoint() + } let new_row = if row_index >= row { // Overflow-safe: a stored dimension key can be arbitrarily large (the // dimension setters do not bound it), so validate BEFORE the addition. @@ -3700,12 +3739,16 @@ fn Worksheet::insert_rows( row_index } updated_rows[new_row] = dim + dimension_index += 1 } let break_start = row - 1 // Break ids are 0-based (id = row - 1), so a shifted id must stay strictly // below the row maximum. Check the original ids overflow-safely, before // adjust_page_breaks_after_insert performs `id + count`. - for brk in self.row_breaks { + for index, brk in self.row_breaks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } if brk.id >= break_start && brk.id > cell_ref_max_rows - 1 - count { raise InvalidCellRef(value="\{0}:\{brk.id}") } @@ -3714,9 +3757,11 @@ fn Worksheet::insert_rows( self.row_breaks, break_start, count, + cancelled=translation_budget.cancelled, ) // Commit: every stage above succeeded, so applying the staged state cannot // fail partway. + translation_budget.checkpoint() self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() self.cells.clear() @@ -3782,8 +3827,12 @@ fn Worksheet::remove_rows( translation_budget, fn(cell : Cell) -> Bool { cell.row < row || cell.row > end_row }, ) + translation_budget.checkpoint() let updated_cells : Array[Cell] = [] - for stored_cell in self.cells { + for index, stored_cell in self.cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } if stored_cell.row >= row && stored_cell.row <= end_row { continue } @@ -3811,21 +3860,21 @@ fn Worksheet::remove_rows( style_id: cell.style_id, }) } - self.invalidate_shared_formula_masters_index() - self.invalidate_cell_index() - self.cells.clear() - self.cells.append(updated_cells) let updated_merges : Array[String] = [] - for range_ref in self.merged_cells { + for index, range_ref in self.merged_cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_range_after_row_remove(range_ref, row, count) { Some(value) => updated_merges.push(value) None => () } } - self.merged_cells.clear() - self.merged_cells.append(updated_merges) let updated_links : Array[Hyperlink] = [] - for link in self.hyperlinks { + for index, link in self.hyperlinks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_hyperlink_ref_after_row_remove(link.reference, row, count) { Some(reference) => updated_links.push({ @@ -3839,72 +3888,112 @@ fn Worksheet::remove_rows( None => () } } - self.hyperlinks.clear() - self.hyperlinks.append(updated_links) - match self.auto_filter { + translation_budget.checkpoint() + let updated_filter = match self.auto_filter { Some(filter) => match adjust_auto_filter_after_row_remove(filter, row, count) { - Some(updated) => self.auto_filter = Some(updated) - None => self.auto_filter = None + Some(updated) => Some(updated) + None => None } - None => () + None => None } let updated_tables : Array[Table] = [] - for table in self.tables { - match adjust_table_after_row_remove(table, row, count) { + for index, table in self.tables { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + match + adjust_table_after_row_remove( + table, + row, + count, + cancelled=translation_budget.cancelled, + ) { Some(value) => updated_tables.push(value) None => () } } - self.tables.clear() - self.tables.append(updated_tables) let updated_sparklines : Array[SparklineGroup] = [] - for group in self.sparkline_groups { - let adjusted = adjust_sparkline_group_after_row_remove(group, row, count) + for index, group in self.sparkline_groups { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + let adjusted = adjust_sparkline_group_after_row_remove( + group, + row, + count, + cancelled=translation_budget.cancelled, + ) if adjusted.sparklines.length() > 0 { updated_sparklines.push(adjusted) } } - self.sparkline_groups.clear() - self.sparkline_groups.append(updated_sparklines) let updated_images : Array[Image] = [] - for image in self.images { + for index, image in self.images { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_cell_after_row_remove(image.reference, row, count) { Some(reference) => updated_images.push(image_with_reference(image, reference)) None => () } } - self.images.clear() - self.images.append(updated_images) let updated_charts : Array[Chart] = [] - for chart in self.charts { + for index, chart in self.charts { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_cell_after_row_remove(chart.reference, row, count) { Some(reference) => updated_charts.push(chart_with_reference(chart, reference)) None => () } } - self.charts.clear() - self.charts.append(updated_charts) let updated_rows : Map[Int, RowDimension] = Map([]) + let mut dimension_index = 0 for row_index, dim in self.row_dimensions { + if (dimension_index & 4095) == 0 { + translation_budget.checkpoint() + } if row_index < row { updated_rows[row_index] = dim } else if row_index > end_row { updated_rows[row_index - count] = dim } - } - self.row_dimensions.clear() - for row_index, dim in updated_rows { - self.row_dimensions[row_index] = dim + dimension_index += 1 } let break_start = row - 1 let updated_breaks = adjust_page_breaks_after_remove( self.row_breaks, break_start, count, + cancelled=translation_budget.cancelled, ) + // One final cancellation point precedes the only mutation phase. No + // cancellation callback runs after this point, so cancellation is atomic. + translation_budget.checkpoint() + self.invalidate_shared_formula_masters_index() + self.invalidate_cell_index() + self.cells.clear() + self.cells.append(updated_cells) + self.merged_cells.clear() + self.merged_cells.append(updated_merges) + self.hyperlinks.clear() + self.hyperlinks.append(updated_links) + self.auto_filter = updated_filter + self.tables.clear() + self.tables.append(updated_tables) + self.sparkline_groups.clear() + self.sparkline_groups.append(updated_sparklines) + self.images.clear() + self.images.append(updated_images) + self.charts.clear() + self.charts.append(updated_charts) + self.row_dimensions.clear() + for row_index, dim in updated_rows { + self.row_dimensions[row_index] = dim + } self.row_breaks.clear() self.row_breaks.append(updated_breaks) } @@ -4017,14 +4106,19 @@ fn duplicate_xml_sqref_copies( entries : ArrayView[String], row : Int, target_row : Int, + cancelled? : () -> Bool = () => false, ) -> Array[String] raise XlsxError { let copies : Array[String] = [] - for xml in entries { + for index, xml in entries { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } match duplicate_xml_sqref(xml, row, target_row) { Some(value) => copies.push(value) None => () } } + check_read_cancelled(cancelled) copies } @@ -4049,19 +4143,26 @@ fn duplicate_merge_cell_copies( merged_cells : ArrayView[String], row : Int, target_row : Int, + cancelled? : () -> Bool = () => false, ) -> Array[String] raise XlsxError { let copies : Array[String] = [] let mut source_row = row if row > target_row { source_row = row + 1 } - for range_ref in merged_cells { + for index, range_ref in merged_cells { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let (min_row, _min_col, max_row, _max_col) = parse_range_ref(range_ref) if min_row < target_row && target_row < max_row { return copies } } - for range_ref in merged_cells { + for index, range_ref in merged_cells { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let (min_row, min_col, max_row, max_col) = parse_range_ref(range_ref) if min_row == max_row && min_row == source_row { let from_ref = cell_ref_from(target_row, min_col) @@ -4072,6 +4173,7 @@ fn duplicate_merge_cell_copies( } } } + check_read_cancelled(cancelled) copies } @@ -4123,7 +4225,10 @@ fn Worksheet::duplicate_row_to( // transactional shift. Once that insert succeeds, the remaining appends and // map assignments below are infallible state installation. let duplicated_cells : Array[Cell] = [] - for stored_cell in self.cells { + for index, stored_cell in self.cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } if stored_cell.row != row { continue } @@ -4151,19 +4256,28 @@ fn Worksheet::duplicate_row_to( self.conditional_formats, row, target_row, + cancelled=translation_budget.cancelled, ) let data_validation_copies = duplicate_xml_sqref_copies( self.data_validations, row, target_row, + cancelled=translation_budget.cancelled, ) let shifted_merges : Array[String] = [] - for range_ref in self.merged_cells { + for index, range_ref in self.merged_cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } shifted_merges.push(adjust_range_after_row_insert(range_ref, target_row, 1)) } let merge_copies = duplicate_merge_cell_copies( - shifted_merges, row, target_row, + shifted_merges, + row, + target_row, + cancelled=translation_budget.cancelled, ) + translation_budget.checkpoint() self.insert_rows( target_row, 1, @@ -4221,12 +4335,16 @@ fn Worksheet::insert_cols( translation_budget, fn(_cell : Cell) -> Bool { true }, ) + translation_budget.checkpoint() // Transactional (see insert_rows): stage everything into locals — reference // shifts raise once a coordinate leaves the grid, column dimensions and page // breaks are bound-checked explicitly — and commit only once every stage has // succeeded, so an overflowing insert leaves the worksheet unchanged. let updated_cells : Array[Cell] = [] - for stored_cell in self.cells { + for index, stored_cell in self.cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let cell = normal_formula_cell_for_structural_edit( stored_cell, shared_formula_masters, translation_budget, ) @@ -4252,11 +4370,17 @@ fn Worksheet::insert_cols( } } let updated_merges : Array[String] = [] - for range_ref in self.merged_cells { + for index, range_ref in self.merged_cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } updated_merges.push(adjust_range_after_col_insert(range_ref, col, count)) } let updated_links : Array[Hyperlink] = [] - for link in self.hyperlinks { + for index, link in self.hyperlinks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_hyperlink_ref_after_col_insert( link.reference, col, @@ -4271,34 +4395,63 @@ fn Worksheet::insert_cols( tooltip: link.tooltip, }) } + translation_budget.checkpoint() let updated_filter = match self.auto_filter { Some(filter) => Some(adjust_auto_filter_after_col_insert(filter, col, count)) None => None } let updated_tables : Array[Table] = [] - for table in self.tables { - updated_tables.push(adjust_table_after_col_insert(table, col, count)) + for index, table in self.tables { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + updated_tables.push( + adjust_table_after_col_insert( + table, + col, + count, + cancelled=translation_budget.cancelled, + ), + ) } let updated_sparklines : Array[SparklineGroup] = [] - for group in self.sparkline_groups { - let adjusted = adjust_sparkline_group_after_col_insert(group, col, count) + for index, group in self.sparkline_groups { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + let adjusted = adjust_sparkline_group_after_col_insert( + group, + col, + count, + cancelled=translation_budget.cancelled, + ) if adjusted.sparklines.length() > 0 { updated_sparklines.push(adjusted) } } let updated_images : Array[Image] = [] - for image in self.images { + for index, image in self.images { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_cell_after_col_insert(image.reference, col, count) updated_images.push(image_with_reference(image, reference)) } let updated_charts : Array[Chart] = [] - for chart in self.charts { + for index, chart in self.charts { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } let reference = adjust_cell_after_col_insert(chart.reference, col, count) updated_charts.push(chart_with_reference(chart, reference)) } let updated_cols : Map[Int, ColDimension] = Map([]) + let mut dimension_index = 0 for col_index, dim in self.col_dimensions { + if (dimension_index & 4095) == 0 { + translation_budget.checkpoint() + } let new_col = if col_index >= col { // Overflow-safe (see insert_rows): stored keys are not bounded by their // setters, so validate before the addition. @@ -4310,10 +4463,14 @@ fn Worksheet::insert_cols( col_index } updated_cols[new_col] = dim + dimension_index += 1 } let break_start = col - 1 // 0-based ids (see insert_rows): validate overflow-safely before the shift. - for brk in self.col_breaks { + for index, brk in self.col_breaks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } if brk.id >= break_start && brk.id > cell_ref_max_cols - 1 - count { raise InvalidCellRef(value="\{brk.id}:\{0}") } @@ -4322,8 +4479,10 @@ fn Worksheet::insert_cols( self.col_breaks, break_start, count, + cancelled=translation_budget.cancelled, ) // Commit. + translation_budget.checkpoint() self.invalidate_shared_formula_masters_index() self.invalidate_cell_index() self.cells.clear() @@ -4388,8 +4547,12 @@ fn Worksheet::remove_cols( translation_budget, fn(cell : Cell) -> Bool { cell.col < col || cell.col > end_col }, ) + translation_budget.checkpoint() let updated_cells : Array[Cell] = [] - for stored_cell in self.cells { + for index, stored_cell in self.cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } if stored_cell.col >= col && stored_cell.col <= end_col { continue } @@ -4417,21 +4580,21 @@ fn Worksheet::remove_cols( style_id: cell.style_id, }) } - self.invalidate_shared_formula_masters_index() - self.invalidate_cell_index() - self.cells.clear() - self.cells.append(updated_cells) let updated_merges : Array[String] = [] - for range_ref in self.merged_cells { + for index, range_ref in self.merged_cells { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_range_after_col_remove(range_ref, col, count) { Some(value) => updated_merges.push(value) None => () } } - self.merged_cells.clear() - self.merged_cells.append(updated_merges) let updated_links : Array[Hyperlink] = [] - for link in self.hyperlinks { + for index, link in self.hyperlinks { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_hyperlink_ref_after_col_remove(link.reference, col, count) { Some(reference) => updated_links.push({ @@ -4445,72 +4608,110 @@ fn Worksheet::remove_cols( None => () } } - self.hyperlinks.clear() - self.hyperlinks.append(updated_links) - match self.auto_filter { + translation_budget.checkpoint() + let updated_filter = match self.auto_filter { Some(filter) => match adjust_auto_filter_after_col_remove(filter, col, count) { - Some(updated) => self.auto_filter = Some(updated) - None => self.auto_filter = None + Some(updated) => Some(updated) + None => None } - None => () + None => None } let updated_tables : Array[Table] = [] - for table in self.tables { - match adjust_table_after_col_remove(table, col, count) { + for index, table in self.tables { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + match + adjust_table_after_col_remove( + table, + col, + count, + cancelled=translation_budget.cancelled, + ) { Some(value) => updated_tables.push(value) None => () } } - self.tables.clear() - self.tables.append(updated_tables) let updated_sparklines : Array[SparklineGroup] = [] - for group in self.sparkline_groups { - let adjusted = adjust_sparkline_group_after_col_remove(group, col, count) + for index, group in self.sparkline_groups { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } + let adjusted = adjust_sparkline_group_after_col_remove( + group, + col, + count, + cancelled=translation_budget.cancelled, + ) if adjusted.sparklines.length() > 0 { updated_sparklines.push(adjusted) } } - self.sparkline_groups.clear() - self.sparkline_groups.append(updated_sparklines) let updated_images : Array[Image] = [] - for image in self.images { + for index, image in self.images { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_cell_after_col_remove(image.reference, col, count) { Some(reference) => updated_images.push(image_with_reference(image, reference)) None => () } } - self.images.clear() - self.images.append(updated_images) let updated_charts : Array[Chart] = [] - for chart in self.charts { + for index, chart in self.charts { + if (index & 4095) == 0 { + translation_budget.checkpoint() + } match adjust_cell_after_col_remove(chart.reference, col, count) { Some(reference) => updated_charts.push(chart_with_reference(chart, reference)) None => () } } - self.charts.clear() - self.charts.append(updated_charts) let updated_cols : Map[Int, ColDimension] = Map([]) + let mut dimension_index = 0 for col_index, dim in self.col_dimensions { + if (dimension_index & 4095) == 0 { + translation_budget.checkpoint() + } if col_index < col { updated_cols[col_index] = dim } else if col_index > end_col { updated_cols[col_index - count] = dim } - } - self.col_dimensions.clear() - for col_index, dim in updated_cols { - self.col_dimensions[col_index] = dim + dimension_index += 1 } let break_start = col - 1 let updated_breaks = adjust_page_breaks_after_remove( self.col_breaks, break_start, count, + cancelled=translation_budget.cancelled, ) + translation_budget.checkpoint() + self.invalidate_shared_formula_masters_index() + self.invalidate_cell_index() + self.cells.clear() + self.cells.append(updated_cells) + self.merged_cells.clear() + self.merged_cells.append(updated_merges) + self.hyperlinks.clear() + self.hyperlinks.append(updated_links) + self.auto_filter = updated_filter + self.tables.clear() + self.tables.append(updated_tables) + self.sparkline_groups.clear() + self.sparkline_groups.append(updated_sparklines) + self.images.clear() + self.images.append(updated_images) + self.charts.clear() + self.charts.append(updated_charts) + self.col_dimensions.clear() + for col_index, dim in updated_cols { + self.col_dimensions[col_index] = dim + } self.col_breaks.clear() self.col_breaks.append(updated_breaks) } @@ -4572,12 +4773,17 @@ fn adjust_page_breaks_after_insert( breaks : Array[PageBreak], start : Int, count : Int, -) -> Array[PageBreak] { + cancelled? : () -> Bool = () => false, +) -> Array[PageBreak] raise XlsxError { let updated : Array[PageBreak] = [] - for brk in breaks { + for index, brk in breaks { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let new_id = if brk.id >= start { brk.id + count } else { brk.id } updated.push({ id: new_id, min: brk.min, max: brk.max, manual: brk.manual }) } + check_read_cancelled(cancelled) updated } @@ -4586,10 +4792,14 @@ fn adjust_page_breaks_after_remove( breaks : Array[PageBreak], start : Int, count : Int, -) -> Array[PageBreak] { + cancelled? : () -> Bool = () => false, +) -> Array[PageBreak] raise XlsxError { let updated : Array[PageBreak] = [] let end = start + count - 1 - for brk in breaks { + for index, brk in breaks { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } if brk.id < start { updated.push(brk) continue @@ -4603,6 +4813,7 @@ fn adjust_page_breaks_after_remove( }) } } + check_read_cancelled(cancelled) updated } @@ -4859,11 +5070,18 @@ fn adjust_cell_after_col_remove( } ///| -fn clone_table_columns(columns : ArrayView[String]) -> Array[String] { +fn clone_table_columns( + columns : ArrayView[String], + cancelled? : () -> Bool = () => false, +) -> Array[String] raise XlsxError { let out : Array[String] = [] - for column in columns { + for index, column in columns { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } out.push(column) } + check_read_cancelled(cancelled) out } @@ -4877,6 +5095,7 @@ fn adjust_table_after_row_insert( table : Table, row : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> Table raise XlsxError { let range_ref = adjust_range_after_row_insert(table.range_ref, row, count) { @@ -4885,7 +5104,7 @@ fn adjust_table_after_row_insert( display_name: table.display_name, range: range_ref, range_ref, - columns: clone_table_columns(table.columns), + columns: clone_table_columns(table.columns, cancelled~), style_name: table.style_name, show_first_column: table.show_first_column, show_last_column: table.show_last_column, @@ -4900,6 +5119,7 @@ fn adjust_table_after_row_remove( table : Table, row : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> Table? raise XlsxError { match adjust_range_after_row_remove(table.range_ref, row, count) { Some(range_ref) => @@ -4909,7 +5129,7 @@ fn adjust_table_after_row_remove( display_name: table.display_name, range: range_ref, range_ref, - columns: clone_table_columns(table.columns), + columns: clone_table_columns(table.columns, cancelled~), style_name: table.style_name, show_first_column: table.show_first_column, show_last_column: table.show_last_column, @@ -4926,17 +5146,21 @@ fn adjust_table_after_col_insert( table : Table, col : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> Table raise XlsxError { let (min_row, min_col, max_row, max_col) = parse_range_ref(table.range_ref) let (new_min_col, new_max_col) = adjust_bounds_for_insert( min_col, max_col, col, count, ) - let columns = clone_table_columns(table.columns) + let columns = clone_table_columns(table.columns, cancelled~) if col > min_col && col <= max_col { let insert_index = col - min_col let base = columns.length() + 1 let mut i = 0 while i < count { + if (i & 4095) == 0 { + check_read_cancelled(cancelled) + } columns.insert(insert_index + i, table_default_column_name(base + i)) i = i + 1 } @@ -4964,11 +5188,12 @@ fn adjust_table_after_col_remove( table : Table, col : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> Table? raise XlsxError { let (min_row, min_col, max_row, max_col) = parse_range_ref(table.range_ref) match adjust_bounds_for_remove(min_col, max_col, col, count) { Some((new_min_col, new_max_col)) => { - let columns = clone_table_columns(table.columns) + let columns = clone_table_columns(table.columns, cancelled~) let end_col = col + count - 1 if !(end_col < min_col || col > max_col) { let overlap_start = if col > min_col { col } else { min_col } @@ -4977,6 +5202,9 @@ fn adjust_table_after_col_remove( let remove_index = overlap_start - min_col let mut i = 0 while i < remove_count { + if (i & 4095) == 0 { + check_read_cancelled(cancelled) + } ignore(columns.remove(remove_index)) i = i + 1 } @@ -5007,9 +5235,13 @@ fn adjust_sparkline_group_after_row_insert( group : SparklineGroup, row : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> SparklineGroup raise XlsxError { let updated : Array[Sparkline] = [] - for sparkline in group.sparklines { + for index, sparkline in group.sparklines { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let range_ref = adjust_range_after_row_insert( sparkline.range_ref, row, @@ -5030,9 +5262,13 @@ fn adjust_sparkline_group_after_row_remove( group : SparklineGroup, row : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> SparklineGroup raise XlsxError { let updated : Array[Sparkline] = [] - for sparkline in group.sparklines { + for index, sparkline in group.sparklines { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let range_ref = adjust_range_after_row_remove( sparkline.range_ref, row, @@ -5056,9 +5292,13 @@ fn adjust_sparkline_group_after_col_insert( group : SparklineGroup, col : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> SparklineGroup raise XlsxError { let updated : Array[Sparkline] = [] - for sparkline in group.sparklines { + for index, sparkline in group.sparklines { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let range_ref = adjust_range_after_col_insert( sparkline.range_ref, col, @@ -5079,9 +5319,13 @@ fn adjust_sparkline_group_after_col_remove( group : SparklineGroup, col : Int, count : Int, + cancelled? : () -> Bool = () => false, ) -> SparklineGroup raise XlsxError { let updated : Array[Sparkline] = [] - for sparkline in group.sparklines { + for index, sparkline in group.sparklines { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } let range_ref = adjust_range_after_col_remove( sparkline.range_ref, col, From 738c98a281b37a28c765dc266c2aa555f5fa5eaf Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:08:40 +0800 Subject: [PATCH 118/150] fix(xlsx): authenticate relationship and image parts --- xlsx/cell_images.mbt | 31 +++--- xlsx/cell_images_test.mbt | 19 ++++ xlsx/header_footer_image_read.mbt | 38 +++---- xlsx/image_types.mbt | 146 +++++++++++++++++++++++++++ xlsx/io_password_test.mbt | 3 +- xlsx/read.mbt | 154 ++++++++++++++++++++-------- xlsx/read_drawing_xml.mbt | 37 ++++--- xlsx/read_package_parts.mbt | 162 ++++++++++++++++++++++++++---- xlsx/rich_value_images.mbt | 46 +++++---- xlsx/rich_value_images_test.mbt | 22 ++++ xlsx/write_ooxml_constants.mbt | 3 + 11 files changed, 514 insertions(+), 147 deletions(-) diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index fb23d3df..4fd7d983 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -22,6 +22,7 @@ fn parse_cell_images( rels_xml : StringView, source_part : StringView, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, archive : @zip.Archive, budget? : ReadBudget, cancelled? : () -> Bool = () => false, @@ -112,17 +113,19 @@ fn parse_cell_images( cancelled~, ) match archive.get(media_path) { - Some(data) => { - let extension = cell_image_extension(media_path) - let content_type = match image_content_type(extension[1:]) { - Some(value) => value - None => "" - } + Some(_) => { + let (data, identity) = load_image_part( + archive, + content_types, + media_path, + "cell image", + cancelled~, + ) images.push({ name: unescape_xml_text(name), data: data.to_owned(), - extension, - content_type, + extension: "." + identity.extension, + content_type: identity.content_type, alt_text: unescape_xml_text(alt_text), }) } @@ -132,16 +135,6 @@ fn parse_cell_images( images } -///| -/// Returns the file extension (with leading dot) of a media path, or the -/// empty string when there is none. -fn cell_image_extension(path : String) -> String { - match path.rev_find(".") { - Some(pos) => path[pos:].to_owned() - None => "" - } -} - ///| /// Builds an `Image` positioned in `reference` from an embedded cell /// image. Cell images have no anchor geometry, so offsets/size are zero @@ -241,6 +234,7 @@ test "parse_cell_images handles unprefixed blip without matching blipFill" { rels_xml, "xl/cellimages.xml", archive_part_name_index(archive), + image_reader_test_content_types(), archive, ) inspect(images.length(), content="1") @@ -262,6 +256,7 @@ test "parse_cell_images resolves targets relative to its source part" { rels_xml, "xl/cellimages.xml", archive_part_name_index(archive), + image_reader_test_content_types(), archive, ) assert_eq(images.length(), 1) diff --git a/xlsx/cell_images_test.mbt b/xlsx/cell_images_test.mbt index 22c3aa57..883a7369 100644 --- a/xlsx/cell_images_test.mbt +++ b/xlsx/cell_images_test.mbt @@ -1,8 +1,27 @@ +///| +fn ensure_cell_image_png_content_type(archive : @zip.Archive) -> Unit raise { + let path = "[Content_Types].xml" + let bytes = match archive.get(path) { + Some(value) => value + None => fail("written workbook is missing [Content_Types].xml") + } + let xml = @encoding/utf8.decode(bytes) + if xml.contains("Extension=\"png\"") { + return + } + let patched = xml.replace( + old="", + new="", + ) + assert_true(archive.replace(path, @encoding/utf8.encode(patched))) +} + ///| /// Injects synthetic WPS embedded-cell-image parts into a written /// workbook: cellimages.xml, its relationships, and one media file. fn inject_cell_images(base : Bytes, media : Bytes) -> Bytes raise { let archive = @zip.read(base) + ensure_cell_image_png_content_type(archive) let cellimages_xml = #| #| diff --git a/xlsx/header_footer_image_read.mbt b/xlsx/header_footer_image_read.mbt index 596680bc..956e66a8 100644 --- a/xlsx/header_footer_image_read.mbt +++ b/xlsx/header_footer_image_read.mbt @@ -82,6 +82,7 @@ fn parse_header_footer_images_from_vml( vml_rels_xml : String?, vml_part : StringView, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, archive : @zip.Archive, budget? : ReadBudget, cancelled? : () -> Bool = () => false, @@ -142,23 +143,18 @@ fn parse_header_footer_images_from_vml( part_names, cancelled~, ) - let image_bytes = match archive.get(image_path) { - Some(value) => value - None => raise MissingPart(path=image_path) - } - let extension = match extension_from_path(image_path) { - Some(value) => value - None => continue - } - let content_type = match image_content_type(extension) { - Some(value) => value - None => continue - } + let (image_bytes, identity) = load_image_part( + archive, + content_types, + image_path, + "header/footer image", + cancelled~, + ) let image : HeaderFooterImage = { position, data: image_bytes.to_owned(), - extension, - content_type, + extension: identity.extension, + content_type: identity.content_type, is_footer, first_page, width, @@ -212,19 +208,13 @@ test "header/footer image read wb: parse images with r:id fallback and duplicate let archive = @zip.Archive::new() let image1 = @encoding/utf8.encode("img1") let image2 = @encoding/utf8.encode("img2") - let image3 = @encoding/utf8.encode("img3") - let image4 = @encoding/utf8.encode("img4") archive.add("xl/media/image1.png", image1) archive.add("xl/media/image2.png", image2) - archive.add("xl/media/image3.bin", image3) - archive.add("xl/media/image4", image4) let rels_xml = #| #| #| - #| - #| #| let vml_xml = #| @@ -237,12 +227,6 @@ test "header/footer image read wb: parse images with r:id fallback and duplicate #| #| #| - #| - #| - #| - #| - #| - #| #| let images = parse_header_footer_images_from_vml( @@ -250,6 +234,7 @@ test "header/footer image read wb: parse images with r:id fallback and duplicate Some(rels_xml), "xl/drawings/vmlDrawing1.vml", Map([]), + image_reader_test_content_types(), archive, ) inspect(images.length(), content="1") @@ -280,6 +265,7 @@ test "header/footer image read wb: missing target part raises MissingPart" { Some(rels_xml), "xl/drawings/vmlDrawing1.vml", Map([]), + image_reader_test_content_types(), archive, ), ) catch { diff --git a/xlsx/image_types.mbt b/xlsx/image_types.mbt index dc86ebb4..4b564b99 100644 --- a/xlsx/image_types.mbt +++ b/xlsx/image_types.mbt @@ -37,6 +37,152 @@ fn image_content_type(extension : StringView) -> String? { } } +///| +priv struct ImagePartIdentity { + extension : String + content_type : String +} + +///| +fn canonical_image_extension_for_content_type( + content_type : StringView, +) -> String? { + match content_type.to_owned().to_lower() { + "image/bmp" => Some("bmp") + "image/emf" => Some("emf") + "image/emz" => Some("emz") + "image/gif" => Some("gif") + "image/x-icon" => Some("ico") + "image/jpeg" => Some("jpeg") + "image/png" => Some("png") + "image/svg+xml" => Some("svg") + "image/tiff" => Some("tiff") + "image/wmf" => Some("wmf") + "image/wmz" => Some("wmz") + _ => None + } +} + +///| +/// Resolves an embedded image's identity from `[Content_Types].xml`, never +/// from a filename guess. A compatible filename extension is preserved (for +/// example `jpg`/`tif`); an override on an unconventional name receives the +/// canonical extension for its declared supported media type. +fn image_part_identity( + content_types : @ooxml.PackageContentTypes, + part_name : StringView, + role : StringView, + cancelled? : () -> Bool = () => false, +) -> ImagePartIdentity raise XlsxError { + let declared = declared_part_content_type( + content_types, + logical_archive_part_path(part_name), + role, + cancelled~, + ) + let content_type = declared.to_lower() + let canonical = match + canonical_image_extension_for_content_type(content_type) { + Some(value) => value + None => raise InvalidXml(msg="\{role.to_owned()} content type is invalid") + } + let extension = match extension_from_path(part_name) { + Some(candidate) => + match image_content_type(candidate) { + Some(value) if value == content_type => candidate + _ => canonical + } + None => canonical + } + { extension, content_type } +} + +///| +fn load_image_part( + archive : @zip.Archive, + content_types : @ooxml.PackageContentTypes, + path : StringView, + role : StringView, + cancelled? : () -> Bool = () => false, +) -> (BytesView, ImagePartIdentity) raise XlsxError { + let bytes = match archive.get(path) { + Some(value) => value + None => raise MissingPart(path=path.to_owned()) + } + let identity = image_part_identity(content_types, path, role, cancelled~) + (bytes, identity) +} + +///| +/// Shared manifest for package-private image reader tests. Production readers +/// always receive the manifest parsed from the archive. +fn image_reader_test_content_types() -> @ooxml.PackageContentTypes { + try! @ooxml.parse_package_content_types( + ( + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + ), + ) +} + +///| +test "image part identity comes from supported manifest declarations" { + let missing = try! @ooxml.parse_package_content_types( + "", + ) + let conflicting = try! @ooxml.parse_package_content_types( + "", + ) + for + entry in [ + (missing, "drawing image content type is missing"), + (conflicting, "drawing image content type is invalid"), + ] { + let (content_types, expected) = entry + try + image_part_identity(content_types, "xl/media/image1.png", "drawing image") + catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected image content-type error") + } noraise { + _ => fail("expected image content-type rejection") + } + } + + let overridden_types = try! @ooxml.parse_package_content_types( + "", + ) + let identity = image_part_identity( + overridden_types, "xl/media/opaque.bin", "drawing image", + ) + assert_eq(identity.extension, "png") + assert_eq(identity.content_type, "image/png") + + let supported_conflict = try! @ooxml.parse_package_content_types( + "", + ) + let jpeg = image_part_identity( + supported_conflict, "xl/media/image1.png", "drawing image", + ) + assert_eq(jpeg.extension, "jpeg") + assert_eq(jpeg.content_type, "image/jpeg") +} + ///| test "image types wb: normalize image extension rejects dot-only and empty" { let dot_only : Result[String, Error] = Ok(normalize_image_extension(".")) catch { diff --git a/xlsx/io_password_test.mbt b/xlsx/io_password_test.mbt index 6950fcad..dc61b9bc 100644 --- a/xlsx/io_password_test.mbt +++ b/xlsx/io_password_test.mbt @@ -100,7 +100,8 @@ test "standard encryption distinguishes wrong passwords from malformed XLSX" { error => Err(error) } match malformed { - Err(@xlsx.MissingPart(path~)) => inspect(path, content="_rels/.rels") + Err(@xlsx.MissingPart(path~)) => + inspect(path, content="[Content_Types].xml") _ => fail("expected a parsed-package failure, not InvalidPassword") } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index cba5bfae..4e1dfa65 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3578,10 +3578,15 @@ fn read_zip_archive_core_unchecked( let workbook_rels_path = actual_relationship_part_path( workbook_xml_part_path, part_names, ) - let workbook_rels = match archive.get(workbook_rels_path) { - Some(value) => decode(value) - None => raise MissingPart(path=workbook_rels_path) - } + let workbook_rels = decode( + load_relationship_part( + archive, + content_types, + workbook_rels_path, + "workbook relationships", + cancelled~, + ), + ) let sheets = parse_workbook_sheets( workbook_source_xml, limits.max_workbook_sheets, @@ -3791,8 +3796,17 @@ fn read_zip_archive_core_unchecked( ) let cell_images = match archive.get(cell_images_part) { Some(value) => { + let cell_images_rels_path = actual_relationship_part_path( + cell_images_part, part_names, + ) let rels_xml = match - archive.get(actual_relationship_part_path(cell_images_part, part_names)) { + load_optional_relationship_part( + archive, + content_types, + cell_images_rels_path, + "cell image relationships", + cancelled~, + ) { Some(rels) => decode(rels) None => "" } @@ -3801,6 +3815,7 @@ fn read_zip_archive_core_unchecked( rels_xml, cell_images_part, part_names, + content_types, archive, budget=read_budget, cancelled~, @@ -3811,6 +3826,7 @@ fn read_zip_archive_core_unchecked( let rich_value_images = parse_rich_value_images( archive, part_names, + content_types, decode, budget=read_budget, cancelled~, @@ -3820,13 +3836,33 @@ fn read_zip_archive_core_unchecked( Some(data) => { for _, target in data.rel_targets { match archive.get(target) { - Some(bytes) => rich_value_media[target] = bytes.to_owned() + Some(_) => { + let (bytes, identity) = load_image_part( + archive, + content_types, + target, + "rich value image", + cancelled~, + ) + rich_value_media[target] = bytes.to_owned() + data.media_identities[target] = identity + } None => () } } for _, target in data.web_rel_targets { match archive.get(target) { - Some(bytes) => rich_value_media[target] = bytes.to_owned() + Some(_) => { + let (bytes, identity) = load_image_part( + archive, + content_types, + target, + "rich value web image", + cancelled~, + ) + rich_value_media[target] = bytes.to_owned() + data.media_identities[target] = identity + } None => () } } @@ -4102,7 +4138,14 @@ fn read_zip_archive_core_unchecked( legacy_drawing_hf_rel_id is Some(_) || picture_rel_id is Some(_) let rels_path = actual_relationship_part_path(sheet_path, part_names) - let rels_xml = match archive.get(rels_path) { + let rels_xml = match + load_optional_relationship_part( + archive, + content_types, + rels_path, + "worksheet relationships", + cancelled~, + ) { Some(rels_bytes) => decode(rels_bytes) None => if needs_rels { raise MissingPart(path=rels_path) } else { "" } } @@ -4172,27 +4215,18 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) - let image_bytes = match archive.get(image_path) { - Some(value) => value - None => raise MissingPart(path=image_path) - } - let path_text = target.to_string() - let extension = match path_text.rev_find(".") { - Some(pos) => - if pos + 1 < path_text.length() { - let view = path_text[pos + 1:] - view.to_owned().to_lower() - } else { - raise InvalidSheetBackground(msg="image extension missing") - } - None => raise InvalidSheetBackground(msg="image extension missing") - } - let content_type = match image_content_type(extension) { - Some(value) => value - None => - raise InvalidSheetBackground(msg="unsupported image extension") - } - Some({ data: image_bytes.to_owned(), extension, content_type }) + let (image_bytes, identity) = load_image_part( + archive, + content_types, + image_path, + "sheet background image", + cancelled~, + ) + Some({ + data: image_bytes.to_owned(), + extension: identity.extension, + content_type: identity.content_type, + }) } None => None } @@ -4331,10 +4365,13 @@ fn read_zip_archive_core_unchecked( let pivot_rels_path = actual_relationship_part_path( pivot_path, part_names, ) - let pivot_rels_bytes = match archive.get(pivot_rels_path) { - Some(value) => value - None => raise MissingPart(path=pivot_rels_path) - } + let pivot_rels_bytes = load_relationship_part( + archive, + content_types, + pivot_rels_path, + "pivot table relationships", + cancelled~, + ) let pivot_rels_xml = decode(pivot_rels_bytes) let cache_targets = parse_internal_relationship_targets( pivot_rels_xml, @@ -4383,8 +4420,17 @@ fn read_zip_archive_core_unchecked( let pivot_id = parse_id_from_path( pivot_path, "xl/pivotTables/pivotTable", ) + let cache_rels_path = actual_relationship_part_path( + cache_path, part_names, + ) let cache_records_xml = match - archive.get(actual_relationship_part_path(cache_path, part_names)) { + load_optional_relationship_part( + archive, + content_types, + cache_rels_path, + "pivot cache relationships", + cancelled~, + ) { Some(value) => { let cache_rels_xml = decode(value) let record_targets = parse_internal_relationship_targets( @@ -4520,7 +4566,13 @@ fn read_zip_archive_core_unchecked( (vml_drawing_hf_xml, vml_drawing_hf_path) { (Some(xml), Some(path)) => { let vml_rels_xml = match - archive.get(actual_relationship_part_path(path, part_names)) { + load_optional_relationship_part( + archive, + content_types, + actual_relationship_part_path(path, part_names), + "VML relationships", + cancelled~, + ) { Some(value) => Some(decode(value)) None => None } @@ -4529,6 +4581,7 @@ fn read_zip_archive_core_unchecked( vml_rels_xml, path, part_names, + content_types, archive, budget=read_budget, cancelled~, @@ -4678,7 +4731,13 @@ fn read_zip_archive_core_unchecked( ) } let drawing_rels_xml = match - archive.get(actual_relationship_part_path(drawing_path, part_names)) { + load_optional_relationship_part( + archive, + content_types, + actual_relationship_part_path(drawing_path, part_names), + "drawing relationships", + cancelled~, + ) { Some(value) => decode(value) None => "" } @@ -4687,6 +4746,7 @@ fn read_zip_archive_core_unchecked( drawing_rels_xml, drawing_path, part_names, + content_types, drawing_metrics, archive, budget=read_budget, @@ -4821,10 +4881,13 @@ fn read_zip_archive_core_unchecked( let chartsheet_rels_path = actual_relationship_part_path( chartsheet_path, part_names, ) - let chartsheet_rels_bytes = match archive.get(chartsheet_rels_path) { - Some(value) => value - None => raise MissingPart(path=chartsheet_rels_path) - } + let chartsheet_rels_bytes = load_relationship_part( + archive, + content_types, + chartsheet_rels_path, + "chartsheet relationships", + cancelled~, + ) let chartsheet_rels_xml = decode(chartsheet_rels_bytes) let drawing_targets = parse_internal_relationship_targets( chartsheet_rels_xml, @@ -4852,10 +4915,13 @@ fn read_zip_archive_core_unchecked( let drawing_rels_path = actual_relationship_part_path( drawing_path, part_names, ) - let drawing_rels_bytes = match archive.get(drawing_rels_path) { - Some(value) => value - None => raise MissingPart(path=drawing_rels_path) - } + let drawing_rels_bytes = load_relationship_part( + archive, + content_types, + drawing_rels_path, + "drawing relationships", + cancelled~, + ) let drawing_rels_xml = decode(drawing_rels_bytes) let chart_targets = parse_internal_relationship_targets( drawing_rels_xml, diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index cbbac380..5762a98b 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -205,6 +205,7 @@ fn parse_drawing_images( drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, metrics : DrawingMetrics, archive : @zip.Archive, budget? : ReadBudget, @@ -360,24 +361,13 @@ fn parse_drawing_images( part_names, cancelled~, ) - let data = match archive.get(media_path) { - Some(value) => value - None => raise MissingPart(path=media_path) - } - let ext = match media_path.to_string().rev_find(".") { - Some(pos) => - if pos + 1 < media_path.length() { - media_path[pos + 1:] - } else { - "" - } - None => "" - } - let extension = ext.to_owned().to_lower() - let content_type = match image_content_type(extension) { - Some(value) => value - None => "application/octet-stream" - } + let (data, identity) = load_image_part( + archive, + content_types, + media_path, + "drawing image", + cancelled~, + ) let mut hyperlink = "" let mut hyperlink_type = HyperlinkType::Unset match tag_attributes_in(pic_body, "a:hlinkClick") { @@ -398,8 +388,8 @@ fn parse_drawing_images( images.push({ reference, data: data.to_owned(), - extension, - content_type, + extension: identity.extension, + content_type: identity.content_type, offset_x, offset_y, scale_x: 1.0, @@ -1192,6 +1182,7 @@ test "read_drawing_xml wb: parse_drawing_images empty rel map branch" { "", "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), @@ -1220,6 +1211,7 @@ test "read_drawing_xml wb: signed marker offsets are retained" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ) @@ -1246,6 +1238,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), @@ -1265,6 +1258,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), @@ -1287,6 +1281,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), @@ -1306,6 +1301,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), @@ -1328,6 +1324,7 @@ test "read_drawing_xml wb: parse_drawing_images anchor/ext validation errors" { rels_xml, "xl/drawings/drawing1.xml", Map([]), + image_reader_test_content_types(), metrics, archive, ), diff --git a/xlsx/read_package_parts.mbt b/xlsx/read_package_parts.mbt index 30431109..0893ab5b 100644 --- a/xlsx/read_package_parts.mbt +++ b/xlsx/read_package_parts.mbt @@ -10,15 +10,21 @@ fn workbook_content_type_candidates() -> Array[String] { fn workbook_part_from_root_relationships( archive : @zip.Archive, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, decode : (BytesView) -> String raise XlsxError, budget? : ReadBudget, cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { let root_rels_path = actual_archive_part_path(part_names, root_rels_part_path) - let root_rels_xml = match archive.get(root_rels_path) { - Some(bytes) => decode(bytes) - None => raise MissingPart(path=root_rels_path) - } + let root_rels_xml = decode( + load_relationship_part( + archive, + content_types, + root_rels_path, + "root relationships", + cancelled~, + ), + ) let office_document_type = transitional_office_relationship_prefix + "officeDocument" let targets = parse_internal_relationship_targets( @@ -59,20 +65,36 @@ fn require_part_content_type( role : StringView, cancelled? : () -> Bool = () => false, ) -> Unit raise XlsxError { + let declared = declared_part_content_type( + content_types, + part_name, + role, + cancelled~, + ) + let normalized = declared.to_lower() + for candidate in expected { + if normalized == candidate.to_lower() { + return + } + } + raise InvalidXml(msg="\{role.to_owned()} content type is invalid") +} + +///| +/// Returns the manifest identity for a package part. This is the single place +/// where missing content declarations become XLSX package errors. +fn declared_part_content_type( + content_types : @ooxml.PackageContentTypes, + part_name : StringView, + role : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { let declared = content_types.content_type_for(part_name, cancelled~) catch { InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } match declared { - Some(value) => { - let normalized = value.to_lower() - for candidate in expected { - if normalized == candidate.to_lower() { - return - } - } - raise InvalidXml(msg="\{role.to_owned()} content type is invalid") - } + Some(value) => value None => raise InvalidXml(msg="\{role.to_owned()} content type is missing") } } @@ -103,6 +125,83 @@ fn load_typed_part( bytes } +///| +/// Loads and authenticates an OPC relationship part. Relationship XML has one +/// fixed media type regardless of the source part that owns it. +fn load_relationship_part( + archive : @zip.Archive, + content_types : @ooxml.PackageContentTypes, + path : StringView, + role : StringView, + cancelled? : () -> Bool = () => false, +) -> BytesView raise XlsxError { + load_typed_part( + archive, + content_types, + path, + [ct_relationships], + role, + cancelled~, + ) +} + +///| +/// Optional relationship parts remain optional, but an existing part is never +/// consumed unless its manifest identity is the fixed OPC relationships type. +fn load_optional_relationship_part( + archive : @zip.Archive, + content_types : @ooxml.PackageContentTypes, + path : StringView, + role : StringView, + cancelled? : () -> Bool = () => false, +) -> BytesView? raise XlsxError { + match archive.get(path) { + Some(_) => + Some( + load_relationship_part(archive, content_types, path, role, cancelled~), + ) + None => None + } +} + +///| +test "read package parts: relationship parts require their fixed manifest identity" { + let archive = @zip.Archive::new() + archive.add("custom/_rels/source.xml.rels", b"") + let missing = try! @ooxml.parse_package_content_types( + "", + ) + let wrong = try! @ooxml.parse_package_content_types( + "", + ) + let valid = try! @ooxml.parse_package_content_types( + "", + ) + for + entry in [ + (missing, "test relationships content type is missing"), + (wrong, "test relationships content type is invalid"), + ] { + let (content_types, expected) = entry + try + load_relationship_part( + archive, content_types, "custom/_rels/source.xml.rels", "test relationships", + ) + catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected relationship content-type error") + } noraise { + _ => fail("expected relationship content-type rejection") + } + } + assert_eq( + load_relationship_part( + archive, valid, "custom/_rels/source.xml.rels", "test relationships", + ), + b"", + ) +} + ///| test "read package parts: workbook media types are case-insensitive" { assert_true( @@ -125,13 +224,8 @@ fn resolve_workbook_xml_part_path( budget? : ReadBudget, cancelled? : () -> Bool = () => false, ) -> (String, @ooxml.PackageContentTypes) raise XlsxError { - let workbook_part = workbook_part_from_root_relationships( - archive, - part_names, - decode, - budget?, - cancelled~, - ) + // The content-type manifest authenticates every consumed package part, + // including the root relationship part, so it must be parsed first. let content_types_path = actual_archive_part_path( part_names, content_types_part_path, ) @@ -153,6 +247,14 @@ fn resolve_workbook_xml_part_path( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } + let workbook_part = workbook_part_from_root_relationships( + archive, + part_names, + content_types, + decode, + budget?, + cancelled~, + ) require_part_content_type( content_types, workbook_part, @@ -185,6 +287,12 @@ fn resolve_workbook_xml_part_path_for_test( ///| test "read package parts: root relationship is required" { let archive = @zip.Archive::new() + archive.add( + content_types_part_path, + @encoding/utf8.encode( + "", + ), + ) try resolve_workbook_xml_part_path_for_test(archive) catch { MissingPart(path~) => inspect(path, content="_rels/.rels") _ => fail("unexpected missing-root error") @@ -199,6 +307,7 @@ test "read package parts: workbook path follows the root relationship" { let content_types = #| #| + #| #| #| #| @@ -224,9 +333,16 @@ fn workbook_part_from_root_target_for_test( target.to_owned() + "\"/>" archive.add(root_rels_part_path, @encoding/utf8.encode(root_relationships)) + let content_types = @ooxml.parse_package_content_types( + "", + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } workbook_part_from_root_relationships( archive, archive_part_name_index(archive), + content_types, decode_utf8_or_invalid_xml, ) } @@ -258,6 +374,7 @@ test "read package parts: external workbook relationship is never resolved as a let archive = @zip.Archive::new() let content_types = #| + #| #| #| let root_relationships = @@ -281,6 +398,7 @@ test "read package parts: root target must have a workbook content type" { let content_types = #| #| + #| #| #| #| @@ -343,7 +461,7 @@ test "read package parts: invalid root workbook target raises InvalidXml" { } match result { Err(InvalidXml(msg~)) => - inspect(msg, content="content type part name empty") + inspect(msg, content="content type override part name is invalid") _ => fail("expected content type part name empty") } } @@ -353,6 +471,7 @@ test "read package parts: stale workbook override cannot redirect the root relat let archive = @zip.Archive::new() let content_types = #| + #| #| #| #| @@ -373,6 +492,7 @@ test "read package parts: default-declared workbook follows the root relationshi let archive = @zip.Archive::new() let content_types = #| + #| #| #| let root_relationships = diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index 11262b1e..1df61d4b 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -23,6 +23,8 @@ struct RichValueImages { /// blip relationship id -> media target (from /// rdRichValueWebImage.xml.rels) web_rel_targets : Map[String, String] + /// media target -> manifest-authenticated image identity + media_identities : Map[String, ImagePartIdentity] } ///| @@ -31,6 +33,7 @@ struct RichValueImages { fn parse_rich_value_images( archive : @zip.Archive, part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, decode : (BytesView) -> String raise XlsxError, budget? : ReadBudget, cancelled? : () -> Bool = () => false, @@ -50,8 +53,17 @@ fn parse_rich_value_images( Some(value) => parse_rich_value_rel_ids(decode(value)) None => [] } + let rich_value_rels_path = actual_archive_part_path( + part_names, "xl/richData/_rels/richValueRel.xml.rels", + ) let raw_rel_targets = match - archive.get("xl/richData/_rels/richValueRel.xml.rels") { + load_optional_relationship_part( + archive, + content_types, + rich_value_rels_path, + "rich value relationships", + cancelled~, + ) { Some(value) => parse_internal_relationship_targets( decode(value), @@ -74,8 +86,17 @@ fn parse_rich_value_images( Some(value) => parse_web_image_blip_ids(decode(value)) None => [] } + let rich_value_web_rels_path = actual_archive_part_path( + part_names, "xl/richData/_rels/rdRichValueWebImage.xml.rels", + ) let raw_web_rel_targets = match - archive.get("xl/richData/_rels/rdRichValueWebImage.xml.rels") { + load_optional_relationship_part( + archive, + content_types, + rich_value_web_rels_path, + "rich value web image relationships", + cancelled~, + ) { Some(value) => parse_internal_relationship_targets( decode(value), @@ -102,6 +123,7 @@ fn parse_rich_value_images( rel_targets, web_blip_ids, web_rel_targets, + media_identities: Map([]), }) } @@ -123,6 +145,7 @@ test "rich value image relationships use source-relative OPC resolution" { let parsed = parse_rich_value_images( archive, archive_part_name_index(archive), + image_reader_test_content_types(), value => { @encoding/utf8.decode(value) catch { _ => raise InvalidXml(msg="invalid utf8") @@ -483,16 +506,15 @@ fn Workbook::rich_value_image_for_cell( } match self.rich_value_media.get(media_path) { Some(bytes) => { - let extension = cell_image_extension(media_path) - let content_type = match image_content_type(extension_no_dot(extension)) { + let identity = match data.media_identities.get(media_path) { Some(value) => value - None => "" + None => return None } Some({ reference: canonical, data: bytes, - extension, - content_type, + extension: "." + identity.extension, + content_type: identity.content_type, offset_x: 0, offset_y: 0, scale_x: 1.0, @@ -513,16 +535,6 @@ fn Workbook::rich_value_image_for_cell( } } -///| -/// Returns a file extension without its leading dot. -fn extension_no_dot(extension : String) -> String { - if extension.has_prefix(".") { - extension[1:].to_owned() - } else { - extension - } -} - ///| test "parse_rich_values does not create a phantom entry from rvData" { let xml = diff --git a/xlsx/rich_value_images_test.mbt b/xlsx/rich_value_images_test.mbt index 72179784..91d89ffa 100644 --- a/xlsx/rich_value_images_test.mbt +++ b/xlsx/rich_value_images_test.mbt @@ -1,9 +1,28 @@ +///| +fn ensure_rich_value_png_content_type(archive : @zip.Archive) -> Unit raise { + let path = "[Content_Types].xml" + let bytes = match archive.get(path) { + Some(value) => value + None => fail("written workbook is missing [Content_Types].xml") + } + let xml = @encoding/utf8.decode(bytes) + if xml.contains("Extension=\"png\"") { + return + } + let patched = xml.replace( + old="", + new="", + ) + assert_true(archive.replace(path, @encoding/utf8.encode(patched))) +} + ///| /// Rewrites a written workbook into a rich-value "Place in cell" image /// layout: injects a `vm` attribute onto the A1 cell and adds the /// metadata, richData, and media parts a modern embedded image needs. fn inject_rich_value_image(base : Bytes, media : Bytes) -> Bytes raise { let archive = @zip.read(base) + ensure_rich_value_png_content_type(archive) let patched = @zip.Archive::new() for entry in archive.entries() { if entry.name() == "xl/worksheets/sheet1.xml" { @@ -117,6 +136,7 @@ test "rich-value image ignored when cell is not a #VALUE! error" { /// lengths, which Excelize refuses to resolve. fn inject_mismatched_rich_value(base : Bytes) -> Bytes raise { let archive = @zip.read(base) + ensure_rich_value_png_content_type(archive) let patched = @zip.Archive::new() for entry in archive.entries() { if entry.name() == "xl/worksheets/sheet1.xml" { @@ -191,6 +211,7 @@ test "rich-value image suppressed when keys and values length mismatch" { /// (WebImageIdentifier) instead of a local place-in-cell image. fn inject_web_image(base : Bytes, media : Bytes) -> Bytes raise { let archive = @zip.read(base) + ensure_rich_value_png_content_type(archive) let patched = @zip.Archive::new() for entry in archive.entries() { if entry.name() == "xl/worksheets/sheet1.xml" { @@ -273,6 +294,7 @@ test "get_pictures resolves an IMAGE() web image" { /// is present, so no image resolves. fn inject_local_key_wins(base : Bytes, media : Bytes) -> Bytes raise { let archive = @zip.read(base) + ensure_rich_value_png_content_type(archive) let patched = @zip.Archive::new() for entry in archive.entries() { if entry.name() == "xl/worksheets/sheet1.xml" { diff --git a/xlsx/write_ooxml_constants.mbt b/xlsx/write_ooxml_constants.mbt index b0457e8f..2fe1ac5c 100644 --- a/xlsx/write_ooxml_constants.mbt +++ b/xlsx/write_ooxml_constants.mbt @@ -52,6 +52,9 @@ let rel_hyperlink = "http://schemas.openxmlformats.org/officeDocument/2006/relat ///| let rel_comments = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments" +///| +let ct_relationships = "application/vnd.openxmlformats-package.relationships+xml" + ///| let ct_drawing = "application/vnd.openxmlformats-officedocument.drawing+xml" From ec122920bc85309ee8cbdcd035f775a5daa2830c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:18:14 +0800 Subject: [PATCH 119/150] fix(ooxml): scope MCE processing rules correctly --- ooxml/mce_process_scope.mbt | 95 ++++++++++++++++++++++++++++++ ooxml/mce_projection.mbt | 108 +++++++++++++++++++++------------- ooxml/mce_projection_test.mbt | 33 ++++++++++- ooxml/read_parse.mbt | 63 +++++++++----------- ooxml/read_parse_test.mbt | 12 ++++ 5 files changed, 234 insertions(+), 77 deletions(-) create mode 100644 ooxml/mce_process_scope.mbt diff --git a/ooxml/mce_process_scope.mbt b/ooxml/mce_process_scope.mbt new file mode 100644 index 00000000..c997507e --- /dev/null +++ b/ooxml/mce_process_scope.mbt @@ -0,0 +1,95 @@ +///| +/// `mc:Ignorable` resets inherited processing rules for each namespace it +/// names. Generations make that reset O(1): older `ProcessContent` entries stay +/// reference-counted until their XML scope ends, but only the current +/// generation participates in element selection. +priv struct MceProcessKey { + namespace_id : Int + generation : Int + local_name : String +} derive(Eq, Hash) + +///| +priv struct MceProcessGenerationReset { + namespace_id : Int + previous_generation : Int? +} + +///| +priv struct MceProcessTracker { + key_counts : Map[MceProcessKey, Int] + generations : Map[Int, Int] + mut next_generation : Int +} + +///| +fn MceProcessTracker::new() -> MceProcessTracker { + { key_counts: Map([]), generations: Map([]), next_generation: 1 } +} + +///| +fn MceProcessTracker::reset( + self : MceProcessTracker, + namespace_id : Int, +) -> MceProcessGenerationReset { + let reset : MceProcessGenerationReset = { + namespace_id, + previous_generation: self.generations.get(namespace_id), + } + self.generations[namespace_id] = self.next_generation + self.next_generation += 1 + reset +} + +///| +fn MceProcessTracker::add( + self : MceProcessTracker, + namespace_id : Int, + local_name : String, +) -> MceProcessKey { + let key : MceProcessKey = { + namespace_id, + generation: self.generations.get_or_default(namespace_id, 0), + local_name, + } + self.key_counts[key] = self.key_counts.get_or_default(key, 0) + 1 + key +} + +///| +fn MceProcessTracker::processes( + self : MceProcessTracker, + namespace_id : Int, + local_name : String, +) -> Bool { + let generation = self.generations.get_or_default(namespace_id, 0) + self.key_counts.get_or_default({ namespace_id, generation, local_name }, 0) > + 0 || + self.key_counts.get_or_default( + { namespace_id, generation, local_name: "*" }, + 0, + ) > + 0 +} + +///| +fn MceProcessTracker::leave( + self : MceProcessTracker, + keys : ArrayView[MceProcessKey], + resets : ArrayView[MceProcessGenerationReset], +) -> Unit { + for key in keys { + let remaining = self.key_counts.get_or_default(key, 0) - 1 + if remaining <= 0 { + ignore(self.key_counts.remove(key)) + } else { + self.key_counts[key] = remaining + } + } + for reset in resets { + match reset.previous_generation { + Some(generation) => self.generations[reset.namespace_id] = generation + None => ignore(self.generations.remove(reset.namespace_id)) + } + } +} diff --git a/ooxml/mce_projection.mbt b/ooxml/mce_projection.mbt index a1a7c7b1..dbc51d63 100644 --- a/ooxml/mce_projection.mbt +++ b/ooxml/mce_projection.mbt @@ -8,7 +8,7 @@ let max_xml_mce_token_chars = 1024 priv struct XmlMceName { namespace_id : Int local_name : String -} derive(Eq, Hash) +} ///| priv struct XmlMceExtensionName { @@ -19,7 +19,8 @@ priv struct XmlMceExtensionName { ///| priv struct XmlMceScopeDelta { ignorable_namespace_ids : Array[Int] - process_content_names : Array[XmlMceName] + process_content_keys : Array[MceProcessKey] + process_generation_resets : Array[MceProcessGenerationReset] } ///| @@ -33,7 +34,7 @@ priv struct XmlMceContext { understood_namespaces : Set[String] extension_elements : Set[XmlMceExtensionName] ignorable_namespace_counts : Map[Int, Int] - process_content_name_counts : Map[XmlMceName, Int] + process_content : MceProcessTracker max_work_chars : Int mut work_chars : Int } @@ -57,7 +58,7 @@ fn XmlMceContext::new( understood_namespaces: understood, extension_elements: extensions, ignorable_namespace_counts: Map([]), - process_content_name_counts: Map([]), + process_content: MceProcessTracker::new(), max_work_chars, work_chars: 0, } @@ -102,17 +103,6 @@ fn XmlMceContext::add_ignorable( 1 } -///| -fn XmlMceContext::add_process_content( - self : XmlMceContext, - name : XmlMceName, -) -> Unit { - self.process_content_name_counts[name] = self.process_content_name_counts.get_or_default( - name, 0, - ) + - 1 -} - ///| fn XmlMceContext::is_ignorable( self : XmlMceContext, @@ -132,12 +122,7 @@ fn XmlMceContext::is_effectively_ignorable( ///| fn XmlMceContext::processes(self : XmlMceContext, name : XmlMceName) -> Bool { - self.process_content_name_counts.get_or_default(name, 0) > 0 || - self.process_content_name_counts.get_or_default( - { namespace_id: name.namespace_id, local_name: "*" }, - 0, - ) > - 0 + self.process_content.processes(name.namespace_id, name.local_name) } ///| @@ -146,6 +131,10 @@ fn XmlMceContext::leave_scope( delta : XmlMceScopeDelta?, ) -> Unit { guard delta is Some(delta) else { return } + self.process_content.leave( + delta.process_content_keys, + delta.process_generation_resets, + ) for namespace_id in delta.ignorable_namespace_ids { let remaining = self.ignorable_namespace_counts.get_or_default( namespace_id, 0, @@ -157,14 +146,6 @@ fn XmlMceContext::leave_scope( self.ignorable_namespace_counts[namespace_id] = remaining } } - for name in delta.process_content_names { - let remaining = self.process_content_name_counts.get_or_default(name, 0) - 1 - if remaining <= 0 { - ignore(self.process_content_name_counts.remove(name)) - } else { - self.process_content_name_counts[name] = remaining - } - } } ///| @@ -264,6 +245,7 @@ fn xml_mce_enter_scope( match scanner.attribute(markup_compatibility_namespace, "Ignorable") { Some(value) => { context.charge(value) + let reset_namespaces : Set[Int] = Set([]) for prefix in xml_mce_tokens(value, "Ignorable") { let (namespace_uri, namespace_id) = xml_mce_resolve_prefix( scanner, prefix, "Ignorable", @@ -278,9 +260,18 @@ fn xml_mce_enter_scope( None => delta = Some({ ignorable_namespace_ids: [namespace_id], - process_content_names: [], + process_content_keys: [], + process_generation_resets: [], }) } + if !reset_namespaces.contains(namespace_id) { + reset_namespaces.add(namespace_id) + let reset = context.process_content.reset(namespace_id) + match delta { + Some(value) => value.process_generation_resets.push(reset) + None => () + } + } } } None => () @@ -293,13 +284,17 @@ fn xml_mce_enter_scope( if !context.is_ignorable(name.namespace_id) { raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") } - context.add_process_content(name) + let key = context.process_content.add( + name.namespace_id, + name.local_name, + ) match delta { - Some(value) => value.process_content_names.push(name) + Some(value) => value.process_content_keys.push(key) None => delta = Some({ ignorable_namespace_ids: [], - process_content_names: [name], + process_content_keys: [key], + process_generation_resets: [], }) } } @@ -489,12 +484,27 @@ fn xml_mce_state_for_current_element( } _ => () } - let scope_entry = xml_mce_enter_scope(scanner, context) let declarations = scanner.current_namespace_declarations() + let namespace_uri = scanner.namespace_uri().to_owned() + let namespace_id = scanner.current_namespace_identity() + let local_name = scanner.local_name().to_owned() if depth == 1 { - if scanner.namespace_uri() == markup_compatibility_namespace { + if namespace_uri == markup_compatibility_namespace { raise InvalidXml(msg="MCE document element is invalid") } + if context.is_extension_element(namespace_uri, local_name) { + return { + active: true, + scope_delta: None, + pending_bindings: None, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + opaque_descendants: true, + } + } + let scope_entry = xml_mce_enter_scope(scanner, context) xml_mce_require_must_understand(scope_entry) return { active: true, @@ -508,9 +518,24 @@ fn xml_mce_state_for_current_element( } } let parent_state = parent.unwrap() - let namespace_uri = scanner.namespace_uri().to_owned() - let namespace_id = scanner.current_namespace_identity() - let local_name = scanner.local_name().to_owned() + if !parent_state.is_alternate_content && + namespace_uri != markup_compatibility_namespace && + context.is_extension_element(namespace_uri, local_name) { + scanner.insert_current_attributes( + xml_mce_namespace_attributes(scanner, parent_state.pending_bindings), + ) + return { + active: true, + scope_delta: None, + pending_bindings: None, + is_alternate_content: false, + alternate_choice_seen: false, + alternate_fallback_seen: false, + alternate_selected: false, + opaque_descendants: true, + } + } + let scope_entry = xml_mce_enter_scope(scanner, context) let mut active = parent_state.active let mut pending_bindings : XmlMceBindingScope? = None let mut is_alternate_content = false @@ -622,9 +647,10 @@ fn xml_mce_state_for_current_element( /// subtrees are removed, selected `Choice`/`Fallback` and `ProcessContent` /// wrappers are unwrapped, and namespace declarations from removed wrappers /// are retained on promoted children. Processing is streaming, cancellable, -/// and bounded by `max_output_chars`. Descendants of application-defined -/// `extension_elements` remain opaque: they are namespace-validated as XML but -/// MCE directives inside their payload are preserved rather than interpreted. +/// and bounded by `max_output_chars`. Application-defined `extension_elements` +/// and their descendants remain opaque: they are namespace-validated as XML, +/// but MCE directives on or inside their payload are preserved rather than +/// interpreted. pub fn project_xml_markup_compatibility( xml : StringView, understood_namespaces : ArrayView[String], diff --git a/ooxml/mce_projection_test.mbt b/ooxml/mce_projection_test.mbt index 3ce22d49..d36a5f49 100644 --- a/ooxml/mce_projection_test.mbt +++ b/ooxml/mce_projection_test.mbt @@ -37,13 +37,44 @@ test "MCE projection unwraps ProcessContent and ignores opaque extensions" { ///| test "MCE projection preserves configured extension payloads verbatim" { let source = - #| + #| let projected = @ooxml.project_xml_markup_compatibility(source, ["urn:app"], extension_elements=[ ("urn:app", "ext"), ]) inspect(projected, content=source) } +///| +test "MCE projection promotes wrapper namespaces onto opaque extensions" { + let source = + #| + let projected = @ooxml.project_xml_markup_compatibility(source, ["urn:app"], extension_elements=[ + ("urn:app", "ext"), + ]) + assert_false(projected.contains(" + let projected = @ooxml.project_xml_markup_compatibility(source, ["urn:app"]) + assert_true(projected.contains("a:Before")) + assert_true(projected.contains("a:Inside")) + assert_true(projected.contains("a:After")) + assert_false(projected.contains("a:Blocked")) + assert_false(projected.contains(" RelationshipMceContext { { ignorable_namespace_counts: Map([]), - process_content_name_counts: Map([]), + process_content: MceProcessTracker::new(), work_chars: 0, } } @@ -332,17 +333,6 @@ fn RelationshipMceContext::add_ignorable( 1 } -///| -fn RelationshipMceContext::add_process_content( - self : RelationshipMceContext, - name : RelationshipMceName, -) -> Unit { - self.process_content_name_counts[name] = self.process_content_name_counts.get_or_default( - name, 0, - ) + - 1 -} - ///| fn RelationshipMceContext::is_ignorable( self : RelationshipMceContext, @@ -372,14 +362,7 @@ fn RelationshipMceContext::processes( self : RelationshipMceContext, name : RelationshipMceName, ) -> Bool { - if self.process_content_name_counts.get_or_default(name, 0) > 0 { - return true - } - self.process_content_name_counts.get_or_default( - { namespace_id: name.namespace_id, local_name: "*" }, - 0, - ) > - 0 + self.process_content.processes(name.namespace_id, name.local_name) } ///| @@ -388,6 +371,10 @@ fn RelationshipMceContext::leave_scope( delta : RelationshipMceScopeDelta?, ) -> Unit { guard delta is Some(delta) else { return } + self.process_content.leave( + delta.process_content_keys, + delta.process_generation_resets, + ) for namespace_id in delta.ignorable_namespace_ids { let remaining = self.ignorable_namespace_counts.get_or_default( namespace_id, 0, @@ -399,14 +386,6 @@ fn RelationshipMceContext::leave_scope( self.ignorable_namespace_counts[namespace_id] = remaining } } - for name in delta.process_content_names { - let remaining = self.process_content_name_counts.get_or_default(name, 0) - 1 - if remaining <= 0 { - ignore(self.process_content_name_counts.remove(name)) - } else { - self.process_content_name_counts[name] = remaining - } - } } ///| @@ -498,6 +477,7 @@ fn relationship_mce_enter_scope( match ignorable_attribute { Some(value) => { context.charge(value) + let reset_namespaces : Set[Int] = Set([]) for prefix in relationship_mce_tokens(value, "Ignorable") { let (namespace_uri, namespace_id) = relationship_mce_resolve_prefix( scanner, prefix, "Ignorable", @@ -512,9 +492,18 @@ fn relationship_mce_enter_scope( None => delta = Some({ ignorable_namespace_ids: [namespace_id], - process_content_names: [], + process_content_keys: [], + process_generation_resets: [], }) } + if !reset_namespaces.contains(namespace_id) { + reset_namespaces.add(namespace_id) + let reset = context.process_content.reset(namespace_id) + match delta { + Some(value) => value.process_generation_resets.push(reset) + None => () + } + } } } None => () @@ -532,13 +521,17 @@ fn relationship_mce_enter_scope( if !context.is_ignorable(key.namespace_id) { raise InvalidXml(msg="MCE ProcessContent namespace is not ignorable") } - context.add_process_content(key) + let process_key = context.process_content.add( + key.namespace_id, + key.local_name, + ) match delta { - Some(value) => value.process_content_names.push(key) + Some(value) => value.process_content_keys.push(process_key) None => delta = Some({ ignorable_namespace_ids: [], - process_content_names: [key], + process_content_keys: [process_key], + process_generation_resets: [], }) } } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 305585ac..ebf8b1bb 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -205,6 +205,18 @@ test "ooxml read_parse: relationships apply OPC Markup Compatibility" { assert_eq(understood.length(), 1) } +///| +test "ooxml read_parse: nested Ignorable resets and restores ProcessContent" { + let xml = + #| + let targets = @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") + assert_eq(targets.get("before"), Some("before.xml")) + assert_eq(targets.get("inside"), Some("inside.xml")) + assert_eq(targets.get("after"), Some("after.xml")) + assert_eq(targets.get("blocked"), None) + assert_eq(targets.length(), 3) +} + ///| test "ooxml read_parse: MCE names retain one copy of a long namespace URI" { let names = StringBuilder::new() From 3cb28eb275a4f19872e7eade0cf7409755c159f5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:25:34 +0800 Subject: [PATCH 120/150] fix(ooxml): enforce OPC lexical grammars --- ooxml/read_parse.mbt | 180 ++++++------- ooxml/read_parse_test.mbt | 149 ++++++++++- ooxml/relationship_iri.mbt | 505 +++++++++++++++++++++++++++++++++++++ 3 files changed, 724 insertions(+), 110 deletions(-) create mode 100644 ooxml/relationship_iri.mbt diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index ea22d753..30a30d55 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -144,91 +144,6 @@ fn relationship_attribute( value } -///| -fn relationship_uri_ascii_character(character : UInt16) -> Bool { - (character >= 'A' && character <= 'Z') || - (character >= 'a' && character <= 'z') || - (character >= '0' && character <= '9') || - character == '-' || - character == '.' || - character == '_' || - character == '~' || - character == ':' || - character == '/' || - character == '?' || - character == '#' || - character == '[' || - character == ']' || - character == '@' || - character == '!' || - character == '$' || - character == '&' || - character == ('\'' : UInt16) || - character == '(' || - character == ')' || - character == '*' || - character == '+' || - character == ',' || - character == ';' || - character == '=' || - character == '%' -} - -///| -fn relationship_uri_hex_digit(character : UInt16) -> Bool { - (character >= '0' && character <= '9') || - (character >= 'A' && character <= 'F') || - (character >= 'a' && character <= 'f') -} - -///| -fn relationship_uri_is_valid(value : StringView, absolute : Bool) -> Bool { - if value == "" { - return false - } - let mut index = 0 - while index < value.length() { - let character = value[index] - if character == ('%' : UInt16) { - if index + 2 >= value.length() || - !relationship_uri_hex_digit(value[index + 1]) || - !relationship_uri_hex_digit(value[index + 2]) { - return false - } - index = index + 3 - continue - } - if character < 0x80 && !relationship_uri_ascii_character(character) { - return false - } - index = index + 1 - } - if !absolute { - return true - } - let mut scheme_end = 0 - while scheme_end < value.length() && value[scheme_end] != (':' : UInt16) { - scheme_end = scheme_end + 1 - } - if scheme_end == 0 || - scheme_end >= value.length() || - !value[0].to_char().unwrap().is_ascii_alphabetic() { - return false - } - for index in 1..= 'A' && character <= 'Z') || - (character >= 'a' && character <= 'z') || - (character >= '0' && character <= '9') || - character == '+' || - character == '-' || - character == '.') { - return false - } - } - true -} - ///| priv enum ParsedRelationshipTargetMode { InternalTarget @@ -908,7 +823,7 @@ fn parse_relationship_targets_by_types_selected( if rel.length() > max_relationship_type_chars { raise InvalidXml(msg="relationship type is too long") } - if !relationship_uri_is_valid(rel, true) { + if !relationship_type_iri_valid(rel) { raise InvalidXml(msg="relationship type is invalid") } if id.length() > max_relationship_id_chars { @@ -920,7 +835,8 @@ fn parse_relationship_targets_by_types_selected( if target.length() > max_relationship_target_chars { raise InvalidXml(msg="relationship target is too long") } - if !relationship_uri_is_valid(target, false) { + let external_target = target_mode is ExternalTarget + if !relationship_target_iri_valid(target, external_target) { raise InvalidXml(msg="relationship target is invalid") } if ids.contains(id) { @@ -1070,6 +986,24 @@ fn content_type_skip_optional_whitespace( index } +///| +fn content_type_quoted_character(character : UInt16) -> Bool { + character == ('\t' : UInt16) || + character == (' ' : UInt16) || + character == 0x21 || + (character >= 0x23 && character <= 0x5b) || + (character >= 0x5d && character <= 0x7e) || + (character >= 0x80 && character <= 0xff) +} + +///| +fn content_type_quoted_pair_character(character : UInt16) -> Bool { + character == ('\t' : UInt16) || + character == (' ' : UInt16) || + (character >= 0x21 && character <= 0x7e) || + (character >= 0x80 && character <= 0xff) +} + ///| fn content_type_media_type_is_valid(value : StringView) -> Bool { let mut index = 0 @@ -1091,10 +1025,13 @@ fn content_type_media_type_is_valid(value : StringView) -> Bool { return false } while true { - index = content_type_skip_optional_whitespace(value, index) if index == value.length() { return true } + index = content_type_skip_optional_whitespace(value, index) + if index == value.length() { + return false + } if value[index] != (';' : UInt16) { return false } @@ -1103,14 +1040,12 @@ fn content_type_media_type_is_valid(value : StringView) -> Bool { while index < value.length() && content_type_token_character(value[index]) { index = index + 1 } - if index == name_start { + if index == name_start || + index >= value.length() || + value[index] != ('=' : UInt16) { return false } - index = content_type_skip_optional_whitespace(value, index) - if index >= value.length() || value[index] != ('=' : UInt16) { - return false - } - index = content_type_skip_optional_whitespace(value, index + 1) + index = index + 1 if index >= value.length() { return false } @@ -1127,11 +1062,10 @@ fn content_type_media_type_is_valid(value : StringView) -> Bool { if character == ('\\' : UInt16) { index = index + 1 if index >= value.length() || - value[index] < 0x20 || - value[index] >= 0x7f { + !content_type_quoted_pair_character(value[index]) { return false } - } else if character < 0x20 || character >= 0x7f { + } else if !content_type_quoted_character(character) { return false } index = index + 1 @@ -1173,12 +1107,41 @@ fn content_type_part_name_is_valid( ///| fn content_type_extension_is_valid(value : StringView) -> Bool { - value != "" && - value.length() <= 1024 && - !value.contains(".") && - !value.contains("/") && - !value.contains("\\") && - is_valid_opc_part_segment(value) + if value == "" || value.length() > 1024 { + return false + } + let mut index = 0 + while index < value.length() { + let character = value[index] + if (character >= 'A' && character <= 'Z') || + (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || + character == '!' || + character == '$' || + character == '&' || + character == ('\'' : UInt16) || + character == '(' || + character == ')' || + character == '*' || + character == '+' || + character == ',' || + character == ':' || + character == '=' || + character == '@' || + character == '-' || + character == '_' || + character == '~' { + index += 1 + } else if character == '%' && + index + 2 < value.length() && + opc_ascii_hex_digit(value[index + 1]) && + opc_ascii_hex_digit(value[index + 2]) { + index += 3 + } else { + return false + } + } + true } ///| @@ -1283,7 +1246,7 @@ pub fn parse_package_content_types( raise InvalidXml(msg="content type override part name missing") } let content_type = match scanner.attribute("", "ContentType") { - Some(value) => collapse_schema_whitespace(value) + Some(value) => value None => raise InvalidXml(msg="content type override content type missing") } @@ -1322,11 +1285,11 @@ pub fn parse_package_content_types( "default", ) let extension = match scanner.attribute("", "Extension") { - Some(value) => collapse_schema_whitespace(value).to_lower() + Some(value) => value None => raise InvalidXml(msg="content type default extension missing") } let content_type = match scanner.attribute("", "ContentType") { - Some(value) => collapse_schema_whitespace(value) + Some(value) => value None => raise InvalidXml(msg="content type default content type missing") } @@ -1336,6 +1299,7 @@ pub fn parse_package_content_types( if !content_type_extension_is_valid(extension) { raise InvalidXml(msg="content type default extension is invalid") } + let extension_key = extension.to_lower() if content_type.length() > max_content_type_attribute_chars || !content_type_media_type_is_valid(content_type) { raise InvalidXml(msg="content type default media type is invalid") @@ -1345,10 +1309,10 @@ pub fn parse_package_content_types( raise InvalidXml(msg="content type retained text limit exceeded") } retained_chars = retained_chars + retained - if defaults.contains(extension) { + if defaults.contains(extension_key) { raise InvalidXml(msg="duplicate content type default") } - defaults[extension] = content_type + defaults[extension_key] = content_type } _ => raise InvalidXml(msg="content types element is invalid") } diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index ebf8b1bb..9cf303e7 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -470,6 +470,92 @@ test "ooxml read_parse: unwanted relationships remain schema-valid and ids stay } } +///| +test "ooxml read_parse: relationship IRIs follow OPC component grammar" { + let unicode_xml = + #| + let unicode_targets = @ooxml.parse_internal_relationship_targets( + unicode_xml, "https://example.com/类型", + ) + assert_eq(unicode_targets.get("unicode"), Some("路径/文件.xml?键=值")) + let external_xml = + #| + let ipv6_targets = @ooxml.parse_external_relationship_targets( + external_xml, "https://[2001:db8::1]/type", + ) + assert_eq(ipv6_targets.get("ipv6"), Some("https://[::1]:443/book.xlsx")) + let windows_targets = @ooxml.parse_external_relationship_targets( + external_xml, "urn:windows-file", + ) + assert_eq(windows_targets.get("windows"), Some("file:///C:\\Temp\\book.xlsx")) +} + +///| +test "ooxml read_parse: malformed relationship IRIs fail closed" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "relationship type is invalid", + ), + ( + ( + #| + ), + "relationship type is invalid", + ), + ( + ( + #| + ), + "relationship type is invalid", + ), + ( + ( + #| + ), + "relationship type is invalid", + ), + ( + ( + #| + ), + "relationship target is invalid", + ), + ( + ( + #| + ), + "relationship target is invalid", + ), + ( + ( + #| + ), + "relationship target is invalid", + ), + ( + ( + #| + ), + "relationship target is invalid", + ), + ] + for case in cases { + let (record, expected) = case + let xml = "" + + record + + "" + try @ooxml.parse_internal_relationship_targets(xml, "urn:wanted") catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected malformed relationship IRI error") + } noraise { + _ => fail("expected malformed relationship IRI rejection") + } + } +} + ///| test "ooxml read_parse: relationship limits accept writer boundaries" { let xml = StringBuilder::new() @@ -590,6 +676,23 @@ test "ooxml read_parse: content type lookup honors part identity then default" { ) } +///| +test "ooxml read_parse: content type lexical grammar is preserved" { + let xml = + #| + #| + #| + #| + assert_eq( + @ooxml.find_part_content_type(xml, "custom/file.x%2f"), + Some("application/example ; charset=\"café\""), + ) + assert_eq( + @ooxml.find_part_content_type(xml, "custom/file.A(B)"), + Some("application/parenthesized"), + ) +} + ///| test "ooxml read_parse: content type scans poll cancellation internally" { let xml = StringBuilder::new() @@ -658,6 +761,48 @@ test "ooxml read_parse: content type grammar fails closed" { ), "content type default media type is invalid", ), + ( + ( + #| + ), + "content type default media type is invalid", + ), + ( + ( + #| + ), + "content type default media type is invalid", + ), + ( + ( + #| + ), + "content type default media type is invalid", + ), + ( + ( + #| + ), + "content type override media type is invalid", + ), + ( + ( + #| + ), + "content type default extension is invalid", + ), + ( + ( + #| + ), + "content type default extension is invalid", + ), + ( + ( + #| + ), + "content type default extension is invalid", + ), ( ( #| @@ -677,13 +822,13 @@ test "ooxml read_parse: content type grammar fails closed" { "content types document has no declarations", ), ] - for case in cases { + for index, case in cases { let (xml, expected) = case try @ooxml.find_part_content_type(xml, "xl/workbook.xml") catch { InvalidXml(msg~) => assert_eq(msg, expected) _ => fail("unexpected content type grammar error") } noraise { - _ => fail("expected content type grammar rejection") + _ => fail("expected content type grammar rejection for case \{index}") } } } diff --git a/ooxml/relationship_iri.mbt b/ooxml/relationship_iri.mbt new file mode 100644 index 00000000..7993c80b --- /dev/null +++ b/ooxml/relationship_iri.mbt @@ -0,0 +1,505 @@ +// RFC 3987 IRI-reference grammar for OPC relationships. Relationship Types +// are absolute IRIs without fragments. Targets are IRI references; external +// targets also accept the narrow legacy Windows `file:///` spellings emitted +// by Microsoft Office. + +///| +fn relationship_ascii_hex_digit(character : Char) -> Bool { + character.is_ascii_digit() || + (character >= 'A' && character <= 'F') || + (character >= 'a' && character <= 'f') +} + +///| +fn relationship_iri_private(character : Char) -> Bool { + let value = character.to_int() + (value >= 0xe000 && value <= 0xf8ff) || + (value >= 0xf0000 && value <= 0xffffd) || + (value >= 0x100000 && value <= 0x10fffd) +} + +///| +fn relationship_iri_unreserved(character : Char) -> Bool { + opc_uri_unreserved(character) || opc_iri_ucschar(character) +} + +///| +fn relationship_iri_pchar(character : Char) -> Bool { + relationship_iri_unreserved(character) || + opc_uri_sub_delimiter(character) || + character == ':' || + character == '@' +} + +///| +fn relationship_iri_query_char(character : Char) -> Bool { + relationship_iri_pchar(character) || + relationship_iri_private(character) || + character == '/' || + character == '?' +} + +///| +fn relationship_iri_fragment_char(character : Char) -> Bool { + relationship_iri_pchar(character) || character == '/' || character == '?' +} + +///| +fn relationship_iri_userinfo_char(character : Char) -> Bool { + relationship_iri_unreserved(character) || + opc_uri_sub_delimiter(character) || + character == ':' +} + +///| +fn relationship_iri_reg_name_char(character : Char) -> Bool { + relationship_iri_unreserved(character) || opc_uri_sub_delimiter(character) +} + +///| +fn relationship_iri_chars_valid( + characters : ArrayView[Char], + start : Int, + end : Int, + allowed : (Char) -> Bool, +) -> Bool { + let mut index = start + while index < end { + if characters[index] == '%' { + if index + 2 >= end || + !relationship_ascii_hex_digit(characters[index + 1]) || + !relationship_ascii_hex_digit(characters[index + 2]) { + return false + } + index += 3 + } else { + if !allowed(characters[index]) { + return false + } + index += 1 + } + } + true +} + +///| +fn relationship_ipv4_literal_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + let mut index = start + let mut components = 0 + while index < end { + if components == 4 { + return false + } + let component_start = index + let mut value = 0 + while index < end && characters[index].is_ascii_digit() { + value = value * 10 + characters[index].to_int() - '0'.to_int() + index += 1 + } + let digits = index - component_start + if digits == 0 || + digits > 3 || + value > 255 || + (digits > 1 && characters[component_start] == '0') { + return false + } + components += 1 + if index == end { + break + } + if characters[index] != '.' { + return false + } + index += 1 + } + components == 4 +} + +///| +// RFC 3986 IP-literal validation. A compressed `::` must replace at least one +// 16-bit group; an IPv4 tail occupies the final two groups. +fn relationship_ipv6_literal_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + if start >= end { + return false + } + let mut index = start + let mut groups = 0 + let mut compressed = false + if characters[index] == ':' { + if index + 1 >= end || characters[index + 1] != ':' { + return false + } + compressed = true + index += 2 + if index == end { + return true + } + } + while index < end { + let group_start = index + let mut has_dot = false + while index < end && characters[index] != ':' { + if characters[index] == '.' { + has_dot = true + } + index += 1 + } + if has_dot { + if index != end || + !relationship_ipv4_literal_valid(characters, group_start, end) { + return false + } + groups += 2 + } else { + let digits = index - group_start + if digits == 0 || digits > 4 { + return false + } + for digit_index = group_start + digit_index < index + digit_index = digit_index + 1 { + if !relationship_ascii_hex_digit(characters[digit_index]) { + return false + } + } + groups += 1 + } + if index == end { + break + } + if index + 1 < end && characters[index + 1] == ':' { + if compressed { + return false + } + compressed = true + index += 2 + if index == end { + break + } + } else { + index += 1 + if index == end { + return false + } + } + } + if compressed { + groups < 8 + } else { + groups == 8 + } +} + +///| +fn relationship_ipv_future_literal_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + if start >= end || (characters[start] != 'v' && characters[start] != 'V') { + return false + } + let mut index = start + 1 + let version_start = index + while index < end && relationship_ascii_hex_digit(characters[index]) { + index += 1 + } + if index == version_start || index >= end || characters[index] != '.' { + return false + } + index += 1 + let address_start = index + while index < end && + ( + opc_uri_unreserved(characters[index]) || + opc_uri_sub_delimiter(characters[index]) || + characters[index] == ':' + ) { + index += 1 + } + index == end && index > address_start +} + +///| +fn relationship_iri_authority_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + let mut host_start = start + let mut userinfo_at = -1 + for index = start; index < end; index = index + 1 { + if characters[index] == '@' { + if userinfo_at >= 0 { + return false + } + userinfo_at = index + } + } + if userinfo_at >= 0 { + if !relationship_iri_chars_valid( + characters, start, userinfo_at, relationship_iri_userinfo_char, + ) { + return false + } + host_start = userinfo_at + 1 + } + if host_start < end && characters[host_start] == '[' { + let mut close = -1 + for index = host_start + 1; index < end; index = index + 1 { + if characters[index] == ']' { + close = index + break + } + } + if close < 0 || + ( + !relationship_ipv6_literal_valid(characters, host_start + 1, close) && + !relationship_ipv_future_literal_valid( + characters, + host_start + 1, + close, + ) + ) { + return false + } + if close + 1 == end { + return true + } + if characters[close + 1] != ':' { + return false + } + for index = close + 2; index < end; index = index + 1 { + if !characters[index].is_ascii_digit() { + return false + } + } + return true + } + let mut port_at = -1 + for index = host_start; index < end; index = index + 1 { + if characters[index] == ':' { + if port_at >= 0 { + return false + } + port_at = index + } else if characters[index] == '[' || characters[index] == ']' { + return false + } + } + let host_end = if port_at >= 0 { port_at } else { end } + if !relationship_iri_chars_valid( + characters, host_start, host_end, relationship_iri_reg_name_char, + ) { + return false + } + if port_at >= 0 { + for index = port_at + 1; index < end; index = index + 1 { + if !characters[index].is_ascii_digit() { + return false + } + } + } + true +} + +///| +fn relationship_uri_scheme_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + if start >= end || !characters[start].is_ascii_alphabetic() { + return false + } + for index = start + 1; index < end; index = index + 1 { + let character = characters[index] + if !character.is_ascii_alphabetic() && + !character.is_ascii_digit() && + character != '+' && + character != '-' && + character != '.' { + return false + } + } + true +} + +///| +fn relationship_iri_reference_valid(value : StringView) -> Bool { + if value == "" { + return false + } + let characters = value.to_array() + let mut query_at = characters.length() + let mut fragment_at = characters.length() + for index, character in characters { + if character == '#' { + if fragment_at < characters.length() { + return false + } + fragment_at = index + } else if character == '?' && + query_at == characters.length() && + fragment_at == characters.length() { + query_at = index + } + } + let hierarchy_end = if query_at < fragment_at { + query_at + } else { + fragment_at + } + let mut scheme_at = -1 + for index = 0; index < hierarchy_end; index = index + 1 { + if characters[index] == ':' { + scheme_at = index + break + } + if characters[index] == '/' { + break + } + } + let mut path_start = 0 + if scheme_at >= 0 { + if !relationship_uri_scheme_valid(characters, 0, scheme_at) { + return false + } + path_start = scheme_at + 1 + } + if path_start + 1 < hierarchy_end && + characters[path_start] == '/' && + characters[path_start + 1] == '/' { + let authority_start = path_start + 2 + let mut authority_end = authority_start + while authority_end < hierarchy_end && characters[authority_end] != '/' { + authority_end += 1 + } + if !relationship_iri_authority_valid( + characters, authority_start, authority_end, + ) { + return false + } + path_start = authority_end + } + if !relationship_iri_chars_valid(characters, path_start, hierarchy_end, fn( + character, + ) { + relationship_iri_pchar(character) || character == '/' + }) { + return false + } + if query_at < characters.length() && + !relationship_iri_chars_valid( + characters, + query_at + 1, + fragment_at, + relationship_iri_query_char, + ) { + return false + } + if fragment_at < characters.length() && + !relationship_iri_chars_valid( + characters, + fragment_at + 1, + characters.length(), + relationship_iri_fragment_char, + ) { + return false + } + true +} + +///| +fn relationship_has_uri_scheme(value : StringView) -> Bool { + let characters = value.to_array() + for index, character in characters { + if character == ':' { + return relationship_uri_scheme_valid(characters, 0, index) + } + if character == '/' || character == '?' || character == '#' { + return false + } + } + false +} + +///| +fn office_file_path_characters_valid( + characters : ArrayView[Char], + start : Int, + end : Int, +) -> Bool { + relationship_iri_chars_valid(characters, start, end, fn(character) { + relationship_iri_pchar(character) || character == '/' || character == '\\' + }) +} + +///| +fn office_unc_path_valid(characters : ArrayView[Char]) -> Bool { + if characters.length() < 5 || characters[0] != '\\' || characters[1] != '\\' { + return false + } + let mut server_end = 2 + while server_end < characters.length() && + characters[server_end] != '\\' && + characters[server_end] != '/' { + server_end += 1 + } + if server_end == 2 || + server_end == characters.length() || + !relationship_iri_chars_valid( + characters, 2, server_end, relationship_iri_reg_name_char, + ) { + return false + } + let share_start = server_end + 1 + let mut share_end = share_start + while share_end < characters.length() && + characters[share_end] != '\\' && + characters[share_end] != '/' { + share_end += 1 + } + share_end > share_start && + relationship_iri_chars_valid( + characters, share_start, share_end, relationship_iri_pchar, + ) && + office_file_path_characters_valid(characters, share_end, characters.length()) +} + +///| +fn office_windows_file_iri_valid(value : StringView) -> Bool { + let owned = value.to_owned() + if !owned.to_lower().has_prefix("file:///") { + return false + } + let characters = owned["file:///".length():].to_array() + if characters.length() >= 3 && + characters[0].is_ascii_alphabetic() && + characters[1] == ':' && + (characters[2] == '\\' || characters[2] == '/') { + return office_file_path_characters_valid(characters, 2, characters.length()) + } + office_unc_path_valid(characters) +} + +///| +fn relationship_type_iri_valid(value : StringView) -> Bool { + relationship_has_uri_scheme(value) && + relationship_iri_reference_valid(value) && + !value.contains("#") +} + +///| +fn relationship_target_iri_valid(value : StringView, external : Bool) -> Bool { + relationship_iri_reference_valid(value) || + (external && office_windows_file_iri_valid(value)) +} From b8ab40090c539b3b6c48ae4590a453c43220929b Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:28:52 +0800 Subject: [PATCH 121/150] fix(xlsx): clamp relationship paths at package root --- xlsx/ooxml_rels.mbt | 57 +++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/xlsx/ooxml_rels.mbt b/xlsx/ooxml_rels.mbt index 0b92e1bf..909842d4 100644 --- a/xlsx/ooxml_rels.mbt +++ b/xlsx/ooxml_rels.mbt @@ -246,10 +246,15 @@ fn normalize_rel_part_path( raise InvalidXml(msg="relationship target invalid") } } else if segment == ".." { - if segments.length() == 0 || terminal { + if terminal { raise InvalidXml(msg="relationship target invalid") } - ignore(segments.pop()) + // Relationship references are merged against an absolute package-root + // base. RFC 3986 remove-dot-segments therefore clamps excess parents at + // that root instead of treating them as an invalid escape. + if segments.length() > 0 { + ignore(segments.pop()) + } } else { let valid = @ooxml.is_valid_opc_part_segment_cancellable( segment, @@ -503,34 +508,31 @@ test "ooxml_rels: resolve target normalizes dot segments" { } ///| -test "ooxml_rels: resolve target rejects escaping root via dot segments" { +test "ooxml_rels: resolve target clamps excess parents at the package root" { let rel_result = Ok(resolve_rel_target("../../../sheet1.xml", "worksheets")) catch { e => Err(e) } match rel_result { Err(InvalidXml(msg~)) => inspect(msg, content="relationship target invalid") - _ => fail("expected InvalidXml for relative escape in resolve_rel_target") - } - let workbook_result = Ok(resolve_workbook_rel_target("../../sheet1.xml")) catch { - e => Err(e) - } - match workbook_result { - Err(InvalidXml(msg~)) => inspect(msg, content="relationship target invalid") - _ => - fail( - "expected InvalidXml for relative escape in resolve_workbook_rel_target", - ) + _ => fail("expected xl-only resolver to reject a root-level package part") } - - let relocated_result = Ok( + inspect(resolve_workbook_rel_target("../../sheet1.xml"), content="sheet1.xml") + inspect( resolve_part_rel_target("custom/book.xml", "../../sheet1.xml"), - ) catch { - e => Err(e) - } - match relocated_result { - Err(InvalidXml(msg~)) => inspect(msg, content="relationship target invalid") - _ => fail("expected relocated relationship target to reject a root escape") - } + content="sheet1.xml", + ) + inspect( + resolve_part_rel_target("custom/book.xml", "../../../../sheet1.xml"), + content="sheet1.xml", + ) + inspect( + resolve_part_rel_target("custom/book.xml", "/../sheet1.xml"), + content="sheet1.xml", + ) + inspect( + resolve_rel_target("../../../xl/worksheets/sheet1.xml", "worksheets"), + content="xl/worksheets/sheet1.xml", + ) } ///| @@ -548,6 +550,15 @@ test "ooxml_rels: normalize and resolve invalid-path guards" { Err(InvalidXml(msg~)) => inspect(msg, content="relationship target invalid") _ => fail("expected InvalidXml for non-xl absolute resolve_rel_target") } + + for target in ["../..", "../../.", "../../../"] { + try resolve_part_rel_target("custom/book.xml", target) catch { + InvalidXml(msg~) => assert_eq(msg, "relationship target invalid") + _ => fail("unexpected final relationship path error") + } noraise { + _ => fail("directory-valued relationship target was accepted: \{target}") + } + } } ///| From fc393b0f47c9e11342ec64be8911c066ad2720d4 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:42:43 +0800 Subject: [PATCH 122/150] fix(xlsx): shift structural sqref ranges --- xlsx/sqref.mbt | 56 +++++ xlsx/worksheet.mbt | 517 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 536 insertions(+), 37 deletions(-) diff --git a/xlsx/sqref.mbt b/xlsx/sqref.mbt index 8cfffeef..c75c9570 100644 --- a/xlsx/sqref.mbt +++ b/xlsx/sqref.mbt @@ -21,6 +21,62 @@ fn xml_whitespace_tokens(value : StringView) -> Array[String] { tokens } +///| +let max_structural_sqref_tokens = 65_536 + +///| +let max_structural_sqref_token_chars = 128 + +///| +fn xml_whitespace_tokens_cancellable( + value : StringView, + cancelled? : () -> Bool = () => false, +) -> Array[String] raise XlsxError { + let tokens : Array[String] = [] + let mut current = StringBuilder::new() + let mut has_chars = false + let mut current_chars = 0 + for index, ch in value { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } + if is_attr_space(ch) { + if has_chars { + if tokens.length() >= max_structural_sqref_tokens { + raise ResourceLimitExceeded( + kind="sqref_tokens", + limit=max_structural_sqref_tokens, + actual=tokens.length() + 1, + ) + } + tokens.push(current.to_string()) + current = StringBuilder::new() + has_chars = false + current_chars = 0 + } + } else { + current_chars += if ch.to_int() > 0xffff { 2 } else { 1 } + if current_chars > max_structural_sqref_token_chars { + raise InvalidXml(msg="sqref reference invalid") + } + current.write_char(ch) + has_chars = true + } + } + if has_chars { + if tokens.length() >= max_structural_sqref_tokens { + raise ResourceLimitExceeded( + kind="sqref_tokens", + limit=max_structural_sqref_tokens, + actual=tokens.length() + 1, + ) + } + tokens.push(current.to_string()) + } + check_read_cancelled(cancelled) + tokens +} + ///| fn invalid_coordinate_xml(field : StringView) -> Unit raise XlsxError { raise InvalidXml(msg="\{field.to_owned()} reference invalid") diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index de37396a..4074962a 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2493,6 +2493,23 @@ fn Worksheet::next_x14_cf_rule_id(self : Worksheet) -> String { "{00000000-0000-0000-0000-000000000000}" } +///| +fn next_available_x14_cf_rule_id( + start : Int, + used : Map[String, X14DataBarProps], +) -> (String, Int) { + let mut counter = start + while true { + let suffix = pad_left_dec(counter, 12) + let id = "{00000000-0000-0000-0000-\{suffix}}" + counter += 1 + if used.get(id) is None { + return (id, counter) + } + } + ("{00000000-0000-0000-0000-000000000000}", counter) +} + ///| fn inject_x14_id_ext_lst_into_cf_rule( rule_xml : String, @@ -3610,12 +3627,13 @@ fn Worksheet::insert_rows( ) translation_budget.checkpoint() // Transactional: stage every shifted collection into locals first — cell, - // merge, hyperlink, filter, table, sparkline, image, and chart references - // shift through `cell_ref_from`/range adjusters that raise once a coordinate - // leaves the grid, and the two numerically-shifted collections (row - // dimensions, page breaks) are bound-checked explicitly below. Nothing on - // `self` is mutated until every stage succeeds, so an overflowing insert - // raises and leaves the worksheet unchanged instead of half-shifted. + // merge, conditional-format, validation, hyperlink, filter, table, + // sparkline, image, and chart references shift through + // `cell_ref_from`/range adjusters that raise once a coordinate leaves the + // grid, and the two numerically-shifted collections (row dimensions, page + // breaks) are bound-checked explicitly below. Nothing on `self` is mutated + // until every stage succeeds, so an overflowing insert raises and leaves the + // worksheet unchanged instead of half-shifted. let updated_cells : Array[Cell] = [] for index, stored_cell in self.cells { if (index & 4095) == 0 { @@ -3652,6 +3670,27 @@ fn Worksheet::insert_rows( } updated_merges.push(adjust_range_after_row_insert(range_ref, row, count)) } + let updated_conditional_formats = adjust_xml_sqrefs_after_insert( + self.conditional_formats, + row, + count, + SqrefRows, + cancelled=translation_budget.cancelled, + ) + let updated_x14_data_bars = adjust_x14_data_bars_after_insert( + self.x14_data_bars, + row, + count, + SqrefRows, + cancelled=translation_budget.cancelled, + ) + let updated_data_validations = adjust_xml_sqrefs_after_insert( + self.data_validations, + row, + count, + SqrefRows, + cancelled=translation_budget.cancelled, + ) let updated_links : Array[Hyperlink] = [] for index, link in self.hyperlinks { if (index & 4095) == 0 { @@ -3768,6 +3807,14 @@ fn Worksheet::insert_rows( self.cells.append(updated_cells) self.merged_cells.clear() self.merged_cells.append(updated_merges) + self.conditional_formats.clear() + self.conditional_formats.append(updated_conditional_formats) + self.x14_data_bars.clear() + for id, props in updated_x14_data_bars { + self.x14_data_bars[id] = props + } + self.data_validations.clear() + self.data_validations.append(updated_data_validations) self.hyperlinks.clear() self.hyperlinks.append(updated_links) self.auto_filter = updated_filter @@ -4030,33 +4077,194 @@ fn format_cell_ref_with_abs( } ///| -fn duplicate_sqref_for_row( - row : Int, - target_row : Int, +fn split_sqref_range(text : StringView) -> (String, String) { + let left = StringBuilder::new() + let right = StringBuilder::new() + let mut seen_sep = false + for ch in text { + if !seen_sep && ch == ':' { + seen_sep = true + } else if seen_sep { + right.write_char(ch) + } else { + left.write_char(ch) + } + } + if seen_sep { + (left.to_string(), right.to_string()) + } else { + let cell = text.to_owned() + (cell, cell) + } +} + +///| +fn decode_structural_sqref( + encoded : StringView, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + @ooxml.decode_xml_attribute(encoded, cancelled~) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } +} + +///| +priv enum SqrefInsertAxis { + SqrefRows + SqrefColumns +} + +///| +fn adjust_sqref_after_insert( sqref : StringView, -) -> Array[String] raise XlsxError { - fn split_sqref_range(text : StringView) -> (String, String) { - let left = StringBuilder::new() - let right = StringBuilder::new() - let mut seen_sep = false - for ch in text { - if !seen_sep && ch == ':' { - seen_sep = true - } else if seen_sep { - right.write_char(ch) - } else { - left.write_char(ch) + start : Int, + count : Int, + axis : SqrefInsertAxis, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + let adjusted : Array[String] = [] + let tokens = xml_whitespace_tokens_cancellable(sqref, cancelled~) + if tokens.length() == 0 { + raise InvalidXml(msg="sqref reference invalid") + } + for index, token in tokens { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } + if token.length() > 128 { + raise InvalidXml(msg="sqref reference invalid") + } + let had_range = token.contains(":") + let range_ref = if had_range { token } else { token + ":" + token } + let (min_row, min_col, max_row, max_col) = parse_range_ref(range_ref) + let (new_min_row, new_min_col, new_max_row, new_max_col) = match axis { + SqrefRows => { + let (next_min, next_max) = adjust_bounds_for_insert( + min_row, max_row, start, count, + ) + (next_min, min_col, next_max, max_col) + } + SqrefColumns => { + let (next_min, next_max) = adjust_bounds_for_insert( + min_col, max_col, start, count, + ) + (min_row, next_min, max_row, next_max) } } - if seen_sep { - (left.to_string(), right.to_string()) + let first = cell_ref_from(new_min_row, new_min_col) + if had_range { + adjusted.push("\{first}:\{cell_ref_from(new_max_row, new_max_col)}") } else { - let cell = text.to_owned() - (cell, cell) + adjusted.push(first) + } + } + check_read_cancelled(cancelled) + adjusted.join(" ") +} + +///| +fn adjust_xml_sqref_after_insert( + xml : StringView, + start : Int, + count : Int, + axis : SqrefInsertAxis, + cancelled? : () -> Bool = () => false, +) -> String raise XlsxError { + let tag_text = match tag_attributes_in(xml, "conditionalFormatting") { + Some(value) => value + None => + match tag_attributes_in(xml, "dataValidation") { + Some(value) => value + None => return xml.to_owned() + } + } + let encoded = match attr_value(tag_text, "sqref") { + Some(value) => value + None => return xml.to_owned() + } + let decoded = decode_structural_sqref(encoded, cancelled~) + let shifted = adjust_sqref_after_insert( + decoded, + start, + count, + axis, + cancelled~, + ) + match replace_attr_value_in_open_tag(xml, "sqref", shifted) { + Some(value) => value + None => raise InvalidXml(msg="sqref update failed") + } +} + +///| +fn adjust_xml_sqrefs_after_insert( + entries : ArrayView[String], + start : Int, + count : Int, + axis : SqrefInsertAxis, + cancelled? : () -> Bool = () => false, +) -> Array[String] raise XlsxError { + let shifted : Array[String] = [] + for index, xml in entries { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } + shifted.push( + adjust_xml_sqref_after_insert(xml, start, count, axis, cancelled~), + ) + } + check_read_cancelled(cancelled) + shifted +} + +///| +fn adjust_x14_data_bars_after_insert( + entries : Map[String, X14DataBarProps], + start : Int, + count : Int, + axis : SqrefInsertAxis, + cancelled? : () -> Bool = () => false, +) -> Map[String, X14DataBarProps] raise XlsxError { + let shifted : Map[String, X14DataBarProps] = Map([]) + let mut index = 0 + for id, props in entries { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) } + shifted[id] = { + sqref: adjust_sqref_after_insert( + props.sqref, + start, + count, + axis, + cancelled~, + ), + bar_direction: props.bar_direction, + bar_solid: props.bar_solid, + bar_border_color: props.bar_border_color, + } + index += 1 } + check_read_cancelled(cancelled) + shifted +} + +///| +fn duplicate_sqref_for_row( + row : Int, + target_row : Int, + sqref : StringView, + cancelled? : () -> Bool = () => false, +) -> Array[String] raise XlsxError { let refs : Array[String] = [] - for text in xml_whitespace_tokens(sqref) { + for index, text in xml_whitespace_tokens_cancellable(sqref, cancelled~) { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } + if text.length() > 128 { + raise InvalidXml(msg="sqref reference invalid") + } let (start_ref, end_ref) = split_sqref_range(text) let (row1, col1, abs_col1, abs_row1) = parse_cell_ref_with_abs(start_ref) let (row2, col2, abs_col2, abs_row2) = parse_cell_ref_with_abs(end_ref) @@ -4070,6 +4278,7 @@ fn duplicate_sqref_for_row( refs.push("\{from_ref}:\{to_ref}") } } + check_read_cancelled(cancelled) refs } @@ -4080,6 +4289,7 @@ fn duplicate_xml_sqref( xml : StringView, row : Int, target_row : Int, + cancelled? : () -> Bool = () => false, ) -> String? raise XlsxError { let tag_text = match tag_attributes_in(xml, "conditionalFormatting") { Some(value) => value @@ -4089,11 +4299,12 @@ fn duplicate_xml_sqref( None => return None } } - let sqref = match attr_value(tag_text, "sqref") { + let encoded_sqref = match attr_value(tag_text, "sqref") { Some(value) => value None => return None } - let refs = duplicate_sqref_for_row(row, target_row, sqref) + let sqref = decode_structural_sqref(encoded_sqref, cancelled~) + let refs = duplicate_sqref_for_row(row, target_row, sqref, cancelled~) if refs.length() == 0 { return None } @@ -4101,6 +4312,124 @@ fn duplicate_xml_sqref( replace_attr_value_in_open_tag(xml, "sqref", joined) } +///| +priv struct ConditionalFormatCopies { + xml : Array[String] + x14_data_bars : Map[String, X14DataBarProps] + next_x14_cf_rule_id_counter : Int +} + +///| +fn remap_duplicated_x14_ids( + xml : StringView, + sqref : String, + source : Map[String, X14DataBarProps], + used : Map[String, X14DataBarProps], + start_counter : Int, +) -> (String, Map[String, X14DataBarProps], Int) raise XlsxError { + let chunks = Array::from_iter(xml.split("")) + if chunks.length() <= 1 { + return (xml.to_owned(), Map([]), start_counter) + } + let additions : Map[String, X14DataBarProps] = Map([]) + let builder = StringBuilder::new() + builder.write_view(chunks[0]) + let mut counter = start_counter + for index = 1; index < chunks.length(); index = index + 1 { + let chunk = chunks[index] + let end = match chunk.find("") { + Some(value) => value + None => { + builder.write_view("") + builder.write_view(chunk) + continue + } + } + let encoded_id = chunk[:end] + let id = @ooxml.decode_xml_attribute(encoded_id) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + match source.get(id) { + Some(props) => { + let (new_id, next_counter) = next_available_x14_cf_rule_id( + counter, used, + ) + counter = next_counter + let duplicated = { + sqref, + bar_direction: props.bar_direction, + bar_solid: props.bar_solid, + bar_border_color: props.bar_border_color, + } + used[new_id] = duplicated + additions[new_id] = duplicated + builder.write_view("") + builder.write_view(escape_xml_text(new_id)) + builder.write_view("") + } + None => { + builder.write_view("") + builder.write_view(encoded_id) + builder.write_view("") + } + } + builder.write_view(chunk[end + "".length():]) + } + (builder.to_string(), additions, counter) +} + +///| +fn duplicate_conditional_format_copies( + entries : ArrayView[String], + x14_data_bars : Map[String, X14DataBarProps], + next_x14_cf_rule_id_counter : Int, + row : Int, + target_row : Int, + cancelled? : () -> Bool = () => false, +) -> ConditionalFormatCopies raise XlsxError { + let copies : Array[String] = [] + let additions : Map[String, X14DataBarProps] = Map([]) + let used : Map[String, X14DataBarProps] = Map([]) + for id, props in x14_data_bars { + used[id] = props + } + let mut counter = next_x14_cf_rule_id_counter + for index, xml in entries { + if (index & 4095) == 0 { + check_read_cancelled(cancelled) + } + match duplicate_xml_sqref(xml, row, target_row, cancelled~) { + Some(copy) => { + let tag = match tag_attributes_in(copy, "conditionalFormatting") { + Some(value) => value + None => raise InvalidXml(msg="conditionalFormatting tag missing") + } + let encoded_sqref = match attr_value(tag, "sqref") { + Some(value) => value + None => raise InvalidXml(msg="conditionalFormatting sqref missing") + } + let sqref = decode_structural_sqref(encoded_sqref, cancelled~) + let (remapped, new_bars, next_counter) = remap_duplicated_x14_ids( + copy, sqref, x14_data_bars, used, counter, + ) + counter = next_counter + for id, props in new_bars { + additions[id] = props + } + copies.push(remapped) + } + None => () + } + } + check_read_cancelled(cancelled) + { + xml: copies, + x14_data_bars: additions, + next_x14_cf_rule_id_counter: counter, + } +} + ///| fn duplicate_xml_sqref_copies( entries : ArrayView[String], @@ -4113,7 +4442,7 @@ fn duplicate_xml_sqref_copies( if (index & 4095) == 0 { check_read_cancelled(cancelled) } - match duplicate_xml_sqref(xml, row, target_row) { + match duplicate_xml_sqref(xml, row, target_row, cancelled~) { Some(value) => copies.push(value) None => () } @@ -4252,8 +4581,10 @@ fn Worksheet::duplicate_row_to( }) } let row_dim = self.row_dimensions.get(row) - let conditional_format_copies = duplicate_xml_sqref_copies( + let conditional_format_copies = duplicate_conditional_format_copies( self.conditional_formats, + self.x14_data_bars, + self.x14_cf_rule_id_counter, row, target_row, cancelled=translation_budget.cancelled, @@ -4295,7 +4626,11 @@ fn Worksheet::duplicate_row_to( } None => self.row_dimensions.remove(target_row) } - self.conditional_formats.append(conditional_format_copies) + self.conditional_formats.append(conditional_format_copies.xml) + for id, props in conditional_format_copies.x14_data_bars { + self.x14_data_bars[id] = props + } + self.x14_cf_rule_id_counter = conditional_format_copies.next_x14_cf_rule_id_counter self.data_validations.append(data_validation_copies) self.merged_cells.append(merge_copies) } @@ -4336,10 +4671,11 @@ fn Worksheet::insert_cols( fn(_cell : Cell) -> Bool { true }, ) translation_budget.checkpoint() - // Transactional (see insert_rows): stage everything into locals — reference - // shifts raise once a coordinate leaves the grid, column dimensions and page - // breaks are bound-checked explicitly — and commit only once every stage has - // succeeded, so an overflowing insert leaves the worksheet unchanged. + // Transactional (see insert_rows): stage everything into locals — including + // conditional-format and validation sqrefs. Reference shifts raise once a + // coordinate leaves the grid, column dimensions and page breaks are + // bound-checked explicitly, and the state commits only after every stage has + // succeeded. let updated_cells : Array[Cell] = [] for index, stored_cell in self.cells { if (index & 4095) == 0 { @@ -4376,6 +4712,27 @@ fn Worksheet::insert_cols( } updated_merges.push(adjust_range_after_col_insert(range_ref, col, count)) } + let updated_conditional_formats = adjust_xml_sqrefs_after_insert( + self.conditional_formats, + col, + count, + SqrefColumns, + cancelled=translation_budget.cancelled, + ) + let updated_x14_data_bars = adjust_x14_data_bars_after_insert( + self.x14_data_bars, + col, + count, + SqrefColumns, + cancelled=translation_budget.cancelled, + ) + let updated_data_validations = adjust_xml_sqrefs_after_insert( + self.data_validations, + col, + count, + SqrefColumns, + cancelled=translation_budget.cancelled, + ) let updated_links : Array[Hyperlink] = [] for index, link in self.hyperlinks { if (index & 4095) == 0 { @@ -4489,6 +4846,14 @@ fn Worksheet::insert_cols( self.cells.append(updated_cells) self.merged_cells.clear() self.merged_cells.append(updated_merges) + self.conditional_formats.clear() + self.conditional_formats.append(updated_conditional_formats) + self.x14_data_bars.clear() + for id, props in updated_x14_data_bars { + self.x14_data_bars[id] = props + } + self.data_validations.clear() + self.data_validations.append(updated_data_validations) self.hyperlinks.clear() self.hyperlinks.append(updated_links) self.auto_filter = updated_filter @@ -6270,12 +6635,90 @@ test "worksheet wb: duplicate-row helper edge branches" { } ///| -test "worksheet wb: duplicate row stages malformed sqref before mutation" { +test "worksheet wb: upward duplicate decodes and shifts existing sqrefs" { + let sheet = Worksheet::new("EntitySqref") + sheet.set_cell("A2", "tail") + sheet.set_cell("A5", "source") + sheet.add_data_validation_xml( + "", + ) + sheet.add_conditional_format_xml( + "1=1", + ) + sheet.duplicate_row_to(5, 2) + debug_inspect(sheet.get_cell("A2"), content="Some(\"source\")") + debug_inspect(sheet.get_cell("A3"), content="Some(\"tail\")") + debug_inspect(sheet.get_cell("A6"), content="Some(\"source\")") + inspect(sheet.data_validations().length(), content="2") + assert_true(sheet.data_validations()[0].contains("sqref=\"A6 B6\"")) + assert_true(sheet.data_validations()[1].contains("sqref=\"A2:A2 B2:B2\"")) + inspect(sheet.conditional_formats().length(), content="2") + assert_true(sheet.conditional_formats()[0].contains("sqref=\"A6 B6\"")) + assert_true(sheet.conditional_formats()[1].contains("sqref=\"A2:A2 B2:B2\"")) +} + +///| +test "worksheet wb: duplicate row remaps x14 conditional-format ids" { + let sheet = Worksheet::new("X14Duplicate") + let data_bar = ConditionalFormatOptions::new("data_bar") + data_bar.set_bar_solid(true) + data_bar.set_bar_border_color("#0000FF") + sheet.set_conditional_format("B5:C5", [data_bar]) + let source_xml = sheet.conditional_formats()[0] + sheet.duplicate_row_to(5, 2) + inspect(sheet.conditional_formats().length(), content="2") + assert_true(sheet.conditional_formats()[0].contains("sqref=\"B6:C6\"")) + assert_true(sheet.conditional_formats()[1].contains("sqref=\"B2:C2\"")) + assert_true(sheet.conditional_formats()[1] != source_xml) + assert_eq(sheet.x14_data_bars.length(), 2) + let mut shifted = 0 + let mut duplicated = 0 + for _, props in sheet.x14_data_bars { + if props.sqref == "B6:C6" { + shifted += 1 + } else if props.sqref == "B2:C2" { + duplicated += 1 + } + } + assert_eq(shifted, 1) + assert_eq(duplicated, 1) +} + +///| +test "worksheet wb: row and column inserts shift sqref ranges" { + let rows = Worksheet::new("Rows") + rows.add_conditional_format_xml( + "1=1", + ) + rows.insert_rows(3, 2) + assert_true(rows.conditional_formats()[0].contains("sqref=\"A2:B6 C7\"")) + + let cols = Worksheet::new("Columns") + cols.add_data_validation_xml( + "", + ) + cols.insert_cols(2, 1) + assert_true(cols.data_validations()[0].contains("sqref=\"C2 D3:E4\"")) + + let x14 = Worksheet::new("X14") + let data_bar = ConditionalFormatOptions::new("data_bar") + data_bar.set_bar_solid(true) + data_bar.set_bar_border_color("#0000FF") + x14.set_conditional_format("B2:C4", [data_bar]) + x14.insert_rows(3, 2) + assert_eq(x14.x14_data_bars.length(), 1) + for _, props in x14.x14_data_bars { + assert_eq(props.sqref, "B2:C6") + } +} + +///| +test "worksheet wb: malformed sqref insertion remains atomic" { let sheet = Worksheet::new("Atomic") sheet.set_cell("A1", "source") sheet.set_cell("A2", "tail") sheet.add_data_validation_xml( - "", + "", ) let result : Result[Unit, Error] = Ok(sheet.duplicate_row_to(1, 2)) catch { error => Err(error) From 5a89f999ef8fb2fc54bef5e7557c4040f86b3b34 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:44:44 +0800 Subject: [PATCH 123/150] fix(xlsx): validate workbook sheet identity --- xlsx/read_workbook_xml.mbt | 88 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/xlsx/read_workbook_xml.mbt b/xlsx/read_workbook_xml.mbt index 4a2184ae..34770597 100644 --- a/xlsx/read_workbook_xml.mbt +++ b/xlsx/read_workbook_xml.mbt @@ -1,6 +1,7 @@ ///| priv struct WorkbookSheetInfo { name : String + sheet_id : UInt state : SheetState rel_id : String } @@ -47,6 +48,7 @@ fn parse_workbook_sheets( ) -> Array[WorkbookSheetInfo] raise XlsxError { let sheets : Array[WorkbookSheetInfo] = [] let sheet_names : Map[String, Bool] = Map([]) + let sheet_ids : Map[UInt, Bool] = Map([]) let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) let mut workbook_namespace : String? = None while workbook_scanner_next(scanner) { @@ -98,6 +100,17 @@ fn parse_workbook_sheets( raise InvalidXml(msg="duplicate sheet name") } sheet_names[normalized_name] = true + let sheet_id = match workbook_scanner_attribute(scanner, "", "sheetId") { + Some(value) => + @string.parse_uint(value, base=10) catch { + _ => raise InvalidXml(msg="sheetId invalid") + } + None => raise InvalidXml(msg="sheetId missing") + } + if sheet_ids.contains(sheet_id) { + raise InvalidXml(msg="duplicate sheetId") + } + sheet_ids[sheet_id] = true let relationship_namespace = if namespace_uri == strict_spreadsheet_namespace { strict_relationship_attribute_namespace @@ -111,14 +124,15 @@ fn parse_workbook_sheets( } let state = match workbook_scanner_attribute(scanner, "", "state") { Some(value) => - match value.to_lower() { + match value { + "visible" => Visible "hidden" => Hidden - "veryhidden" => VeryHidden - _ => Visible + "veryHidden" => VeryHidden + _ => raise InvalidXml(msg="sheet state invalid") } None => Visible } - sheets.push({ name, state, rel_id }) + sheets.push({ name, sheet_id, state, rel_id }) } sheets } @@ -150,6 +164,72 @@ test "read workbook sheets rejects invalid and case-duplicate names" { } } +///| +test "read workbook sheets requires unique unsigned sheet ids" { + let prefix = "" + let suffix = "" + let invalid : Array[(String, String)] = [ + ("", "sheetId missing"), + ("", "sheetId invalid"), + ( + "", "sheetId invalid", + ), + ( + "", + "duplicate sheetId", + ), + ] + for entry in invalid { + let (body, expected) = entry + try parse_workbook_sheets(prefix + body + suffix, 10) catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected sheetId error") + } noraise { + _ => fail("expected invalid sheetId rejection") + } + } + + let maximum = parse_workbook_sheets( + prefix + + "" + + suffix, + 10, + ) + assert_eq(maximum.length(), 1) + assert_eq(maximum[0].sheet_id, 4294967295U) +} + +///| +test "read workbook sheets accepts only exact visibility states" { + let prefix = "" + let suffix = "" + let valid = parse_workbook_sheets( + prefix + + "" + + "" + + "" + + "" + + suffix, + 10, + ) + assert_true(valid[0].state is Visible) + assert_true(valid[1].state is Visible) + assert_true(valid[2].state is Hidden) + assert_true(valid[3].state is VeryHidden) + + for state in ["Visible", "Hidden", "veryhidden", "unknown", ""] { + let xml = prefix + + "" + + suffix + try parse_workbook_sheets(xml, 10) catch { + InvalidXml(msg~) => assert_eq(msg, "sheet state invalid") + _ => fail("unexpected sheet-state error") + } noraise { + _ => fail("expected invalid sheet-state rejection") + } + } +} + ///| fn parse_active_sheet_index(xml : StringView) -> Int? raise XlsxError { let xml_str = xml.to_owned() From b2baeb69368baeeb097bd9d623e15d2f18321b53 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 06:49:54 +0800 Subject: [PATCH 124/150] fix(xlsx): normalize workday date serials --- xlsx/formula_builtins_wbtest.mbt | 67 ++++++++++++++++++++++++ xlsx/formula_eval.mbt | 87 +++++++++++++++++++++++--------- 2 files changed, 129 insertions(+), 25 deletions(-) diff --git a/xlsx/formula_builtins_wbtest.mbt b/xlsx/formula_builtins_wbtest.mbt index 2b44393d..88f85c36 100644 --- a/xlsx/formula_builtins_wbtest.mbt +++ b/xlsx/formula_builtins_wbtest.mbt @@ -54,6 +54,73 @@ fn is_formula_list(value : FormulaValue) -> Bool { } } +///| +test "formula builtins wb: workday functions normalize fractional dates" { + let workbook = Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + let ctx = CalcContext::new() + let cases : Array[(String, Array[Expr], Double)] = [ + ("NETWORKDAYS", [Number(1.9), Number(2.1)], 2.0), + ("NETWORKDAYS.INTL", [Number(1.9), Number(2.1), Number(1.0)], 2.0), + ("NETWORKDAYS", [Number(1.0), Number(1.0), List([Number(1.9)])], 0.0), + ( + "NETWORKDAYS.INTL", + [Number(1.0), Number(1.0), Number(1.0), List([Number(1.9)])], + 0.0, + ), + ("WORKDAY", [Number(1.9), Number(0.9)], 1.0), + ("WORKDAY.INTL", [Number(1.9), Number(0.9), Number(1.0)], 1.0), + ("WORKDAY", [Number(1.0), Number(1.0), List([Number(2.9)])], 3.0), + ( + "WORKDAY.INTL", + [Number(1.0), Number(1.0), Number(1.0), List([Number(2.9)])], + 3.0, + ), + ( + "WORKDAY", + [Number(1.0), Number(1.0), List([Number(2.1), Number(2.9)])], + 3.0, + ), + ] + for entry in cases { + let (name, args, expected) = entry + assert_true( + is_formula_number( + eval_function(workbook, "Sheet1", name, args, ctx), + expected, + ), + ) + } + + let maximum_1900 = maximum_supported_formula_date_serial(false) + assert_true( + is_formula_number( + eval_function( + workbook, + "Sheet1", + "WORKDAY", + [Number(Double::from_int(maximum_1900) + 0.5), Number(0.0)], + ctx, + ), + Double::from_int(maximum_1900), + ), + ) + let maximum_1904 = maximum_supported_formula_date_serial(true) + let ctx_1904 = CalcContext::new(use_1904_dates=true) + assert_true( + is_formula_number( + eval_function( + workbook, + "Sheet1", + "WORKDAY.INTL", + [Number(Double::from_int(maximum_1904) + 0.5), Number(0.0), Number(1.0)], + ctx_1904, + ), + Double::from_int(maximum_1904), + ), + ) +} + ///| test "formula builtins wb: row column and formulatxt guards" { let workbook = Workbook::new() diff --git a/xlsx/formula_eval.mbt b/xlsx/formula_eval.mbt index d7936a37..2b93d5af 100644 --- a/xlsx/formula_eval.mbt +++ b/xlsx/formula_eval.mbt @@ -6179,15 +6179,41 @@ fn formula_date_serial_is_supported( serial >= 0 && serial <= maximum_supported_formula_date_serial(use_1904_dates) } +///| +fn normalized_formula_date_serial( + serial : Double, + use_1904_dates : Bool, +) -> Int? { + if serial.is_nan() || serial.is_inf() { + return None + } + let normalized = Double::floor(serial) + if normalized < 0.0 || + normalized > + Double::from_int(maximum_supported_formula_date_serial(use_1904_dates)) { + return None + } + Some(Double::to_int(normalized)) +} + ///| fn collect_holidays( value : FormulaValue, use_1904_dates? : Bool = false, ) -> Array[Int] { let holidays : Array[Int] = [] + let seen : Map[Int, Bool] = Map([]) for item in flatten_values([value]) { match value_as_date_serial(item, use_1904_dates~) { - Ok(serial) => holidays.push(Double::to_int(Double::ceil(serial))) + Ok(serial) => + match normalized_formula_date_serial(serial, use_1904_dates) { + Some(day) => + if !seen.contains(day) { + seen[day] = true + holidays.push(day) + } + None => () + } Err(_) => () } } @@ -6200,7 +6226,7 @@ fn workday_intl_adjust( sign : Int, holidays : Array[Int], weekend_mask : Array[Int], - start_serial : Double, + start_date : Int, use_1904_dates? : Bool = false, ) -> Int? { let mut adjusted = end_date @@ -6209,7 +6235,7 @@ fn workday_intl_adjust( if holiday > adjusted { break } - if holiday > Double::to_int(Double::ceil(start_serial)) { + if holiday > start_date { if is_workday_mask( weekend_mask, Double::from_int(holiday), @@ -6235,7 +6261,7 @@ fn workday_intl_adjust( if holiday < adjusted { continue } - if holiday < Double::to_int(Double::ceil(start_serial)) { + if holiday < start_date { if is_workday_mask( weekend_mask, Double::from_int(holiday), @@ -6270,6 +6296,16 @@ fn networkdays_intl_value( holidays : Array[Int], use_1904_dates? : Bool = false, ) -> FormulaValue { + let start_date = match + normalized_formula_date_serial(start_serial, use_1904_dates) { + Some(value) => value + None => return Error(formula_error_num) + } + let end_date = match + normalized_formula_date_serial(end_serial, use_1904_dates) { + Some(value) => value + None => return Error(formula_error_num) + } holidays.sort() let (weekend_mask, workdays) = match weekend_mask_from_value(weekend_value) { Ok(value) => value @@ -6278,8 +6314,8 @@ fn networkdays_intl_value( if workdays == 0 { return Error(formula_error_value) } - let mut start = start_serial - let mut end = end_serial + let mut start = start_date + let mut end = end_date let mut sign = 1 if start > end { sign = -1 @@ -6288,13 +6324,13 @@ fn networkdays_intl_value( end = temp } let offset = end - start - let weeks = Double::to_int(Double::floor(offset / 7.0)) + let weeks = offset / 7 let mut count = weeks * workdays - let mut days_mod = Double::to_int(Double::floor(offset)) % 7 + let mut days_mod = offset % 7 while days_mod >= 0 { if is_workday_mask( weekend_mask, - end - Double::from_int(days_mod), + Double::from_int(end - days_mod), use_1904_dates~, ) { count = count + 1 @@ -6302,10 +6338,9 @@ fn networkdays_intl_value( days_mod = days_mod - 1 } for holiday in holidays { - let holiday_serial = Double::from_int(holiday) - if is_workday_mask(weekend_mask, holiday_serial, use_1904_dates~) && - holiday_serial >= start && - holiday_serial <= end { + if is_workday_mask(weekend_mask, Double::from_int(holiday), use_1904_dates~) && + holiday >= start && + holiday <= end { count = count - 1 } } @@ -6320,9 +6355,18 @@ fn workday_intl_value( holidays : Array[Int], use_1904_dates? : Bool = false, ) -> FormulaValue { + let start_date = match + normalized_formula_date_serial(start_serial, use_1904_dates) { + Some(value) => value + None => return Error(formula_error_num) + } + if days.is_nan() || days.is_inf() { + return Error(formula_error_num) + } holidays.sort() - if days == 0.0 { - return Number(Double::ceil(start_serial)) + let days_int = Double::to_int(trunc_double(days)) + if days_int == 0 { + return Number(Double::from_int(start_date)) } let (weekend_mask, workdays) = match weekend_mask_from_value(weekend_value) { Ok(value) => value @@ -6331,14 +6375,7 @@ fn workday_intl_value( if workdays == 0 { return Error(formula_error_value) } - let mut sign = 1 - if days < 0.0 { - sign = -1 - } - let days_int = Double::to_int(trunc_double(days)) - if days_int == 0 { - return Number(Double::ceil(start_serial)) - } + let sign = if days_int < 0 { -1 } else { 1 } let maximum_serial = maximum_supported_formula_date_serial(use_1904_dates) // Bound the arithmetic before multiplying the week offset by seven. Any // larger workday count must leave the supported 0000..9999 date domain, @@ -6348,7 +6385,7 @@ fn workday_intl_value( } let offset = days_int / workdays let mut days_mod = days_int % workdays - let mut end_date = Double::to_int(Double::ceil(start_serial)) + offset * 7 + let mut end_date = start_date + offset * 7 if !formula_date_serial_is_supported(end_date, use_1904_dates) { return Error(formula_error_num) } @@ -6388,7 +6425,7 @@ fn workday_intl_value( sign, holidays, weekend_mask, - start_serial, + start_date, use_1904_dates~, ) { Some(adjusted) => Number(Double::from_int(adjusted)) From 628c665c6bd79908acb735e65fd1070442e71b94 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 07:42:19 +0800 Subject: [PATCH 125/150] fix(xlsx): project DrawingML compatibility branches --- xlsx/read_drawing_xml.mbt | 75 ++++++++++++++++++++++++++++++++++-- xlsx/read_xml_namespaces.mbt | 48 ++++++++++++++++++++--- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 5762a98b..29103da9 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -581,9 +581,6 @@ fn parse_drawing_shapes( for entry in anchors { let (anchor_xml, positioning) = entry charge_drawing_anchor_pass(budget, anchor_xml) - if anchor_xml.contains(" value None => continue @@ -1017,6 +1014,78 @@ fn wb_rels_image_target(target : String) -> String { "" } +///| +test "DrawingML MCE fallback feeds image chart shape and slicer readers" { + let source = + #| + #| 0000 + #| 1000 + #| 2000 + #| 3000 + #| + let drawing = canonicalize_xlsx_drawing_xml( + source, default_max_xml_part_bytes, + ) + assert_false(drawing.contains("bad-image")) + assert_false(drawing.contains("chart-bad")) + assert_false(drawing.contains("bad-shape")) + assert_false(drawing.contains("slicer-bad")) + assert_false(drawing.contains(" + #| + #| + #| + let content_types = try! @ooxml.parse_package_content_types( + ( + #| + #| + #| + #| + ), + ) + let archive = @zip.Archive::new() + archive.add("xl/media/good.png", @encoding/utf8.encode("good-image")) + archive.add("xl/charts/good.xml", @encoding/utf8.encode("good-chart")) + let metrics = drawing_metrics_for_sheet(Worksheet::new("Sheet1")) + let images = parse_drawing_images( + anchors, + rels, + "xl/drawings/drawing1.xml", + Map([]), + content_types, + metrics, + archive, + ) + inspect(images.length(), content="1") + inspect(images[0].name, content="good-image") + let charts = parse_drawing_charts( + anchors, + rels, + "xl/drawings/drawing1.xml", + Map([]), + content_types, + metrics, + archive, + decode_utf8_or_invalid_xml, + ) + inspect(charts.length(), content="1") + inspect(charts[0].xml, content="good-chart") + let shapes = parse_drawing_shapes(anchors) + inspect(shapes.length(), content="1") + inspect(shapes[0].name, content="good-shape") + inspect(shapes[0].shape_type, content="rect") + let slicers = parse_drawing_slicer_anchors(anchors, metrics) + guard slicers.get("slicer-good") is Some(slicer) else { + fail("expected projected slicer fallback") + } + inspect(slicers.length(), content="1") + inspect(slicer.macro_name, content="good-macro") + inspect(slicer.alt_text, content="good-slicer") +} + ///| test "read_drawing_xml wb: parse_xml_int_tag missing/invalid errors" { let missing_tag = "1" diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index f442d9c7..0a967030 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -28,6 +28,12 @@ let strict_drawing_chart_namespace = "http://purl.oclc.org/ooxml/drawingml/chart ///| let drawing_slicer_namespace = "http://schemas.microsoft.com/office/drawing/2010/slicer" +///| +let drawing_main_2010_namespace = "http://schemas.microsoft.com/office/drawing/2010/main" + +///| +let drawing_slicer_2012_namespace = "http://schemas.microsoft.com/office/drawing/2012/slicer" + ///| let markup_compatibility_namespace = "http://schemas.openxmlformats.org/markup-compatibility/2006" @@ -43,6 +49,23 @@ let xlsx_mce_extension_elements : Array[(String, String)] = [ (strict_spreadsheet_namespace, "ext"), ] +///| +let xlsx_drawing_mce_understood_namespaces : Array[String] = [ + transitional_spreadsheet_drawing_namespace, strict_spreadsheet_drawing_namespace, + transitional_drawing_main_namespace, strict_drawing_main_namespace, transitional_drawing_chart_namespace, + strict_drawing_chart_namespace, drawing_slicer_namespace, transitional_relationship_attribute_namespace, + strict_relationship_attribute_namespace, drawing_main_2010_namespace, drawing_slicer_2012_namespace, +] + +///| +let xlsx_drawing_mce_extension_elements : Array[(String, String)] = [ + (transitional_drawing_main_namespace, "ext"), + (strict_drawing_main_namespace, "ext"), + (transitional_drawing_chart_namespace, "ext"), + (strict_drawing_chart_namespace, "ext"), + (drawing_slicer_namespace, "ext"), +] + ///| /// Produces the effective SpreadsheetML view before namespace filtering or /// lexical feature parsing. In particular, core content promoted from an MCE @@ -178,9 +201,12 @@ fn canonicalize_xlsx_worksheet_features( } ///| -/// Canonicalizes DrawingML by expanded namespace identity. Prefix aliases and -/// default namespaces are normalized to the lexical names used by the existing -/// drawing feature readers, while a foreign namespace borrowing `xdr`, `a`, +/// Produces the effective DrawingML view and canonicalizes it by expanded +/// namespace identity. MCE selection must run first, while `Choice.Requires` +/// prefixes still have their original namespace bindings; otherwise a lexical +/// feature reader could consume an unsupported branch ahead of its fallback. +/// Prefix aliases and default namespaces are then normalized to the names used +/// by the drawing readers, while a foreign namespace borrowing `xdr`, `a`, /// `c`, `sle`, or `mc` is defanged by the bounded XML canonicalizer. fn canonicalize_xlsx_drawing_xml( source : StringView, @@ -197,8 +223,18 @@ fn canonicalize_xlsx_drawing_xml( ) { raise InvalidXml(msg="drawing document element is invalid") } - @ooxml.canonicalize_xml_expanded_names( + let projected = @ooxml.project_xml_markup_compatibility( source, + xlsx_drawing_mce_understood_namespaces, + max_output_chars~, + cancelled~, + extension_elements=xlsx_drawing_mce_extension_elements, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + @ooxml.canonicalize_xml_expanded_names( + projected, [], [ (transitional_relationship_attribute_namespace, "r"), @@ -440,8 +476,8 @@ test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" assert_true(canonical.contains("")) assert_true(canonical.contains("")) assert_true(canonical.contains("")) - assert_true(canonical.contains("")) - assert_true(canonical.contains("")) + assert_false(canonical.contains(" let spoofed_canonical = canonicalize_xlsx_drawing_xml( From c6c202781c00b3a53448ef7dbbb1db48d0c90f11 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 07:46:08 +0800 Subject: [PATCH 126/150] fix(xlsx): validate resolved worksheet ownership --- xlsx/pkg.generated.mbti | 2 ++ xlsx/read.mbt | 2 ++ xlsx/workbook.mbt | 35 ++++++++++++++++++--- xlsx/workbook_types.mbt | 4 +++ xlsx/worksheet_cell_index.mbt | 59 +++++++++++++++++++++++++++++++++++ xlsx/worksheet_types.mbt | 4 +++ 6 files changed, 102 insertions(+), 4 deletions(-) diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index f2825603..765c1877 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1747,6 +1747,7 @@ pub fn TableOptions::set_style_name(Self, String) -> Unit pub fn TableOptions::to_repr(Self) -> @debug.Repr pub struct Workbook { + worksheet_owner_token : Array[Unit] sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] @@ -2052,6 +2053,7 @@ pub fn WorkbookProtectionOptions::to_repr(Self) -> @debug.Repr pub fn WorkbookProtectionOptions::with_values(algorithm_name? : String, password? : String, lock_structure? : Bool, lock_windows? : Bool) -> Self pub struct Worksheet { + mut workbook_owner_token : Array[Unit]? mut name : String sheet_views : Array[SheetView] mut dimension_ref : String? diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 4e1dfa65..33516f13 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -3870,6 +3870,7 @@ fn read_zip_archive_core_unchecked( None => () } let workbook : Workbook = { + worksheet_owner_token: [()], sheets: [], chart_sheets: [], sheet_order: [], @@ -4629,6 +4630,7 @@ fn read_zip_archive_core_unchecked( } let cell_index = build_bounded_worksheet_cell_index(cells, read_budget) let sheet = { + workbook_owner_token: Some(workbook.worksheet_owner_token), name, sheet_views, dimension_ref, diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 53a354b5..1d2564b4 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -16,6 +16,7 @@ pub fn Workbook::add_sheet( None => () } let sheet = Worksheet::new(name) + sheet.workbook_owner_token = Some(self.worksheet_owner_token) self.sheets.push(sheet) self.sheet_order.push(Worksheet(self.sheets.length() - 1)) if self.sheet_order.length() == 1 { @@ -108,6 +109,22 @@ fn Workbook::require_sheet( } } +///| +/// Rejects detached or foreign worksheet handles before workbook-owned styles, +/// options, or date-system settings are applied to their cells. +fn Workbook::require_owned_worksheet( + self : Workbook, + worksheet : Worksheet, +) -> Unit raise XlsxError { + match worksheet.workbook_owner_token { + Some(owner) if physical_equal(owner, self.worksheet_owner_token) => () + _ => + raise InvalidSheetOperation( + msg="worksheet does not belong to this workbook", + ) + } +} + ///| pub fn Workbook::chart_sheet(self : Workbook, name : StringView) -> ChartSheet? { let needle = normalize_sheet_name(name) @@ -492,6 +509,7 @@ pub fn Workbook::delete_sheet( ignore(self.sheet_order.remove(order_index)) match entry { Worksheet(idx) => { + self.sheets[idx].workbook_owner_token = None ignore(self.sheets.remove(idx)) let updated : Array[SheetEntry] = [] for item in self.sheet_order { @@ -605,6 +623,8 @@ pub fn Workbook::copy_sheet( to_sheet.name(), to_sheet.state(), ) + to_sheet.workbook_owner_token = None + cloned.workbook_owner_token = Some(self.worksheet_owner_token) self.sheets[to_sheet_idx] = cloned } _ => raise InvalidSheetIndex(index=to_index) @@ -1315,6 +1335,7 @@ fn clone_worksheet( ) -> Worksheet { let cells = clone_cells(sheet.cells) { + workbook_owner_token: None, name, sheet_views: clone_sheet_views(sheet.sheet_views), dimension_ref: sheet.dimension_ref, @@ -1845,8 +1866,10 @@ pub fn Workbook::get_cell_value( /// workbook's formatting options have the same semantics as /// `get_cell_value`. /// -/// `worksheet` must be an entry from this workbook's `sheets()` view. The row -/// and column are 1-based and must fit the XLSX grid. +/// `worksheet` must be a live entry from this workbook's `sheets()` view; +/// detached, deleted, replaced, or foreign handles raise +/// `InvalidSheetOperation`. The row and column are 1-based and must fit the +/// XLSX grid. pub fn Workbook::get_cell_value_from_worksheet_rc( self : Workbook, worksheet : Worksheet, @@ -1855,6 +1878,7 @@ pub fn Workbook::get_cell_value_from_worksheet_rc( raw? : Bool = false, options? : Options, ) -> String? raise XlsxError { + self.require_owned_worksheet(worksheet) ignore(cell_ref_from(row, col)) let resolved_options = match options { Some(value) => value @@ -1881,8 +1905,10 @@ pub fn Workbook::get_cell_value_from_worksheet_rc( /// lets callers apply row/column-inherited styles without repeating a sheet /// name lookup for every coordinate. Returns `None` for an absent cell. /// -/// `worksheet` must be an entry from this workbook's `sheets()` view. The row -/// and column are 1-based and `style_id` must belong to this workbook. +/// `worksheet` must be a live entry from this workbook's `sheets()` view; +/// detached, deleted, replaced, or foreign handles raise +/// `InvalidSheetOperation`. The row and column are 1-based and `style_id` must +/// belong to this workbook. pub fn Workbook::get_cell_value_styled_from_worksheet_rc( self : Workbook, worksheet : Worksheet, @@ -1892,6 +1918,7 @@ pub fn Workbook::get_cell_value_styled_from_worksheet_rc( options? : Options, max_output_chars? : Int, ) -> String? raise XlsxError { + self.require_owned_worksheet(worksheet) ignore(cell_ref_from(row, col)) self.check_style_id(style_id) let resolved_options = match options { diff --git a/xlsx/workbook_types.mbt b/xlsx/workbook_types.mbt index 2dc4799f..2cab967d 100644 --- a/xlsx/workbook_types.mbt +++ b/xlsx/workbook_types.mbt @@ -1,5 +1,8 @@ ///| pub struct Workbook { + /// Opaque identity shared with worksheets currently owned by this workbook. + /// A non-empty array is used solely to provide stable reference identity. + worksheet_owner_token : Array[Unit] sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] @@ -159,6 +162,7 @@ fn defined_name_scope_key(scope : StringView) -> String { ///| pub fn Workbook::new(options? : Options = Options::new()) -> Workbook { { + worksheet_owner_token: [()], sheets: [], chart_sheets: [], sheet_order: [], diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index 6b74fe61..9bd6127a 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -141,6 +141,65 @@ test "resolved worksheet reads avoid repeated names and preserve style precedenc assert_eq(sheet.effective_style_id_rc(3, 2), 0) } +///| +fn expect_unowned_worksheet_error( + action : () -> Unit raise XlsxError, +) -> Unit raise { + try action() catch { + InvalidSheetOperation(msg~) => + inspect(msg, content="worksheet does not belong to this workbook") + error => fail("unexpected worksheet ownership error: \{repr(error)}") + } noraise { + _ => fail("expected worksheet ownership error") + } +} + +///| +test "resolved worksheet reads reject foreign detached and stale handles" { + let owner = Workbook::new() + let owned = owner.add_sheet("Same") + owned.set_cell_value_rc(1, 1, Numeric(1.5)) + let style = owner.add_style(Style::builtin_number_format(2)) + owned.set_cell_style_rc(1, 1, style) + + let foreign_workbook = Workbook::new() + let foreign = foreign_workbook.add_sheet("Same") + foreign.set_cell_value_rc(1, 1, Numeric(9.5)) + expect_unowned_worksheet_error(() => { + ignore(owner.get_cell_value_from_worksheet_rc(foreign, 1, 1)) + }) + expect_unowned_worksheet_error(() => { + ignore(owner.get_cell_value_styled_from_worksheet_rc(foreign, 1, 1, style)) + }) + + let detached = Worksheet::new("Same") + expect_unowned_worksheet_error(() => { + ignore(owner.get_cell_value_from_worksheet_rc(detached, 1, 1)) + }) + + ignore(owner.add_sheet("Keep")) + let stale_deleted = owner.add_sheet("Deleted") + owner.delete_sheet("Deleted") + expect_unowned_worksheet_error(() => { + ignore(owner.get_cell_value_from_worksheet_rc(stale_deleted, 1, 1)) + }) + + let stale_replaced = owner.add_sheet("Destination") + let source_index = owner.sheet_index("Same") + let destination_index = owner.sheet_index("Destination") + owner.copy_sheet(source_index, destination_index) + expect_unowned_worksheet_error(() => { + ignore(owner.get_cell_value_from_worksheet_rc(stale_replaced, 1, 1)) + }) + guard owner.sheet("Destination") is Some(replacement) else { + fail("expected copied destination worksheet") + } + assert_eq( + owner.get_cell_value_from_worksheet_rc(replacement, 1, 1), + Some("1.50"), + ) +} + ///| test "parsed worksheets leave coordinate indexes ready for bounded reads" { let workbook = Workbook::new() diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 89fe13b8..8d0ecc8f 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -92,6 +92,9 @@ pub struct ColDimension { ///| pub struct Worksheet { + /// Present only while this worksheet is a live member of a workbook. The + /// token supports constant-time ownership checks for bulk workbook reads. + mut workbook_owner_token : Array[Unit]? mut name : String sheet_views : Array[SheetView] mut dimension_ref : String? @@ -142,6 +145,7 @@ pub struct Worksheet { ///| pub fn Worksheet::new(name : String) -> Worksheet { { + workbook_owner_token: None, name, sheet_views: [], dimension_ref: None, From 7727742057b1108df379c8d4394df573fda3a420 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 08:23:54 +0800 Subject: [PATCH 127/150] fix(xlsx): hide worksheet ownership tokens --- xlsx/pkg.generated.mbti | 4 ++-- xlsx/workbook_types.mbt | 2 +- xlsx/worksheet_types.mbt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 765c1877..d3a4c2af 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1747,7 +1747,6 @@ pub fn TableOptions::set_style_name(Self, String) -> Unit pub fn TableOptions::to_repr(Self) -> @debug.Repr pub struct Workbook { - worksheet_owner_token : Array[Unit] sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] @@ -1775,6 +1774,7 @@ pub struct Workbook { cell_images : Array[CellImage] rich_value_images : RichValueImages? rich_value_media : Map[String, Bytes] + // private fields } pub fn Workbook::active_sheet_index(Self) -> Int pub fn Workbook::add_chart(Self, StringView, String, String) -> Unit raise XlsxError @@ -2053,7 +2053,6 @@ pub fn WorkbookProtectionOptions::to_repr(Self) -> @debug.Repr pub fn WorkbookProtectionOptions::with_values(algorithm_name? : String, password? : String, lock_structure? : Bool, lock_windows? : Bool) -> Self pub struct Worksheet { - mut workbook_owner_token : Array[Unit]? mut name : String sheet_views : Array[SheetView] mut dimension_ref : String? @@ -2096,6 +2095,7 @@ pub struct Worksheet { mut vml_drawing_xml : String? mut vml_drawing_hf_xml : String? cell_vm : Map[String, Int] + // private fields } pub fn Worksheet::add_chart(Self, String, String) -> Unit raise XlsxError pub fn Worksheet::add_chart_with_options(Self, String, ChartOptions) -> Unit raise XlsxError diff --git a/xlsx/workbook_types.mbt b/xlsx/workbook_types.mbt index 2cab967d..a5945687 100644 --- a/xlsx/workbook_types.mbt +++ b/xlsx/workbook_types.mbt @@ -2,7 +2,7 @@ pub struct Workbook { /// Opaque identity shared with worksheets currently owned by this workbook. /// A non-empty array is used solely to provide stable reference identity. - worksheet_owner_token : Array[Unit] + priv worksheet_owner_token : Array[Unit] sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 8d0ecc8f..48a7dc68 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -94,7 +94,7 @@ pub struct ColDimension { pub struct Worksheet { /// Present only while this worksheet is a live member of a workbook. The /// token supports constant-time ownership checks for bulk workbook reads. - mut workbook_owner_token : Array[Unit]? + priv mut workbook_owner_token : Array[Unit]? mut name : String sheet_views : Array[SheetView] mut dimension_ref : String? From adf411a018d2631f1943ed3ccb481147b89b388f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 08:39:56 +0800 Subject: [PATCH 128/150] fix(xlsx): isolate drawing extension payloads --- xlsx/read_drawing_xml.mbt | 44 +++++++++++++++++++++++ xlsx/read_xml_namespaces.mbt | 69 +++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 29103da9..3a968cb2 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -1086,6 +1086,50 @@ test "DrawingML MCE fallback feeds image chart shape and slicer readers" { inspect(slicer.alt_text, content="good-slicer") } +///| +test "DrawingML extension payload cannot spoof anchors or drawing features" { + let source = + #| + #| 999900 + #| 0000 + #| 1000 + #| 2000 + #| 3000 + #| + let drawing = canonicalize_xlsx_drawing_xml( + source, default_max_xml_part_bytes, + ) + assert_false(drawing.contains("bad-anchor")) + assert_false(drawing.contains("bad-image")) + assert_false(drawing.contains("chart-bad")) + assert_false(drawing.contains("bad-shape")) + assert_false(drawing.contains("slicer-bad")) + assert_false(drawing.contains("extLst")) + let anchors = parse_drawing_anchors(drawing) + inspect(anchors.length(), content="4") + let (image_anchor, _) = anchors[0] + let (chart_anchor, _) = anchors[1] + let (shape_anchor, _) = anchors[2] + let (slicer_anchor, _) = anchors[3] + assert_true(image_anchor.contains("image-good")) + assert_true(chart_anchor.contains("chart-good")) + assert_true(shape_anchor.contains("good-shape")) + assert_true(slicer_anchor.contains("slicer-good")) +} + +///| +test "DrawingML rejects extension entries outside extension lists" { + let source = + #| + try canonicalize_xlsx_drawing_xml(source, default_max_xml_part_bytes) catch { + InvalidXml(msg~) => + inspect(msg, content="drawing extension payload is misplaced") + _ => fail("unexpected misplaced extension error") + } noraise { + _ => fail("expected a misplaced extension payload error") + } +} + ///| test "read_drawing_xml wb: parse_xml_int_tag missing/invalid errors" { let missing_tag = "1" diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 0a967030..8f809def 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -66,6 +66,21 @@ let xlsx_drawing_mce_extension_elements : Array[(String, String)] = [ (drawing_slicer_namespace, "ext"), ] +///| +let xlsx_drawing_extension_list_namespaces : Array[String] = [ + transitional_spreadsheet_drawing_namespace, strict_spreadsheet_drawing_namespace, + transitional_drawing_main_namespace, strict_drawing_main_namespace, transitional_drawing_chart_namespace, + strict_drawing_chart_namespace, drawing_slicer_namespace, drawing_main_2010_namespace, + drawing_slicer_2012_namespace, +] + +///| +let xlsx_drawing_core_element_namespaces : Array[String] = [ + transitional_spreadsheet_drawing_namespace, strict_spreadsheet_drawing_namespace, + transitional_drawing_main_namespace, strict_drawing_main_namespace, transitional_drawing_chart_namespace, + strict_drawing_chart_namespace, drawing_slicer_namespace, +] + ///| /// Produces the effective SpreadsheetML view before namespace filtering or /// lexical feature parsing. In particular, core content promoted from an MCE @@ -223,8 +238,20 @@ fn canonicalize_xlsx_drawing_xml( ) { raise InvalidXml(msg="drawing document element is invalid") } - let projected = @ooxml.project_xml_markup_compatibility( + // Application extension payload is deliberately absent from the lexical + // reader view. Removing its structural container before MCE projection also + // prevents directives inside opaque payload from affecting core selection. + let without_extension_lists = @ooxml.remove_xml_expanded_name_subtrees( source, + xlsx_drawing_extension_list_namespaces, + "extLst", + cancelled~, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + let projected = @ooxml.project_xml_markup_compatibility( + without_extension_lists.unwrap_or(source.to_owned()), xlsx_drawing_mce_understood_namespaces, max_output_chars~, cancelled~, @@ -233,8 +260,40 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } - @ooxml.canonicalize_xml_expanded_names( + // A schema-invalid extension entry outside an extLst must not become an + // alternate route into the first-match drawing readers. Geometry extents + // are empty elements too, so accepting explicit empty start/end spellings + // remains interoperable while descendant markup is rejected. + let projected_scanner = @ooxml.XmlStartTagScanner::new(projected, cancelled~) + let mut extension_depth : Int? = None + while workbook_scanner_next(projected_scanner) { + match extension_depth { + Some(depth) if projected_scanner.depth() > depth => + raise InvalidXml(msg="drawing extension payload is misplaced") + Some(_) => extension_depth = None + None => () + } + if projected_scanner.local_name() == "ext" { + let namespace_uri = projected_scanner.namespace_uri().to_owned() + if xlsx_drawing_extension_list_namespaces.any(candidate => { + candidate == namespace_uri + }) { + extension_depth = Some(projected_scanner.depth()) + } + } + } + // Foreign wrappers are removed with their descendants. Merely renaming a + // wrapper would leave nested core-looking tags visible to lexical readers. + let core_projected = @ooxml.remove_xml_foreign_namespace_subtrees( projected, + xlsx_drawing_core_element_namespaces, + cancelled~, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + @ooxml.canonicalize_xml_expanded_names( + core_projected.unwrap_or(projected), [], [ (transitional_relationship_attribute_namespace, "r"), @@ -466,7 +525,7 @@ test "XLSX XML canonicalization follows extension namespace identity" { } ///| -test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" { +test "DrawingML canonicalization follows aliases and removes foreign subtrees" { let valid = #| let canonical = canonicalize_xlsx_drawing_xml( @@ -483,8 +542,8 @@ test "DrawingML canonicalization follows aliases and defangs borrowed prefixes" let spoofed_canonical = canonicalize_xlsx_drawing_xml( spoofed, default_max_xml_part_bytes, ) - assert_true(spoofed_canonical.contains("<_foreign_xdr_oneCellAnchor ")) - assert_true(spoofed_canonical.contains("<_foreign_mc_AlternateContent/>")) + assert_false(spoofed_canonical.contains("oneCellAnchor")) + assert_false(spoofed_canonical.contains("AlternateContent")) assert_false(spoofed_canonical.contains(" Date: Sun, 19 Jul 2026 08:41:50 +0800 Subject: [PATCH 129/150] fix(office): enforce XLSX scan ceiling upfront --- office/cmd/office/xlsx_read_model.mbt | 31 ++++++++++++-------- office/cmd/office/xlsx_read_wbtest.mbt | 40 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/office/cmd/office/xlsx_read_model.mbt b/office/cmd/office/xlsx_read_model.mbt index 67efd4d1..ae1876df 100644 --- a/office/cmd/office/xlsx_read_model.mbt +++ b/office/cmd/office/xlsx_read_model.mbt @@ -217,6 +217,21 @@ fn canonical_xlsx_range_path( } } +///| +fn validate_xlsx_max_elements(max_elements : Int) -> Unit raise CliFailure { + if max_elements < 1 || max_elements > xlsx_cli_hard_max_scan_cells { + raise xlsx_cli_failure( + "office.invalid_arguments", + "--max-elements must be between 1 and \{xlsx_cli_hard_max_scan_cells}", + details=Json::object({ + "argument": Json::string("max-elements"), + "minimum": Json::number(1), + "maximum": Json::number(xlsx_cli_hard_max_scan_cells.to_double()), + }), + ) + } +} + ///| fn make_xlsx_projection( file : String, @@ -227,17 +242,7 @@ fn make_xlsx_projection( if cancelled() { raise xlsx_cli_failure("office.cancelled", "XLSX read was cancelled") } - if max_elements < 1 || max_elements > docx_cli_hard_max_elements { - raise xlsx_cli_failure( - "office.invalid_arguments", - "--max-elements must be between 1 and \{docx_cli_hard_max_elements}", - details=Json::object({ - "argument": Json::string("max-elements"), - "minimum": Json::number(1), - "maximum": Json::number(docx_cli_hard_max_elements.to_double()), - }), - ) - } + validate_xlsx_max_elements(max_elements) let worksheets : Map[String, @xlsx.Worksheet] = Map([]) for worksheet in workbook.sheets() { if cancelled() { @@ -295,7 +300,7 @@ fn make_xlsx_projection( workbook, sheets, sheet_index, - max_scan_cells: max_elements.min(xlsx_cli_hard_max_scan_cells), + max_scan_cells: max_elements, cancelled, } } @@ -306,6 +311,8 @@ fn open_xlsx_projection( max_elements : Int, cancelled? : () -> Bool = () => false, ) -> XlsxProjection raise { + // Reject format-specific limits before parsing any workbook parts. + validate_xlsx_max_elements(max_elements) let workbook = open_xlsx_read_package(source, cancelled~) make_xlsx_projection(source.file, workbook, max_elements, cancelled~) } diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index c9bd5b6a..13f0da48 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -39,6 +39,46 @@ async fn xlsx_text_test_data( xlsx_text_payload(projection, under, offset, limit, 100_000).data } +///| +test "XLSX projection enforces its documented scan ceiling" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Data")) + let error = xlsx_cli_error(() => { + ignore( + make_xlsx_projection( + "oversized.xlsx", + workbook, + xlsx_cli_hard_max_scan_cells + 1, + ), + ) + }) + assert_eq(error.code, "office.invalid_arguments") + assert_eq( + error.message, + "--max-elements must be between 1 and \{xlsx_cli_hard_max_scan_cells}", + ) + guard error.details is Some(Object(details)) else { + fail("expected XLSX max-elements details") + } + assert_eq( + details.get("maximum"), + Some(Json::number(xlsx_cli_hard_max_scan_cells.to_double())), + ) + let malformed : OfficeReadPackage = { + file: "malformed.xlsx", + format: Xlsx, + archive: @zip.Archive::new(), + } + let upfront = xlsx_cli_error(() => { + ignore(open_xlsx_projection(malformed, xlsx_cli_hard_max_scan_cells + 1)) + }) + assert_eq(upfront.code, "office.invalid_arguments") + let projection = make_xlsx_projection( + "limit.xlsx", workbook, xlsx_cli_hard_max_scan_cells, + ) + assert_eq(projection.max_scan_cells, xlsx_cli_hard_max_scan_cells) +} + ///| async fn xlsx_query_test_data( projection : XlsxProjection, From c68e82305c6285f9e545d8348d9b13b3f7e971e8 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 09:37:48 +0800 Subject: [PATCH 130/150] fix(xlsx): seal worksheet membership --- xlsx/pkg.generated.mbti | 1 - xlsx/workbook_types.mbt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index d3a4c2af..97813bc8 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1747,7 +1747,6 @@ pub fn TableOptions::set_style_name(Self, String) -> Unit pub fn TableOptions::to_repr(Self) -> @debug.Repr pub struct Workbook { - sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] styles : Array[Style] diff --git a/xlsx/workbook_types.mbt b/xlsx/workbook_types.mbt index a5945687..cc27b9bd 100644 --- a/xlsx/workbook_types.mbt +++ b/xlsx/workbook_types.mbt @@ -3,7 +3,7 @@ pub struct Workbook { /// Opaque identity shared with worksheets currently owned by this workbook. /// A non-empty array is used solely to provide stable reference identity. priv worksheet_owner_token : Array[Unit] - sheets : Array[Worksheet] + priv sheets : Array[Worksheet] chart_sheets : Array[ChartSheet] sheet_order : Array[SheetEntry] styles : Array[Style] From 32236d84127102a4def65d519561d86f63d85dc5 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 09:40:55 +0800 Subject: [PATCH 131/150] fix(office): reject XLSX scan limits before I/O --- office/cmd/office/cram/cli.t | 13 +++++++++++++ office/cmd/office/docx_commands.mbt | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/office/cmd/office/cram/cli.t b/office/cmd/office/cram/cli.t index 15bba7c0..ab39dd81 100644 --- a/office/cmd/office/cram/cli.t +++ b/office/cmd/office/cram/cli.t @@ -120,6 +120,19 @@ syntax and missing paths retain stable machine-readable codes. $ jq -c '{success,code:.error.code,resource:.error.details.resource,limit:.error.details.limit}' output-limit.json {"success":false,"code":"office.docx.resource_limit","resource":"successful command output characters","limit":40} +The XLSX-specific scan ceiling is rejected from the filename before package +I/O or parsing. A malformed package therefore cannot mask invalid arguments, +while the larger DOCX ceiling remains available. + + $ printf 'not a zip' > malformed.xlsx + $ office.exe outline malformed.xlsx --max-elements 100001 --json > xlsx-preflight.json 2>&1; echo $? + 1 + $ jq -c '{success,code:.error.code,maximum:.error.details.maximum}' xlsx-preflight.json + {"success":false,"code":"office.invalid_arguments","maximum":100000} + + $ office.exe outline "$TESTDIR/../../../../docx2html/tests/cram/fixtures/single-paragraph.docx" --max-elements 100001 --json | jq -c '{success,schema:.data.schema}' + {"success":true,"schema":"office.docx.outline/1"} + Structured XLSX reads use the same commands and envelope. Positional sheet input canonicalizes to stable name paths; ranges, text, and query scan in tab/row/column order with exact totals. diff --git a/office/cmd/office/docx_commands.mbt b/office/cmd/office/docx_commands.mbt index 0b3d9927..7dd290c7 100644 --- a/office/cmd/office/docx_commands.mbt +++ b/office/cmd/office/docx_commands.mbt @@ -69,6 +69,20 @@ fn cli_max_elements(matches : @argparse.Matches) -> Int raise CliFailure { ) } +///| +/// Applies the tighter XLSX scan ceiling before opening or parsing the input +/// package. Content-based format validation still happens after the bounded +/// read, but an `.xlsx` path must never make an oversized request pay that +/// cost first. DOCX retains its independently documented 200,000-node limit. +fn validate_xlsx_max_elements_before_read( + file : String, + max_elements : Int, +) -> Unit raise CliFailure { + if file.to_lower().has_suffix(".xlsx") { + validate_xlsx_max_elements(max_elements) + } +} + ///| fn cli_max_output_chars(matches : @argparse.Matches) -> Int raise CliFailure { bounded_decimal_argument( @@ -131,6 +145,7 @@ fn resolve_optional_xlsx_under( async fn run_docx_outline(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let max_elements = cli_max_elements(matches) + validate_xlsx_max_elements_before_read(file, max_elements) let max_output = cli_max_output_chars(matches) let source = read_office_package(file, cancelled=() => { @async.is_being_cancelled() @@ -186,6 +201,7 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let selector = required_value(matches, "selector") let max_elements = cli_max_elements(matches) + validate_xlsx_max_elements_before_read(file, max_elements) let max_output = cli_max_output_chars(matches) let source = read_office_package(file, cancelled=() => { @async.is_being_cancelled() @@ -256,6 +272,7 @@ async fn run_docx_get(matches : @argparse.Matches) -> Unit { async fn run_docx_text(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let max_elements = cli_max_elements(matches) + validate_xlsx_max_elements_before_read(file, max_elements) let max_output = cli_max_output_chars(matches) let offset = bounded_decimal_argument( matches, "offset", 0, 0, docx_cli_hard_max_elements, @@ -525,6 +542,7 @@ fn reject_docx_content_selector( async fn run_docx_query(matches : @argparse.Matches) -> Unit { let file = required_value(matches, "file") let max_elements = cli_max_elements(matches) + validate_xlsx_max_elements_before_read(file, max_elements) let max_output = cli_max_output_chars(matches) let offset = bounded_decimal_argument( matches, "offset", 0, 0, docx_cli_hard_max_elements, From 260b08b1b930d1bf6406907f55103ecfaa720f42 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 09:45:54 +0800 Subject: [PATCH 132/150] fix(xlsx): validate drawing reader paths --- xlsx/read_drawing_xml.mbt | 63 +++++++++++++ xlsx/read_xml_namespaces.mbt | 166 ++++++++++++++++++++++++++++++----- 2 files changed, 205 insertions(+), 24 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 3a968cb2..6ad2130e 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -1130,6 +1130,69 @@ test "DrawingML rejects extension entries outside extension lists" { } } +///| +test "DrawingML rejects supported-namespace reader decoys" { + let cases : Array[(String, String)] = [ + ( + ( + #| + ), + "drawing anchor is misplaced", + ), + ( + ( + #| + ), + "drawing reader object is misplaced", + ), + ( + ( + #| + ), + "drawing reader object is misplaced", + ), + ( + ( + #| + ), + "drawing chart graphicData URI is invalid", + ), + ( + ( + #| + ), + "drawing slicer graphicData URI is invalid", + ), + ( + ( + #| + ), + "drawing anchor has multiple reader objects", + ), + ] + for case in cases { + let (source, expected) = case + try + canonicalize_xlsx_drawing_xml(source, default_max_xml_part_bytes) + catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected drawing structure error") + } noraise { + _ => fail("expected supported-namespace drawing decoy rejection") + } + } +} + +///| +test "DrawingML validates Strict chart dialect and payload URI together" { + let source = + #| + let canonical = canonicalize_xlsx_drawing_xml( + source, default_max_xml_part_bytes, + ) + assert_true(canonical.contains("")) +} + ///| test "read_drawing_xml wb: parse_xml_int_tag missing/invalid errors" { let missing_tag = "1" diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 8f809def..e3607ea6 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -81,6 +81,143 @@ let xlsx_drawing_core_element_namespaces : Array[String] = [ strict_drawing_chart_namespace, drawing_slicer_namespace, ] +///| +fn xlsx_drawing_anchor_name(local_name : StringView) -> Bool { + local_name == "oneCellAnchor" || local_name == "twoCellAnchor" +} + +///| +fn xlsx_drawing_reader_object_name(local_name : StringView) -> Bool { + local_name == "pic" || local_name == "sp" || local_name == "graphicFrame" +} + +///| +/// Proves that every element consumed by a namespace-oblivious drawing reader +/// occurs on the schema path that reader assumes. Merely filtering namespaces +/// is insufficient: a supported-namespace decoy nested in `graphicData`, a +/// group, or another anchor would otherwise win the readers' first-match +/// lexical searches. +fn validate_xlsx_drawing_reader_structure( + source : StringView, + cancelled? : () -> Bool = () => false, +) -> Unit raise XlsxError { + let scanner = @ooxml.XmlStartTagScanner::new(source, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != "wsDr" || + ( + scanner.namespace_uri() != transitional_spreadsheet_drawing_namespace && + scanner.namespace_uri() != strict_spreadsheet_drawing_namespace + ) { + raise InvalidXml(msg="drawing document element is invalid") + } + let drawing_namespace = scanner.namespace_uri().to_owned() + let (main_namespace, chart_namespace) = if drawing_namespace == + transitional_spreadsheet_drawing_namespace { + (transitional_drawing_main_namespace, transitional_drawing_chart_namespace) + } else { + (strict_drawing_main_namespace, strict_drawing_chart_namespace) + } + let mut in_reader_anchor = false + let mut reader_object_seen = false + let mut reader_payload_seen = false + let mut graphic_data_uri : String? = None + let mut extension_depth : Int? = None + while workbook_scanner_next(scanner) { + let depth = scanner.depth() + let local_name = scanner.local_name() + let namespace_uri = scanner.namespace_uri().to_owned() + if depth <= 2 { + in_reader_anchor = false + reader_object_seen = false + reader_payload_seen = false + } + if depth <= 5 { + graphic_data_uri = None + } + match extension_depth { + Some(extension_start) if depth > extension_start => + raise InvalidXml(msg="drawing extension payload is misplaced") + Some(_) => extension_depth = None + None => () + } + if local_name == "ext" && + xlsx_drawing_extension_list_namespaces.any(candidate => { + candidate == namespace_uri + }) { + extension_depth = Some(depth) + } + if namespace_uri == drawing_namespace && + xlsx_drawing_anchor_name(local_name) { + if depth != 2 || + scanner.parent_local_name() != Some("wsDr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing anchor is misplaced") + } + in_reader_anchor = true + reader_object_seen = false + reader_payload_seen = false + } else if namespace_uri == drawing_namespace && + xlsx_drawing_reader_object_name(local_name) { + if depth != 3 || + !in_reader_anchor || + !xlsx_drawing_anchor_name(scanner.parent_local_name().unwrap_or("")) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing reader object is misplaced") + } + if reader_object_seen { + raise InvalidXml(msg="drawing anchor has multiple reader objects") + } + reader_object_seen = true + } else if namespace_uri == main_namespace && local_name == "graphic" { + if depth != 4 || + scanner.parent_local_name() != Some("graphicFrame") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing graphic element is misplaced") + } + } else if namespace_uri == main_namespace && local_name == "graphicData" { + if depth != 5 || + scanner.parent_local_name() != Some("graphic") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing graphicData element is misplaced") + } + graphic_data_uri = workbook_scanner_attribute(scanner, "", "uri") + } else if ( + namespace_uri == transitional_drawing_chart_namespace || + namespace_uri == strict_drawing_chart_namespace + ) && + local_name == "chart" { + if namespace_uri != chart_namespace || + depth != 6 || + scanner.parent_local_name() != Some("graphicData") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing chart payload is misplaced") + } + if graphic_data_uri != Some(chart_namespace) { + raise InvalidXml(msg="drawing chart graphicData URI is invalid") + } + if reader_payload_seen { + raise InvalidXml(msg="drawing anchor has multiple reader payloads") + } + reader_payload_seen = true + } else if namespace_uri == drawing_slicer_namespace && + local_name == "slicer" { + if depth != 6 || + scanner.parent_local_name() != Some("graphicData") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing slicer payload is misplaced") + } + if graphic_data_uri != Some(drawing_slicer_namespace) { + raise InvalidXml(msg="drawing slicer graphicData URI is invalid") + } + if reader_payload_seen { + raise InvalidXml(msg="drawing anchor has multiple reader payloads") + } + reader_payload_seen = true + } + } +} + ///| /// Produces the effective SpreadsheetML view before namespace filtering or /// lexical feature parsing. In particular, core content promoted from an MCE @@ -260,28 +397,6 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } - // A schema-invalid extension entry outside an extLst must not become an - // alternate route into the first-match drawing readers. Geometry extents - // are empty elements too, so accepting explicit empty start/end spellings - // remains interoperable while descendant markup is rejected. - let projected_scanner = @ooxml.XmlStartTagScanner::new(projected, cancelled~) - let mut extension_depth : Int? = None - while workbook_scanner_next(projected_scanner) { - match extension_depth { - Some(depth) if projected_scanner.depth() > depth => - raise InvalidXml(msg="drawing extension payload is misplaced") - Some(_) => extension_depth = None - None => () - } - if projected_scanner.local_name() == "ext" { - let namespace_uri = projected_scanner.namespace_uri().to_owned() - if xlsx_drawing_extension_list_namespaces.any(candidate => { - candidate == namespace_uri - }) { - extension_depth = Some(projected_scanner.depth()) - } - } - } // Foreign wrappers are removed with their descendants. Merely renaming a // wrapper would leave nested core-looking tags visible to lexical readers. let core_projected = @ooxml.remove_xml_foreign_namespace_subtrees( @@ -292,8 +407,10 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } + let reader_source = core_projected.unwrap_or(projected) + validate_xlsx_drawing_reader_structure(reader_source, cancelled~) @ooxml.canonicalize_xml_expanded_names( - core_projected.unwrap_or(projected), + reader_source, [], [ (transitional_relationship_attribute_namespace, "r"), @@ -527,12 +644,13 @@ test "XLSX XML canonicalization follows extension namespace identity" { ///| test "DrawingML canonicalization follows aliases and removes foreign subtrees" { let valid = - #| + #| let canonical = canonicalize_xlsx_drawing_xml( valid, default_max_xml_part_bytes, ) assert_true(canonical.contains("")) + assert_true(canonical.contains("")) assert_true(canonical.contains("")) assert_true(canonical.contains("")) assert_false(canonical.contains(" Date: Sun, 19 Jul 2026 09:48:14 +0800 Subject: [PATCH 133/150] fix(xlsx): meter DrawingML preprocessing --- xlsx/read.mbt | 4 +- xlsx/read_drawing_xml.mbt | 24 ++++++++++ xlsx/read_xml_namespaces.mbt | 88 +++++++++++++++++++++++++++++++----- 3 files changed, 102 insertions(+), 14 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 33516f13..325b55a4 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4712,14 +4712,12 @@ fn read_zip_archive_core_unchecked( None => raise MissingPart(path=drawing_path) } let drawing_source_xml = decode_source(drawing_bytes) - read_budget.charge_work(drawing_source_xml.length()) let drawing_xml = canonicalize_xlsx_drawing_xml( drawing_source_xml, limits.max_xml_part_bytes, + budget=read_budget, cancelled~, ) - read_budget.checkpoint() - read_budget.charge_work(drawing_xml.length()) let drawing_metrics = drawing_metrics_for_sheet(sheet) let drawing_anchors = parse_drawing_anchors( drawing_xml, diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 6ad2130e..15716c29 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -1193,6 +1193,30 @@ test "DrawingML validates Strict chart dialect and payload URI together" { assert_true(canonical.contains("")) } +///| +test "DrawingML preprocessing charges every bounded pass" { + let source = + #| + // Charging only the source and final output would fit this ceiling. The + // independent root-validation, extension-removal, and MCE passes must not. + let limit = source.length() * 2 + let budget = ReadBudget::new( + ReadLimits::with_values(max_parser_work_units=limit), + ) + try + canonicalize_xlsx_drawing_xml(source, default_max_xml_part_bytes, budget~) + catch { + ResourceLimitExceeded(kind~, limit=actual_limit, actual~) => { + assert_eq(kind, "parser_work_units") + assert_eq(actual_limit, limit) + assert_true(actual > actual_limit) + } + _ => fail("unexpected DrawingML preprocessing limit error") + } noraise { + _ => fail("expected DrawingML preprocessing work rejection") + } +} + ///| test "read_drawing_xml wb: parse_xml_int_tag missing/invalid errors" { let missing_tag = "1" diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index e3607ea6..c3c6fc40 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -352,6 +352,34 @@ fn canonicalize_xlsx_worksheet_features( ) } +///| +fn charge_xlsx_drawing_xml_pass( + budget : ReadBudget?, + xml : StringView, +) -> Unit raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => () + } +} + +///| +fn charge_xlsx_drawing_xml_materialization( + budget : ReadBudget?, + xml : StringView, +) -> Unit raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => () + } +} + ///| /// Produces the effective DrawingML view and canonicalizes it by expanded /// namespace identity. MCE selection must run first, while `Choice.Requires` @@ -363,8 +391,10 @@ fn canonicalize_xlsx_worksheet_features( fn canonicalize_xlsx_drawing_xml( source : StringView, max_output_chars : Int, + budget? : ReadBudget, cancelled? : () -> Bool = () => false, ) -> String raise XlsxError { + charge_xlsx_drawing_xml_pass(budget, source) let scanner = @ooxml.XmlStartTagScanner::new(source, cancelled~) if !workbook_scanner_next(scanner) || scanner.depth() != 1 || @@ -378,6 +408,7 @@ fn canonicalize_xlsx_drawing_xml( // Application extension payload is deliberately absent from the lexical // reader view. Removing its structural container before MCE projection also // prevents directives inside opaque payload from affecting core selection. + charge_xlsx_drawing_xml_pass(budget, source) let without_extension_lists = @ooxml.remove_xml_expanded_name_subtrees( source, xlsx_drawing_extension_list_namespaces, @@ -387,18 +418,42 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } - let projected = @ooxml.project_xml_markup_compatibility( - without_extension_lists.unwrap_or(source.to_owned()), - xlsx_drawing_mce_understood_namespaces, - max_output_chars~, - cancelled~, - extension_elements=xlsx_drawing_mce_extension_elements, - ) catch { - InvalidXml(msg~) => raise InvalidXml(msg~) - ReadCancelled => raise ReadCancelled + match without_extension_lists { + Some(value) => charge_xlsx_drawing_xml_materialization(budget, value) + None => () } + let projected = match without_extension_lists { + Some(extension_free) => { + charge_xlsx_drawing_xml_pass(budget, extension_free) + @ooxml.project_xml_markup_compatibility( + extension_free, + xlsx_drawing_mce_understood_namespaces, + max_output_chars~, + cancelled~, + extension_elements=xlsx_drawing_mce_extension_elements, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + } + None => { + charge_xlsx_drawing_xml_pass(budget, source) + @ooxml.project_xml_markup_compatibility( + source, + xlsx_drawing_mce_understood_namespaces, + max_output_chars~, + cancelled~, + extension_elements=xlsx_drawing_mce_extension_elements, + ) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + } + } + charge_xlsx_drawing_xml_materialization(budget, projected) // Foreign wrappers are removed with their descendants. Merely renaming a // wrapper would leave nested core-looking tags visible to lexical readers. + charge_xlsx_drawing_xml_pass(budget, projected) let core_projected = @ooxml.remove_xml_foreign_namespace_subtrees( projected, xlsx_drawing_core_element_namespaces, @@ -407,9 +462,18 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } - let reader_source = core_projected.unwrap_or(projected) + match core_projected { + Some(value) => charge_xlsx_drawing_xml_materialization(budget, value) + None => () + } + let reader_source = match core_projected { + Some(value) => value + None => projected + } + charge_xlsx_drawing_xml_pass(budget, reader_source) validate_xlsx_drawing_reader_structure(reader_source, cancelled~) - @ooxml.canonicalize_xml_expanded_names( + charge_xlsx_drawing_xml_pass(budget, reader_source) + let canonical = @ooxml.canonicalize_xml_expanded_names( reader_source, [], [ @@ -432,6 +496,8 @@ fn canonicalize_xlsx_drawing_xml( InvalidXml(msg~) => raise InvalidXml(msg~) ReadCancelled => raise ReadCancelled } + charge_xlsx_drawing_xml_materialization(budget, canonical) + canonical } ///| From 8277ba41ddb991ee906b6475fef5556e26a0fbf3 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 10:01:47 +0800 Subject: [PATCH 134/150] test(office): use schema-valid chart anchors --- office/cmd/office/xlsx_read_wbtest.mbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index 13f0da48..f7c33b31 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -266,7 +266,7 @@ fn xlsx_relocated_test_bytes() -> Bytes raise { let table = #|
let sheet_drawing = - #|00001000 + #|00001000 let sheet_drawing_relationships = #| let sheet_chart = From 8874a6561501652231c0c5d4163b5b03c994c42d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 10:59:19 +0800 Subject: [PATCH 135/150] fix(xlsx): seal workbook tab membership --- inspect/outline.mbt | 2 +- xlsx/pkg.generated.mbti | 2 -- xlsx/workbook_types.mbt | 4 ++-- xlsx2html/render.mbt | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/inspect/outline.mbt b/inspect/outline.mbt index 013b876a..adcd40c1 100644 --- a/inspect/outline.mbt +++ b/inspect/outline.mbt @@ -90,7 +90,7 @@ fn find_chart_sheet( workbook : @xlsx.Workbook, name : String, ) -> @xlsx.ChartSheet? { - for chart_sheet in workbook.chart_sheets { + for chart_sheet in workbook.chart_sheets() { if chart_sheet.name() == name { return Some(chart_sheet) } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 97813bc8..23f1ab17 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -1747,8 +1747,6 @@ pub fn TableOptions::set_style_name(Self, String) -> Unit pub fn TableOptions::to_repr(Self) -> @debug.Repr pub struct Workbook { - chart_sheets : Array[ChartSheet] - sheet_order : Array[SheetEntry] styles : Array[Style] conditional_styles : Array[Style] defined_names : Array[DefinedName] diff --git a/xlsx/workbook_types.mbt b/xlsx/workbook_types.mbt index cc27b9bd..31c06464 100644 --- a/xlsx/workbook_types.mbt +++ b/xlsx/workbook_types.mbt @@ -4,8 +4,8 @@ pub struct Workbook { /// A non-empty array is used solely to provide stable reference identity. priv worksheet_owner_token : Array[Unit] priv sheets : Array[Worksheet] - chart_sheets : Array[ChartSheet] - sheet_order : Array[SheetEntry] + priv chart_sheets : Array[ChartSheet] + priv sheet_order : Array[SheetEntry] styles : Array[Style] conditional_styles : Array[Style] defined_names : Array[DefinedName] diff --git a/xlsx2html/render.mbt b/xlsx2html/render.mbt index a18927f1..b6b5e207 100644 --- a/xlsx2html/render.mbt +++ b/xlsx2html/render.mbt @@ -109,7 +109,7 @@ fn find_chart_sheet( workbook : @xlsx.Workbook, name : String, ) -> @xlsx.ChartSheet? { - for chart_sheet in workbook.chart_sheets { + for chart_sheet in workbook.chart_sheets() { if chart_sheet.name() == name { return Some(chart_sheet) } From 9019a62f16ecbe66d62d5b78bfaa3f400c57a9ad Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 11:05:19 +0800 Subject: [PATCH 136/150] fix(xlsx): bind sheet drawing relationships --- xlsx/read.mbt | 208 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 159 insertions(+), 49 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 325b55a4..72fde936 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2716,51 +2716,161 @@ fn parse_hyperlink_elements( } ///| -fn parse_legacy_drawing_rel_id( +fn parse_sheet_drawing_rel_ids( xml : StringView, - tag_name : StringView, -) -> String? raise XlsxError { - let xml_str = xml.to_owned() - let marker = "<\{tag_name.to_owned()}" - let chars = xml_str.to_array() - let mut pos = 0 - while pos < xml_str.length() { - let tail = xml_str[pos:] - let rel = match tail.find(marker) { - Some(value) => value - None => break + root_local_name : StringView, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, +) -> (String?, String?, String?) raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) } - let start = pos + rel - let after = start + marker.length() - if after < chars.length() { - let ch = chars[after] - if ch != ' ' && - ch != '\t' && - ch != '\n' && - ch != '\r' && - ch != '>' && - ch != '/' { - pos = after - continue - } + None => check_read_cancelled(cancelled) + } + let scanner = @ooxml.XmlStartTagScanner::new(xml, cancelled~) + if !workbook_scanner_next(scanner) || + scanner.depth() != 1 || + scanner.local_name() != root_local_name || + ( + scanner.namespace_uri() != transitional_spreadsheet_namespace && + scanner.namespace_uri() != strict_spreadsheet_namespace + ) { + raise InvalidXml( + msg="\{root_local_name.to_owned()} document element is invalid", + ) + } + let spreadsheet_namespace = scanner.namespace_uri().to_owned() + let (relationship_namespace, other_relationship_namespace) = if spreadsheet_namespace == + strict_spreadsheet_namespace { + ( + strict_relationship_attribute_namespace, transitional_relationship_attribute_namespace, + ) + } else { + ( + transitional_relationship_attribute_namespace, strict_relationship_attribute_namespace, + ) + } + let mut drawing : String? = None + let mut legacy_drawing : String? = None + let mut legacy_drawing_hf : String? = None + while workbook_scanner_next(scanner) { + let local_name = scanner.local_name() + let selected = local_name == "drawing" || + ( + root_local_name == "worksheet" && + (local_name == "legacyDrawing" || local_name == "legacyDrawingHF") + ) + if !selected || + ( + scanner.namespace_uri() != transitional_spreadsheet_namespace && + scanner.namespace_uri() != strict_spreadsheet_namespace + ) { + continue } - let text = xml_str[start + 1:] - let end = match text.find("/>") { - Some(value) => value - None => - match text.find(">") { - Some(value) => value - None => raise InvalidXml(msg="\{tag_name.to_owned()} tag not closed") - } + if scanner.namespace_uri() != spreadsheet_namespace || + scanner.depth() != 2 || + scanner.parent_local_name() != Some(root_local_name) || + scanner.parent_namespace_uri() != Some(spreadsheet_namespace) { + raise InvalidXml( + msg="\{local_name.to_owned()} relationship element is misplaced", + ) } - let tag = text[:end] - let id = match attr_value(tag, "r:id") { - Some(value) => value - None => raise InvalidXml(msg="\{tag_name.to_owned()} id missing") + if workbook_scanner_attribute(scanner, other_relationship_namespace, "id") + is Some(_) { + raise InvalidXml( + msg="\{local_name.to_owned()} relationship namespace is invalid", + ) + } + let id = match + workbook_scanner_attribute(scanner, relationship_namespace, "id") { + Some(value) if value != "" => value + _ => + raise InvalidXml( + msg="\{local_name.to_owned()} relationship id is missing", + ) + } + if local_name == "drawing" { + if drawing is Some(_) { + raise InvalidXml(msg="drawing relationship element is duplicated") + } + drawing = Some(id) + } else if local_name == "legacyDrawing" { + if legacy_drawing is Some(_) { + raise InvalidXml(msg="legacyDrawing relationship element is duplicated") + } + legacy_drawing = Some(id) + } else { + if legacy_drawing_hf is Some(_) { + raise InvalidXml( + msg="legacyDrawingHF relationship element is duplicated", + ) + } + legacy_drawing_hf = Some(id) + } + } + (drawing, legacy_drawing, legacy_drawing_hf) +} + +///| +test "sheet drawing relationships require direct dialect-matched roots" { + let transitional = + #| + let (drawing, legacy, header_footer) = parse_sheet_drawing_rel_ids( + transitional, "worksheet", + ) + assert_true(drawing == Some("drawing")) + assert_true(legacy == Some("legacy")) + assert_true(header_footer == Some("header-footer")) + let strict = + #| + let (strict_drawing, strict_legacy, strict_header_footer) = parse_sheet_drawing_rel_ids( + strict, "chartsheet", + ) + assert_true(strict_drawing == Some("strict-drawing")) + assert_true(strict_legacy is None) + assert_true(strict_header_footer is None) + + let invalid : Array[(String, String, String)] = [ + ( + ( + #| + ), + "worksheet", + "drawing relationship element is misplaced", + ), + ( + ( + #| + ), + "worksheet", + "drawing relationship element is misplaced", + ), + ( + ( + #| + ), + "chartsheet", + "drawing relationship namespace is invalid", + ), + ( + ( + #| + ), + "chartsheet", + "drawing relationship element is duplicated", + ), + ] + for entry in invalid { + let (xml, root, expected) = entry + try parse_sheet_drawing_rel_ids(xml, root) catch { + InvalidXml(msg~) => assert_eq(msg, expected) + _ => fail("unexpected sheet drawing relationship error") + } noraise { + _ => fail("expected invalid sheet drawing relationship rejection") } - return Some(id) } - None } ///| @@ -4082,14 +4192,11 @@ fn read_zip_archive_core_unchecked( let table_part_ids = parse_table_part_ids(sheet_core_xml) let pivot_part_ids = parse_pivot_table_part_ids(sheet_core_xml) let slicer_rel_ids = parse_sheet_slicer_rel_ids(sheet_feature_xml) - let drawing_rel_id = parse_legacy_drawing_rel_id( - sheet_core_xml, "drawing", - ) - let legacy_drawing_rel_id = parse_legacy_drawing_rel_id( - sheet_core_xml, "legacyDrawing", - ) - let legacy_drawing_hf_rel_id = parse_legacy_drawing_rel_id( - sheet_core_xml, "legacyDrawingHF", + let (drawing_rel_id, legacy_drawing_rel_id, legacy_drawing_hf_rel_id) = parse_sheet_drawing_rel_ids( + sheet_source_xml, + "worksheet", + budget=read_budget, + cancelled~, ) let sparkline_groups = parse_sparkline_groups(sheet_feature_xml) let data_validations = parse_data_validations( @@ -4871,8 +4978,11 @@ fn read_zip_archive_core_unchecked( ) read_budget.checkpoint() read_budget.charge_work(chartsheet_xml.length()) - let drawing_rel_id = parse_legacy_drawing_rel_id( - chartsheet_xml, "drawing", + let (drawing_rel_id, _, _) = parse_sheet_drawing_rel_ids( + chartsheet_source_xml, + "chartsheet", + budget=read_budget, + cancelled~, ) let drawing_rel = match drawing_rel_id { Some(value) => value From 383686f6ee9f76117435cbf38d59369d836ff227 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 11:22:36 +0800 Subject: [PATCH 137/150] fix(xlsx): constrain DrawingML reader targets --- xlsx/read_drawing_xml.mbt | 82 +++++- xlsx/read_xml_namespaces.mbt | 466 ++++++++++++++++++++++++++++++++++- 2 files changed, 539 insertions(+), 9 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 15716c29..0a39c4fa 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -199,6 +199,22 @@ fn charge_drawing_anchor_pass( } } +///| +/// Returns the shape-level solid fill, never a line-level fill nested later in +/// `a:ln`. Drawing structure validation has already rejected every other +/// `a:solidFill` path before the canonical reader reaches this helper. +fn drawing_shape_solid_fill_body(sppr_body : StringView) -> String? { + let fill_start = match find_xml_open_tag_start(sppr_body, "a:solidFill") { + Some(value) => value + None => return None + } + match find_xml_open_tag_start(sppr_body, "a:ln") { + Some(line_start) if line_start < fill_start => return None + _ => () + } + extract_tag_body_from(sppr_body, "a:solidFill") +} + ///| fn parse_drawing_images( anchors : ArrayView[(String, PicturePositioning)], @@ -330,7 +346,14 @@ fn parse_drawing_images( Some(value) => unescape_xml_text(value) None => "" } - let lock_aspect_ratio = pic_body.contains("noChangeAspect=\"1\"") + let lock_aspect_ratio = match tag_attributes_in(pic_body, "a:picLocks") { + Some(tag) => + match attr_value(tag, "noChangeAspect") { + Some(value) => parse_bool_attr(value) + None => false + } + None => false + } let client_tag = match tag_attributes_in(anchor_xml, "xdr:clientData") { Some(value) => value None => "" @@ -706,7 +729,7 @@ fn parse_drawing_shapes( } let mut fill_color = "" let mut fill_transparency = 0 - match extract_tag_body_from(sppr_body, "a:solidFill") { + match drawing_shape_solid_fill_body(sppr_body) { Some(fill_body) => match tag_attributes_in(fill_body, "a:srgbClr") { Some(tag) => @@ -1137,7 +1160,7 @@ test "DrawingML rejects supported-namespace reader decoys" { ( #| ), - "drawing anchor is misplaced", + "drawing anchor reader target is misplaced", ), ( ( @@ -1169,6 +1192,42 @@ test "DrawingML rejects supported-namespace reader decoys" { ), "drawing anchor has multiple reader objects", ), + ( + ( + #| + ), + "drawing reader target dialect is mixed", + ), + ( + ( + #| + ), + "drawing blip target is misplaced", + ), + ( + ( + #| + ), + "drawing blip target is duplicated", + ), + ( + ( + #| + ), + "drawing reader target dialect is mixed", + ), + ( + ( + #| + ), + "drawing relationship attribute dialect is invalid", + ), + ( + ( + #|decoyreal + ), + "drawing text target is misplaced", + ), ] for case in cases { let (source, expected) = case @@ -1183,14 +1242,27 @@ test "DrawingML rejects supported-namespace reader decoys" { } } +///| +test "DrawingML shape fill selection excludes line-only fills" { + let line_only = + #| + assert_true(drawing_shape_solid_fill_body(line_only) is None) + let shape_and_line = + #| + assert_eq( + drawing_shape_solid_fill_body(shape_and_line).unwrap_or(""), + "", + ) +} + ///| test "DrawingML validates Strict chart dialect and payload URI together" { let source = - #| + #| let canonical = canonicalize_xlsx_drawing_xml( source, default_max_xml_part_bytes, ) - assert_true(canonical.contains("")) + assert_true(canonical.contains("")) } ///| diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index c3c6fc40..864069e5 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -91,6 +91,29 @@ fn xlsx_drawing_reader_object_name(local_name : StringView) -> Bool { local_name == "pic" || local_name == "sp" || local_name == "graphicFrame" } +///| +fn drawing_reader_relationship_attribute( + scanner : @ooxml.XmlStartTagScanner, + relationship_namespace : StringView, + other_relationship_namespace : StringView, + local_name : StringView, + required : Bool, +) -> String? raise XlsxError { + if workbook_scanner_attribute( + scanner, other_relationship_namespace, local_name, + ) + is Some(_) { + raise InvalidXml(msg="drawing relationship attribute dialect is invalid") + } + match + workbook_scanner_attribute(scanner, relationship_namespace, local_name) { + Some(value) if value != "" => Some(value) + _ if required => + raise InvalidXml(msg="drawing relationship attribute is missing") + _ => None + } +} + ///| /// Proves that every element consumed by a namespace-oblivious drawing reader /// occurs on the schema path that reader assumes. Merely filtering namespaces @@ -112,25 +135,94 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing document element is invalid") } let drawing_namespace = scanner.namespace_uri().to_owned() - let (main_namespace, chart_namespace) = if drawing_namespace == - transitional_spreadsheet_drawing_namespace { - (transitional_drawing_main_namespace, transitional_drawing_chart_namespace) + let ( + main_namespace, + chart_namespace, + other_drawing_namespace, + other_main_namespace, + other_chart_namespace, + relationship_namespace, + other_relationship_namespace, + ) = if drawing_namespace == transitional_spreadsheet_drawing_namespace { + ( + transitional_drawing_main_namespace, transitional_drawing_chart_namespace, + strict_spreadsheet_drawing_namespace, strict_drawing_main_namespace, strict_drawing_chart_namespace, + transitional_relationship_attribute_namespace, strict_relationship_attribute_namespace, + ) } else { - (strict_drawing_main_namespace, strict_drawing_chart_namespace) + ( + strict_drawing_main_namespace, strict_drawing_chart_namespace, transitional_spreadsheet_drawing_namespace, + transitional_drawing_main_namespace, transitional_drawing_chart_namespace, + strict_relationship_attribute_namespace, transitional_relationship_attribute_namespace, + ) } let mut in_reader_anchor = false + let mut anchor_name = "" let mut reader_object_seen = false + let mut current_reader_object = "" let mut reader_payload_seen = false let mut graphic_data_uri : String? = None let mut extension_depth : Int? = None + let mut text_body_depth : Int? = None + let mut solid_fill_depth : Int? = None + let mut color_depth : Int? = None + let mut marker_parent = "" + let mut marker_col_seen = false + let mut marker_col_off_seen = false + let mut marker_row_seen = false + let mut marker_row_off_seen = false + let mut from_seen = false + let mut to_seen = false + let mut anchor_ext_seen = false + let mut client_data_seen = false + let mut non_visual_container_seen = false + let mut c_nv_pr_seen = false + let mut c_nv_pic_pr_seen = false + let mut blip_fill_seen = false + let mut blip_seen = false + let mut hyperlink_seen = false + let mut pic_locks_seen = false + let mut sp_pr_seen = false + let mut xfrm_seen = false + let mut xfrm_ext_seen = false + let mut geometry_seen = false + let mut shape_fill_seen = false + let mut line_seen = false + let mut line_fill_seen = false + let mut color_seen = false + let mut alpha_seen = false + let mut text_body_seen = false + let mut graphic_seen = false + let mut graphic_data_seen = false while workbook_scanner_next(scanner) { let depth = scanner.depth() let local_name = scanner.local_name() let namespace_uri = scanner.namespace_uri().to_owned() + match text_body_depth { + Some(start) if depth <= start => text_body_depth = None + _ => () + } + match solid_fill_depth { + Some(start) if depth <= start => solid_fill_depth = None + _ => () + } + match color_depth { + Some(start) if depth <= start => color_depth = None + _ => () + } if depth <= 2 { in_reader_anchor = false + anchor_name = "" reader_object_seen = false + current_reader_object = "" reader_payload_seen = false + from_seen = false + to_seen = false + anchor_ext_seen = false + client_data_seen = false + } else if depth <= 3 { + current_reader_object = "" + marker_parent = "" } if depth <= 5 { graphic_data_uri = None @@ -141,6 +233,11 @@ fn validate_xlsx_drawing_reader_structure( Some(_) => extension_depth = None None => () } + if namespace_uri == other_drawing_namespace || + namespace_uri == other_main_namespace || + namespace_uri == other_chart_namespace { + raise InvalidXml(msg="drawing reader target dialect is mixed") + } if local_name == "ext" && xlsx_drawing_extension_list_namespaces.any(candidate => { candidate == namespace_uri @@ -155,8 +252,14 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing anchor is misplaced") } in_reader_anchor = true + anchor_name = local_name.to_owned() reader_object_seen = false + current_reader_object = "" reader_payload_seen = false + from_seen = false + to_seen = false + anchor_ext_seen = false + client_data_seen = false } else if namespace_uri == drawing_namespace && xlsx_drawing_reader_object_name(local_name) { if depth != 3 || @@ -169,19 +272,369 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing anchor has multiple reader objects") } reader_object_seen = true + current_reader_object = local_name.to_owned() + non_visual_container_seen = false + c_nv_pr_seen = false + c_nv_pic_pr_seen = false + blip_fill_seen = false + blip_seen = false + hyperlink_seen = false + pic_locks_seen = false + sp_pr_seen = false + xfrm_seen = false + xfrm_ext_seen = false + geometry_seen = false + shape_fill_seen = false + line_seen = false + line_fill_seen = false + color_seen = false + alpha_seen = false + text_body_seen = false + graphic_seen = false + graphic_data_seen = false + } else if namespace_uri == drawing_namespace && + ( + local_name == "from" || + local_name == "to" || + local_name == "ext" || + local_name == "clientData" + ) { + if depth != 3 || + !in_reader_anchor || + scanner.parent_local_name() != Some(anchor_name) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing anchor reader target is misplaced") + } + if local_name == "from" { + if from_seen { + raise InvalidXml(msg="drawing anchor from target is duplicated") + } + from_seen = true + marker_parent = "from" + marker_col_seen = false + marker_col_off_seen = false + marker_row_seen = false + marker_row_off_seen = false + } else if local_name == "to" { + if to_seen { + raise InvalidXml(msg="drawing anchor to target is duplicated") + } + to_seen = true + marker_parent = "to" + marker_col_seen = false + marker_col_off_seen = false + marker_row_seen = false + marker_row_off_seen = false + } else if local_name == "ext" { + if anchor_ext_seen { + raise InvalidXml(msg="drawing anchor ext target is duplicated") + } + anchor_ext_seen = true + } else { + if client_data_seen { + raise InvalidXml(msg="drawing clientData target is duplicated") + } + client_data_seen = true + } + } else if namespace_uri == drawing_namespace && + ( + local_name == "col" || + local_name == "colOff" || + local_name == "row" || + local_name == "rowOff" + ) { + if depth != 4 || + (marker_parent != "from" && marker_parent != "to") || + scanner.parent_local_name() != Some(marker_parent) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing marker target is misplaced") + } + if local_name == "col" { + if marker_col_seen { + raise InvalidXml(msg="drawing marker col target is duplicated") + } + marker_col_seen = true + } else if local_name == "colOff" { + if marker_col_off_seen { + raise InvalidXml(msg="drawing marker colOff target is duplicated") + } + marker_col_off_seen = true + } else if local_name == "row" { + if marker_row_seen { + raise InvalidXml(msg="drawing marker row target is duplicated") + } + marker_row_seen = true + } else { + if marker_row_off_seen { + raise InvalidXml(msg="drawing marker rowOff target is duplicated") + } + marker_row_off_seen = true + } + } else if namespace_uri == drawing_namespace && + ( + local_name == "nvPicPr" || + local_name == "nvSpPr" || + local_name == "nvGraphicFramePr" + ) { + let expected = if current_reader_object == "pic" { + "nvPicPr" + } else if current_reader_object == "sp" { + "nvSpPr" + } else { + "nvGraphicFramePr" + } + if current_reader_object == "" || + local_name != expected || + depth != 4 || + scanner.parent_local_name() != Some(current_reader_object) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing nonvisual target is misplaced") + } + if non_visual_container_seen { + raise InvalidXml(msg="drawing nonvisual target is duplicated") + } + non_visual_container_seen = true + } else if namespace_uri == drawing_namespace && local_name == "cNvPr" { + let expected_parent = if current_reader_object == "pic" { + "nvPicPr" + } else if current_reader_object == "sp" { + "nvSpPr" + } else { + "nvGraphicFramePr" + } + if current_reader_object == "" || + depth != 5 || + scanner.parent_local_name() != Some(expected_parent) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing cNvPr target is misplaced") + } + if c_nv_pr_seen { + raise InvalidXml(msg="drawing cNvPr target is duplicated") + } + c_nv_pr_seen = true + } else if namespace_uri == drawing_namespace && local_name == "cNvPicPr" { + if current_reader_object != "pic" || + depth != 5 || + scanner.parent_local_name() != Some("nvPicPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing cNvPicPr target is misplaced") + } + if c_nv_pic_pr_seen { + raise InvalidXml(msg="drawing cNvPicPr target is duplicated") + } + c_nv_pic_pr_seen = true + } else if namespace_uri == drawing_namespace && local_name == "blipFill" { + if current_reader_object != "pic" || + depth != 4 || + scanner.parent_local_name() != Some("pic") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing blipFill target is misplaced") + } + if blip_fill_seen { + raise InvalidXml(msg="drawing blipFill target is duplicated") + } + blip_fill_seen = true + } else if namespace_uri == drawing_namespace && local_name == "spPr" { + if (current_reader_object != "sp" && current_reader_object != "pic") || + depth != 4 || + scanner.parent_local_name() != Some(current_reader_object) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing spPr target is misplaced") + } + if sp_pr_seen { + raise InvalidXml(msg="drawing spPr target is duplicated") + } + sp_pr_seen = true + } else if namespace_uri == drawing_namespace && local_name == "txBody" { + if current_reader_object != "sp" || + depth != 4 || + scanner.parent_local_name() != Some("sp") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing txBody target is misplaced") + } + if text_body_seen { + raise InvalidXml(msg="drawing txBody target is duplicated") + } + text_body_seen = true + text_body_depth = Some(depth) } else if namespace_uri == main_namespace && local_name == "graphic" { if depth != 4 || + current_reader_object != "graphicFrame" || scanner.parent_local_name() != Some("graphicFrame") || scanner.parent_namespace_uri() != Some(drawing_namespace) { raise InvalidXml(msg="drawing graphic element is misplaced") } + if graphic_seen { + raise InvalidXml(msg="drawing graphic element is duplicated") + } + graphic_seen = true } else if namespace_uri == main_namespace && local_name == "graphicData" { if depth != 5 || + current_reader_object != "graphicFrame" || scanner.parent_local_name() != Some("graphic") || scanner.parent_namespace_uri() != Some(main_namespace) { raise InvalidXml(msg="drawing graphicData element is misplaced") } + if graphic_data_seen { + raise InvalidXml(msg="drawing graphicData element is duplicated") + } + graphic_data_seen = true graphic_data_uri = workbook_scanner_attribute(scanner, "", "uri") + } else if namespace_uri == main_namespace && + local_name == "blip" && + current_reader_object == "pic" { + if depth != 5 || + scanner.parent_local_name() != Some("blipFill") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing blip target is misplaced") + } + if blip_seen { + raise InvalidXml(msg="drawing blip target is duplicated") + } + blip_seen = true + ignore( + drawing_reader_relationship_attribute( + scanner, relationship_namespace, other_relationship_namespace, "embed", + true, + ), + ) + } else if namespace_uri == main_namespace && + local_name == "hlinkClick" && + current_reader_object == "pic" { + if depth != 6 || + scanner.parent_local_name() != Some("cNvPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing hyperlink target is misplaced") + } + if hyperlink_seen { + raise InvalidXml(msg="drawing hyperlink target is duplicated") + } + hyperlink_seen = true + ignore( + drawing_reader_relationship_attribute( + scanner, relationship_namespace, other_relationship_namespace, "id", false, + ), + ) + } else if namespace_uri == main_namespace && local_name == "picLocks" { + if current_reader_object != "pic" || + depth != 6 || + scanner.parent_local_name() != Some("cNvPicPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing picLocks target is misplaced") + } + if pic_locks_seen { + raise InvalidXml(msg="drawing picLocks target is duplicated") + } + pic_locks_seen = true + } else if namespace_uri == main_namespace && + local_name == "xfrm" && + current_reader_object == "sp" { + if depth != 5 || + scanner.parent_local_name() != Some("spPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing xfrm target is misplaced") + } + if xfrm_seen { + raise InvalidXml(msg="drawing xfrm target is duplicated") + } + xfrm_seen = true + } else if namespace_uri == main_namespace && + local_name == "ext" && + current_reader_object == "sp" { + if depth != 6 || + scanner.parent_local_name() != Some("xfrm") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing shape ext target is misplaced") + } + if xfrm_ext_seen { + raise InvalidXml(msg="drawing shape ext target is duplicated") + } + xfrm_ext_seen = true + } else if namespace_uri == main_namespace && + local_name == "prstGeom" && + current_reader_object == "sp" { + if depth != 5 || + scanner.parent_local_name() != Some("spPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing geometry target is misplaced") + } + if geometry_seen { + raise InvalidXml(msg="drawing geometry target is duplicated") + } + geometry_seen = true + } else if namespace_uri == main_namespace && + local_name == "solidFill" && + current_reader_object == "sp" { + let shape_fill = depth == 5 && + scanner.parent_local_name() == Some("spPr") && + scanner.parent_namespace_uri() == Some(drawing_namespace) + let line_fill = depth == 6 && + scanner.parent_local_name() == Some("ln") && + scanner.parent_namespace_uri() == Some(main_namespace) + if !shape_fill && !line_fill { + raise InvalidXml(msg="drawing solidFill target is misplaced") + } + if shape_fill { + if shape_fill_seen { + raise InvalidXml(msg="drawing shape fill target is duplicated") + } + shape_fill_seen = true + } else { + if line_fill_seen { + raise InvalidXml(msg="drawing line fill target is duplicated") + } + line_fill_seen = true + } + solid_fill_depth = Some(depth) + color_seen = false + alpha_seen = false + } else if namespace_uri == main_namespace && + local_name == "srgbClr" && + current_reader_object == "sp" { + let fill_depth = solid_fill_depth.unwrap_or(-1) + if depth != fill_depth + 1 || + scanner.parent_local_name() != Some("solidFill") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing color target is misplaced") + } + if color_seen { + raise InvalidXml(msg="drawing color target is duplicated") + } + color_seen = true + alpha_seen = false + color_depth = Some(depth) + } else if namespace_uri == main_namespace && + local_name == "alpha" && + current_reader_object == "sp" { + let parent_depth = color_depth.unwrap_or(-1) + if depth != parent_depth + 1 || + scanner.parent_local_name() != Some("srgbClr") || + scanner.parent_namespace_uri() != Some(main_namespace) { + raise InvalidXml(msg="drawing alpha target is misplaced") + } + if alpha_seen { + raise InvalidXml(msg="drawing alpha target is duplicated") + } + alpha_seen = true + } else if namespace_uri == main_namespace && + local_name == "ln" && + current_reader_object == "sp" { + if depth != 5 || + scanner.parent_local_name() != Some("spPr") || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="drawing line target is misplaced") + } + if line_seen { + raise InvalidXml(msg="drawing line target is duplicated") + } + line_seen = true + } else if namespace_uri == main_namespace && + local_name == "t" && + current_reader_object == "sp" { + match text_body_depth { + Some(body_depth) if depth > body_depth => () + _ => raise InvalidXml(msg="drawing text target is misplaced") + } } else if ( namespace_uri == transitional_drawing_chart_namespace || namespace_uri == strict_drawing_chart_namespace @@ -199,6 +652,11 @@ fn validate_xlsx_drawing_reader_structure( if reader_payload_seen { raise InvalidXml(msg="drawing anchor has multiple reader payloads") } + ignore( + drawing_reader_relationship_attribute( + scanner, relationship_namespace, other_relationship_namespace, "id", true, + ), + ) reader_payload_seen = true } else if namespace_uri == drawing_slicer_namespace && local_name == "slicer" { From ef41358d37a1eef7a8a622a0e657de62e0ba1567 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 11:30:29 +0800 Subject: [PATCH 138/150] fix(xlsx): bind chartsheets to drawing charts --- office/cmd/office/xlsx_read_wbtest.mbt | 2 +- xlsx/ooxml_rels_error_test.mbt | 55 ++++++++++++++++++++++-- xlsx/read.mbt | 58 +++++++++++++------------- xlsx/read_drawing_xml.mbt | 57 +++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 33 deletions(-) diff --git a/office/cmd/office/xlsx_read_wbtest.mbt b/office/cmd/office/xlsx_read_wbtest.mbt index f7c33b31..1ddfa03b 100644 --- a/office/cmd/office/xlsx_read_wbtest.mbt +++ b/office/cmd/office/xlsx_read_wbtest.mbt @@ -276,7 +276,7 @@ fn xlsx_relocated_test_bytes() -> Bytes raise { let chart_sheet_relationships = #| let drawing = - #| + #|0000 let drawing_relationships = #| let chart = diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index 4078980f..34cf4636 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -381,15 +381,21 @@ test "ooxml rels: pivot cache rels tolerate stale first entries and mixed target } ///| -test "ooxml rels: chartsheet chart rels tolerate stale first entries and mixed targets" { +test "ooxml rels: chartsheet binds the chart selected by drawing r:id" { let workbook = @xlsx.Workbook::new() ignore(workbook.add_sheet("Sheet1")) let chart_xml = #| #| - #| + #| GOOD #| ignore(workbook.add_chart_sheet("Chart1", chart_xml)) + let decoy_chart_xml = + #| + #| + #| BAD + #| + ignore(workbook.add_chart_sheet("Chart2", decoy_chart_xml)) let bytes = @xlsx.write(workbook) let archive = @zip.read(bytes) catch { err => fail("zip read failed: \{repr(err)}") @@ -398,7 +404,7 @@ test "ooxml rels: chartsheet chart rels tolerate stale first entries and mixed t let drawing_rels_xml = decode_zip_entry_utf8(archive, drawing_rels_path) let patched_drawing_rels = prepend_relationship( drawing_rels_xml, "rId0", "http://schemas.openxmlformats.org/officeDocument/2006/relationships/chart", - "../charts/ghostChart.xml", + "../charts/chart2.xml", ).replace_all( old="Target=\"../charts/chart1.xml\"", new="Target=\"/xl/charts/chart1.xml\"", @@ -411,8 +417,49 @@ test "ooxml rels: chartsheet chart rels tolerate stale first entries and mixed t err => fail("zip rewrite failed: \{err}") } let parsed = @xlsx.read(updated) - inspect(parsed.chart_sheets().length(), content="1") + inspect(parsed.chart_sheets().length(), content="2") let parsed_chart = parsed.chart_sheets()[0] inspect(parsed_chart.name(), content="Chart1") inspect(parsed_chart.chart_xml().contains(" + ), + ), + ) + let archive = @zip.read(@xlsx.write(workbook)) catch { + err => fail("zip read failed: \{repr(err)}") + } + let drawing_path = "xl/drawings/drawing2.xml" + guard archive.get(drawing_path) is Some(_) else { + fail("missing chart-sheet drawing fixture") + } + let without_drawing = @zip.Archive::new() + for entry in archive.entries() { + if entry.name() != drawing_path { + without_drawing.add( + entry.name(), + entry.data(), + compression=entry.compression(), + data_descriptor=entry.data_descriptor(), + ) + } + } + let result = Ok(@xlsx.read(@zip.write(without_drawing))) catch { + error => Err(error) + } + match result { + Err(@xlsx.MissingPart(path~)) => assert_eq(path, drawing_path) + _ => fail("expected missing chart-sheet drawing part") + } } diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 72fde936..0bf41f01 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -5015,13 +5015,29 @@ fn read_zip_archive_core_unchecked( part_names, cancelled~, ) - require_part_content_type( + let drawing_bytes = load_typed_part( + archive, content_types, - logical_archive_part_path(drawing_path), + drawing_path, [ct_drawing], "drawing", cancelled~, ) + let drawing_source_xml = decode_source(drawing_bytes) + let drawing_xml = canonicalize_xlsx_drawing_xml( + drawing_source_xml, + limits.max_xml_part_bytes, + budget=read_budget, + cancelled~, + ) + let drawing_anchors = parse_drawing_anchors( + drawing_xml, + budget=read_budget, + ) + let chart_rel_id = single_drawing_chart_rel_id( + drawing_anchors, + budget=read_budget, + ) let drawing_rels_path = actual_relationship_part_path( drawing_path, part_names, ) @@ -5039,38 +5055,24 @@ fn read_zip_archive_core_unchecked( budget=read_budget, cancelled~, ) - let chart_path = match - first_existing_rel_target_path( - chart_targets, - drawing_path, - part_names, - cancelled~, - ) { - Some(path) => path - None => { - let chart_target = match first_relationship_target(chart_targets) { - Some(value) => value - None => raise InvalidXml(msg="chart target missing") - } - actual_relationship_target_path( - drawing_path, - chart_target, - part_names, - cancelled~, - ) - } + let chart_target = match chart_targets.get(chart_rel_id) { + Some(value) => value + None => raise InvalidXml(msg="chartsheet chart target missing") } - require_part_content_type( + let chart_path = actual_relationship_target_path( + drawing_path, + chart_target, + part_names, + cancelled~, + ) + let chart_bytes = load_typed_part( + archive, content_types, - logical_archive_part_path(chart_path), + chart_path, [ct_chart], "chart", cancelled~, ) - let chart_bytes = match archive.get(chart_path) { - Some(value) => value - None => raise MissingPart(path=chart_path) - } let chart_xml = decode(chart_bytes) let chart_sheet = ChartSheet::new(name, chart_xml) chart_sheet.state = state diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 0a39c4fa..dc5a412f 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -199,6 +199,38 @@ fn charge_drawing_anchor_pass( } } +///| +/// Returns the sole chart relationship selected by a chart-sheet drawing. +/// Drawing structure validation has already proved the canonical `c:chart` +/// path and relationship-attribute dialect; this pass enforces chart-sheet +/// cardinality and keeps relationship lookup tied to that exact payload. +fn single_drawing_chart_rel_id( + anchors : ArrayView[(String, PicturePositioning)], + budget? : ReadBudget, +) -> String raise XlsxError { + let mut selected : String? = None + for entry in anchors { + let (anchor_xml, _) = entry + charge_drawing_anchor_pass(budget, anchor_xml) + match tag_attributes_in(anchor_xml, "c:chart") { + Some(tag) => { + if selected is Some(_) { + raise InvalidXml(msg="chartsheet drawing has multiple charts") + } + selected = match attr_value(tag, "r:id") { + Some(value) if value.length() > 0 => Some(value.to_string()) + _ => raise InvalidXml(msg="chartsheet chart rel id missing") + } + } + None => () + } + } + match selected { + Some(value) => value + None => raise InvalidXml(msg="chartsheet chart payload missing") + } +} + ///| /// Returns the shape-level solid fill, never a line-level fill nested later in /// `a:ln`. Drawing structure validation has already rejected every other @@ -1255,6 +1287,31 @@ test "DrawingML shape fill selection excludes line-only fills" { ) } +///| +test "chartsheet drawing selector requires exactly one chart payload" { + let anchor = + #| + let one : ReadOnlyArray[(String, PicturePositioning)] = [(anchor, OneCell)] + assert_eq(single_drawing_chart_rel_id(one), "selected") + let empty : ReadOnlyArray[(String, PicturePositioning)] = [] + try single_drawing_chart_rel_id(empty) catch { + InvalidXml(msg~) => assert_eq(msg, "chartsheet chart payload missing") + _ => fail("unexpected empty chartsheet drawing error") + } noraise { + _ => fail("expected empty chartsheet drawing rejection") + } + let two : ReadOnlyArray[(String, PicturePositioning)] = [ + (anchor, OneCell), + (anchor, OneCell), + ] + try single_drawing_chart_rel_id(two) catch { + InvalidXml(msg~) => assert_eq(msg, "chartsheet drawing has multiple charts") + _ => fail("unexpected duplicate chartsheet chart error") + } noraise { + _ => fail("expected duplicate chartsheet chart rejection") + } +} + ///| test "DrawingML validates Strict chart dialect and payload URI together" { let source = From 50510a77ae1ee5d8c440f6eff2864eba13cb7d00 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 12:40:36 +0800 Subject: [PATCH 139/150] fix(xlsx): accept absolute chart anchors --- xlsx/ooxml_rels_error_test.mbt | 41 ++++++++++++++++++++++++++++++++++ xlsx/read_drawing_xml.mbt | 38 ++++++++++++++++++++++++++++++- xlsx/read_xml_namespaces.mbt | 4 +++- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/xlsx/ooxml_rels_error_test.mbt b/xlsx/ooxml_rels_error_test.mbt index 34cf4636..6e5827d8 100644 --- a/xlsx/ooxml_rels_error_test.mbt +++ b/xlsx/ooxml_rels_error_test.mbt @@ -425,6 +425,47 @@ test "ooxml rels: chartsheet binds the chart selected by drawing r:id" { assert_false(parsed_chart.chart_xml().contains("BAD")) } +///| +test "ooxml rels: chartsheet accepts absoluteAnchor drawings" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + ignore( + workbook.add_chart_sheet( + "Chart1", + ( + #|ABSOLUTE + ), + ), + ) + let archive = @zip.read(@xlsx.write(workbook)) catch { + err => fail("zip read failed: \{repr(err)}") + } + let drawing_path = "xl/drawings/drawing2.xml" + guard archive.get(drawing_path) is Some(_) else { + fail("missing chart-sheet drawing fixture") + } + let absolute_drawing = + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + assert_true( + archive.replace(drawing_path, @encoding/utf8.encode(absolute_drawing)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + inspect(parsed.chart_sheets().length(), content="1") + assert_true(parsed.chart_sheets()[0].chart_xml().contains("ABSOLUTE")) +} + ///| test "ooxml rels: chartsheet requires its selected drawing part" { let workbook = @xlsx.Workbook::new() diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index dc5a412f..90825fa3 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -178,6 +178,28 @@ fn parse_drawing_anchors( anchors.push((full, positioning)) parsed += 1 } + let mut first_absolute = true + for chunk in xml_str.split(" value.checkpoint() + None => () + } + } + let text = chunk.to_owned() + let close_tag = "" + let end = match text.find(close_tag) { + Some(pos) => pos + close_tag.length() + None => raise InvalidXml(msg="absoluteAnchor close missing") + } + let slice = text[:end] + anchors.push((" value.checkpoint() None => () @@ -1452,15 +1474,29 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { _ => fail("expected InvalidXml for twoCellAnchor close missing") } + let absolute_missing_close = "" + let absolute_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + parse_drawing_anchors(absolute_missing_close), + ) catch { + e => Err(e) + } + match absolute_result { + Err(XlsxError::InvalidXml(msg~)) => + inspect(msg, content="absoluteAnchor close missing") + _ => fail("expected InvalidXml for absoluteAnchor close missing") + } + let xml = #| #| #| + #| #| let anchors = parse_drawing_anchors(xml) - inspect(anchors.length(), content="2") + inspect(anchors.length(), content="3") debug_inspect(anchors[0].1, content="Absolute") debug_inspect(anchors[1].1, content="TwoCell") + debug_inspect(anchors[2].1, content="Absolute") } ///| diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 864069e5..c0f49f17 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -83,7 +83,9 @@ let xlsx_drawing_core_element_namespaces : Array[String] = [ ///| fn xlsx_drawing_anchor_name(local_name : StringView) -> Bool { - local_name == "oneCellAnchor" || local_name == "twoCellAnchor" + local_name == "oneCellAnchor" || + local_name == "twoCellAnchor" || + local_name == "absoluteAnchor" } ///| From 6e0515e80bc08565a900d031ca4d5fd715d5a33d Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 13:23:47 +0800 Subject: [PATCH 140/150] fix(xlsx): isolate absolute chart anchors --- xlsx/read.mbt | 1 + xlsx/read_drawing_xml.mbt | 51 +++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 0bf41f01..4ce75877 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -5033,6 +5033,7 @@ fn read_zip_archive_core_unchecked( let drawing_anchors = parse_drawing_anchors( drawing_xml, budget=read_budget, + include_absolute=true, ) let chart_rel_id = single_drawing_chart_rel_id( drawing_anchors, diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 90825fa3..591f0f22 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -109,6 +109,7 @@ fn drawing_axis_span_emu( fn parse_drawing_anchors( drawing_xml : StringView, budget? : ReadBudget, + include_absolute? : Bool = false, ) -> Array[(String, PicturePositioning)] raise XlsxError { match budget { Some(value) => { @@ -178,27 +179,33 @@ fn parse_drawing_anchors( anchors.push((full, positioning)) parsed += 1 } - let mut first_absolute = true - for chunk in xml_str.split(" value.checkpoint() - None => () + // Worksheet drawing readers currently model cell-relative geometry. Keep + // absolute anchors out of those consumers so a drawing they previously + // ignored does not become a hard read failure. Chart sheets opt in because + // their exact chart relationship selection is position-independent. + if include_absolute { + let mut first_absolute = true + for chunk in xml_str.split(" value.checkpoint() + None => () + } + } + let text = chunk.to_owned() + let close_tag = "" + let end = match text.find(close_tag) { + Some(pos) => pos + close_tag.length() + None => raise InvalidXml(msg="absoluteAnchor close missing") + } + let slice = text[:end] + anchors.push((" pos + close_tag.length() - None => raise InvalidXml(msg="absoluteAnchor close missing") - } - let slice = text[:end] - anchors.push((" value.checkpoint() @@ -1476,7 +1483,7 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { let absolute_missing_close = "" let absolute_result : Result[Array[(String, PicturePositioning)], Error] = Ok( - parse_drawing_anchors(absolute_missing_close), + parse_drawing_anchors(absolute_missing_close, include_absolute=true), ) catch { e => Err(e) } @@ -1492,7 +1499,9 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { #| #| #| - let anchors = parse_drawing_anchors(xml) + let worksheet_anchors = parse_drawing_anchors(xml) + inspect(worksheet_anchors.length(), content="2") + let anchors = parse_drawing_anchors(xml, include_absolute=true) inspect(anchors.length(), content="3") debug_inspect(anchors[0].1, content="Absolute") debug_inspect(anchors[1].1, content="TwoCell") From be6358980751d092eea07276d3855633a3fd346b Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 13:37:20 +0800 Subject: [PATCH 141/150] fix(xlsx): tolerate unsupported drawing objects --- xlsx/read_drawing_xml.mbt | 48 +++++++++++++++++++++++++++++++----- xlsx/read_xml_namespaces.mbt | 35 ++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 591f0f22..9a84bf34 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -105,6 +105,16 @@ fn drawing_axis_span_emu( span.to_int() } +///| +fn drawing_anchor_has_unsupported_reader_object(xml : StringView) -> Bool { + // Namespace-aware structure validation has already proved that these + // canonical names can only designate the direct unsupported object (or a + // descendant of its group subtree), so lexical filtering is safe here. + xml.contains(" raise InvalidXml(msg="oneCellAnchor close missing") } let slice = text[:end] - anchors.push((" TwoCell } - anchors.push((full, positioning)) - parsed += 1 + if !drawing_anchor_has_unsupported_reader_object(full) { + anchors.push((full, positioning)) + parsed += 1 + } } // Worksheet drawing readers currently model cell-relative geometry. Keep // absolute anchors out of those consumers so a drawing they previously @@ -203,8 +218,11 @@ fn parse_drawing_anchors( None => raise InvalidXml(msg="absoluteAnchor close missing") } let slice = text[:end] - anchors.push((" + #| 0000 + #| 1000 + #| + let drawing = canonicalize_xlsx_drawing_xml( + source, default_max_xml_part_bytes, + ) + assert_true(drawing.contains("")) + let anchors = parse_drawing_anchors(drawing) + inspect(anchors.length(), content="1") + let (anchor, _) = anchors[0] + assert_true(anchor.contains("name=\"direct\"")) + assert_false(anchor.contains("name=\"nested\"")) +} + ///| test "DrawingML rejects extension entries outside extension lists" { let source = diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index c0f49f17..26d6ac72 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -93,6 +93,11 @@ fn xlsx_drawing_reader_object_name(local_name : StringView) -> Bool { local_name == "pic" || local_name == "sp" || local_name == "graphicFrame" } +///| +fn xlsx_drawing_unsupported_reader_object_name(local_name : StringView) -> Bool { + local_name == "grpSp" || local_name == "cxnSp" || local_name == "contentPart" +} + ///| fn drawing_reader_relationship_attribute( scanner : @ooxml.XmlStartTagScanner, @@ -196,6 +201,7 @@ fn validate_xlsx_drawing_reader_structure( let mut text_body_seen = false let mut graphic_seen = false let mut graphic_data_seen = false + let mut unsupported_reader_object_depth : Int? = None while workbook_scanner_next(scanner) { let depth = scanner.depth() let local_name = scanner.local_name() @@ -240,6 +246,18 @@ fn validate_xlsx_drawing_reader_structure( namespace_uri == other_chart_namespace { raise InvalidXml(msg="drawing reader target dialect is mixed") } + let inside_unsupported_reader_object = match + unsupported_reader_object_depth { + Some(start) if depth > start => true + Some(_) => { + unsupported_reader_object_depth = None + false + } + None => false + } + if inside_unsupported_reader_object { + continue + } if local_name == "ext" && xlsx_drawing_extension_list_namespaces.any(candidate => { candidate == namespace_uri @@ -294,6 +312,23 @@ fn validate_xlsx_drawing_reader_structure( text_body_seen = false graphic_seen = false graphic_data_seen = false + } else if namespace_uri == drawing_namespace && + xlsx_drawing_unsupported_reader_object_name(local_name) { + if depth != 3 || + !in_reader_anchor || + !xlsx_drawing_anchor_name(scanner.parent_local_name().unwrap_or("")) || + scanner.parent_namespace_uri() != Some(drawing_namespace) { + raise InvalidXml(msg="unsupported drawing reader object is misplaced") + } + if reader_object_seen { + raise InvalidXml(msg="drawing anchor has multiple reader objects") + } + // These are valid anchor payloads, but the semantic readers below do + // not model them. Ignore the complete subtree so nested `sp`, `pic`, or + // `graphicFrame` elements cannot be mistaken for direct anchor objects. + reader_object_seen = true + current_reader_object = "" + unsupported_reader_object_depth = Some(depth) } else if namespace_uri == drawing_namespace && ( local_name == "from" || From 5036e266d13077fec4b54a74f811c635b36b4cff Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 13:47:21 +0800 Subject: [PATCH 142/150] fix(xlsx): accept valid DrawingML rich text --- xlsx/read_drawing_xml.mbt | 19 ++++++++----------- xlsx/read_xml_namespaces.mbt | 12 ++++++++++++ xlsx/shape_form_control_slicer_test.mbt | 10 ++++++++++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 9a84bf34..19633677 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -144,9 +144,8 @@ fn parse_drawing_anchors( } } let text = chunk.to_owned() - let close_tag = "" - let end = match text.find(close_tag) { - Some(pos) => pos + close_tag.length() + let end = match find_xml_close_tag_from(text, "xdr:oneCellAnchor", 0) { + Some((_, close_end)) => close_end + 1 None => raise InvalidXml(msg="oneCellAnchor close missing") } let slice = text[:end] @@ -169,9 +168,8 @@ fn parse_drawing_anchors( } } let text = chunk.to_owned() - let close_tag = "" - let end = match text.find(close_tag) { - Some(pos) => pos + close_tag.length() + let end = match find_xml_close_tag_from(text, "xdr:twoCellAnchor", 0) { + Some((_, close_end)) => close_end + 1 None => raise InvalidXml(msg="twoCellAnchor close missing") } let slice = text[:end] @@ -212,9 +210,8 @@ fn parse_drawing_anchors( } } let text = chunk.to_owned() - let close_tag = "" - let end = match text.find(close_tag) { - Some(pos) => pos + close_tag.length() + let end = match find_xml_close_tag_from(text, "xdr:absoluteAnchor", 0) { + Some((_, close_end)) => close_end + 1 None => raise InvalidXml(msg="absoluteAnchor close missing") } let slice = text[:end] @@ -1531,9 +1528,9 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { let xml = #| - #| + #| #| - #| + #| #| let worksheet_anchors = parse_drawing_anchors(xml) inspect(worksheet_anchors.length(), content="2") diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 26d6ac72..718bd753 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -258,6 +258,18 @@ fn validate_xlsx_drawing_reader_structure( if inside_unsupported_reader_object { continue } + // Rich-text run properties use the same DrawingML element names as shape + // styling (for example a:rPr/a:solidFill/a:srgbClr), but they are not + // inputs to the shape-style reader. Keep dialect checks above active and + // ignore only same-dialect DrawingML descendants of the validated txBody; + // xdr/c/sle descendants must still be rejected as reader decoys. + let inside_shape_text_body = match text_body_depth { + Some(start) => depth > start + None => false + } + if inside_shape_text_body && namespace_uri == main_namespace { + continue + } if local_name == "ext" && xlsx_drawing_extension_list_namespaces.any(candidate => { candidate == namespace_uri diff --git a/xlsx/shape_form_control_slicer_test.mbt b/xlsx/shape_form_control_slicer_test.mbt index 987b7d15..6d6329ac 100644 --- a/xlsx/shape_form_control_slicer_test.mbt +++ b/xlsx/shape_form_control_slicer_test.mbt @@ -102,6 +102,16 @@ test "shape size and rich text paragraphs" { inspect(drawing_xml.contains("sz=\"1800\""), content="true") inspect(drawing_xml.contains("typeface=\"Times New Roman\""), content="true") inspect(drawing_xml.contains("val=\"2980B9\""), content="true") + let parsed = @xlsx.read(bytes) + let parsed_sheet = match parsed.sheet("Sheet1") { + Some(value) => value + None => fail("missing parsed sheet") + } + inspect(parsed_sheet.shapes().length(), content="1") + let parsed_shape = parsed_sheet.shapes()[0] + inspect(parsed_shape.paragraph.length(), content="2") + inspect(parsed_shape.paragraph[0].text, content="Rectangle") + inspect(parsed_shape.paragraph[1].text, content="Shape") } ///| From df452a31f21a13e376b074fdaf7ebbcae333fd33 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 14:02:05 +0800 Subject: [PATCH 143/150] fix(xlsx): preserve native absolute drawing geometry --- xlsx/read.mbt | 1 - xlsx/read_drawing_xml.mbt | 736 ++++++++++++++++------------------- xlsx/read_xml_namespaces.mbt | 18 + xlsx/shape_read_test.mbt | 61 +++ xlsx/write.mbt | 28 +- 5 files changed, 427 insertions(+), 417 deletions(-) diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 4ce75877..0bf41f01 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -5033,7 +5033,6 @@ fn read_zip_archive_core_unchecked( let drawing_anchors = parse_drawing_anchors( drawing_xml, budget=read_budget, - include_absolute=true, ) let chart_rel_id = single_drawing_chart_rel_id( drawing_anchors, diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 19633677..4f5c661d 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -115,12 +115,185 @@ fn drawing_anchor_has_unsupported_reader_object(xml : StringView) -> Bool { xml.contains(" Int raise XlsxError { + match attr_value(tag, name) { + Some(value) => + @string.parse_int(value, base=10) catch { + _ => raise InvalidXml(msg=invalid_message) + } + None => raise InvalidXml(msg=missing_message) + } +} + +///| +fn DrawingAnchor::origin_for_read( + self : DrawingAnchor, +) -> DrawingAnchorOrigin raise XlsxError { + if self.native_absolute { + let pos_tag = match tag_attributes_in(self.xml, "xdr:pos") { + Some(value) => value + None => raise InvalidXml(msg="drawing absolute position missing") + } + let x = parse_drawing_int_attr( + pos_tag, "x", "drawing absolute x invalid", "drawing absolute x missing", + ) + let y = parse_drawing_int_attr( + pos_tag, "y", "drawing absolute y invalid", "drawing absolute y missing", + ) + { + reference: drawing_reference_for_read(0, 0, y, x), + col_idx0: 0, + row_idx0: 0, + col_off_emu: x, + row_off_emu: y, + } + } else { + let from_body = match extract_tag_body_from(self.xml, "xdr:from") { + Some(value) => value + None => raise InvalidXml(msg="drawing anchor from missing") + } + let col_idx0 = parse_xml_int_tag( + from_body, "xdr:col", "drawing col invalid", + ) + let row_idx0 = parse_xml_int_tag( + from_body, "xdr:row", "drawing row invalid", + ) + let col_off_emu = parse_xml_int_tag( + from_body, "xdr:colOff", "drawing colOff invalid", + ) + let row_off_emu = parse_xml_int_tag( + from_body, "xdr:rowOff", "drawing rowOff invalid", + ) + { + reference: drawing_reference_for_read( + row_idx0, col_idx0, row_off_emu, col_off_emu, + ), + col_idx0, + row_idx0, + col_off_emu, + row_off_emu, + } + } +} + +///| +fn DrawingAnchor::uses_direct_extent(self : DrawingAnchor) -> Bool { + if self.native_absolute { + true + } else { + match self.positioning { + OneCell => true + TwoCell | Absolute => false + } + } +} + +///| +fn DrawingAnchor::extent_for_read( + self : DrawingAnchor, + metrics : DrawingMetrics, + origin : DrawingAnchorOrigin, + object_name? : String = "", +) -> (Int, Int) raise XlsxError { + let prefix = if object_name == "" { + "drawing" + } else { + "drawing " + object_name + } + if self.uses_direct_extent() { + let ext_tag = match tag_attributes_in(self.xml, "xdr:ext") { + Some(value) => value + None => raise InvalidXml(msg=prefix + " ext missing") + } + let cx = parse_drawing_int_attr( + ext_tag, + "cx", + prefix + " cx invalid", + prefix + " cx missing", + ) + let cy = parse_drawing_int_attr( + ext_tag, + "cy", + prefix + " cy invalid", + prefix + " cy missing", + ) + ignore(validate_drawing_extent_for_read(cx, prefix + " cx")) + ignore(validate_drawing_extent_for_read(cy, prefix + " cy")) + (cx, cy) + } else { + let to_body = match extract_tag_body_from(self.xml, "xdr:to") { + Some(value) => value + None => { + let target = if object_name == "" { "drawing anchor" } else { prefix } + raise InvalidXml(msg=target + " to missing") + } + } + let to_col_idx0 = parse_xml_int_tag( + to_body, + "xdr:col", + prefix + " to col invalid", + ) + let to_row_idx0 = parse_xml_int_tag( + to_body, + "xdr:row", + prefix + " to row invalid", + ) + let to_col_off = parse_xml_int_tag( + to_body, + "xdr:colOff", + prefix + " to colOff invalid", + ) + let to_row_off = parse_xml_int_tag( + to_body, + "xdr:rowOff", + prefix + " to rowOff invalid", + ) + ( + drawing_axis_span_emu( + metrics.cols, + origin.col_idx0, + origin.col_off_emu, + to_col_idx0, + to_col_off, + ), + drawing_axis_span_emu( + metrics.rows, + origin.row_idx0, + origin.row_off_emu, + to_row_idx0, + to_row_off, + ), + ) + } +} + ///| fn parse_drawing_anchors( drawing_xml : StringView, budget? : ReadBudget, - include_absolute? : Bool = false, -) -> Array[(String, PicturePositioning)] raise XlsxError { +) -> Array[DrawingAnchor] raise XlsxError { match budget { Some(value) => { value.checkpoint() @@ -129,98 +302,66 @@ fn parse_drawing_anchors( None => () } let xml_str = drawing_xml.to_owned() - let anchors : Array[(String, PicturePositioning)] = [] - let mut parsed = 0 - let mut first_one = true - for chunk in xml_str.split(" value.checkpoint() + let anchors : Array[DrawingAnchor] = [] + let anchor_names : ReadOnlyArray[String] = [ + "xdr:oneCellAnchor", "xdr:twoCellAnchor", "xdr:absoluteAnchor", + ] + let mut cursor = 0 + let mut scanned = 0 + for ;; { + let mut next_start : Int? = None + let mut next_name = "" + for candidate in anchor_names { + match find_xml_open_tag_start_from(xml_str, candidate, cursor) { + Some(start) => + match next_start { + Some(current) if current <= start => () + _ => { + next_start = Some(start) + next_name = candidate + } + } None => () } } - let text = chunk.to_owned() - let end = match find_xml_close_tag_from(text, "xdr:oneCellAnchor", 0) { - Some((_, close_end)) => close_end + 1 - None => raise InvalidXml(msg="oneCellAnchor close missing") - } - let slice = text[:end] - let full = " value + None => break } - if (parsed & 255) == 0 { + if (scanned & 255) == 0 { match budget { Some(value) => value.checkpoint() None => () } } - let text = chunk.to_owned() - let end = match find_xml_close_tag_from(text, "xdr:twoCellAnchor", 0) { - Some((_, close_end)) => close_end + 1 - None => raise InvalidXml(msg="twoCellAnchor close missing") + let close_end = match find_xml_close_tag_from(xml_str, next_name, start) { + Some((_, value)) => value + None => { + let local_name = next_name[4:].to_owned() + raise InvalidXml(msg=local_name + " close missing") + } } - let slice = text[:end] - let full = " - match attr_value(tag, "editAs") { - Some(value) => - if value.to_string() == "absolute" { - Absolute - } else { - TwoCell - } - None => TwoCell - } - None => TwoCell + let full = xml_str[start:close_end + 1].to_owned() + let native_absolute = next_name == "xdr:absoluteAnchor" + let positioning = if native_absolute { + Absolute + } else if next_name == "xdr:oneCellAnchor" { + OneCell + } else { + match tag_attributes_in(full, "xdr:twoCellAnchor") { + Some(tag) => + match attr_value(tag, "editAs") { + Some(value) if value == "absolute" => Absolute + _ => TwoCell + } + None => TwoCell + } } if !drawing_anchor_has_unsupported_reader_object(full) { - anchors.push((full, positioning)) - parsed += 1 - } - } - // Worksheet drawing readers currently model cell-relative geometry. Keep - // absolute anchors out of those consumers so a drawing they previously - // ignored does not become a hard read failure. Chart sheets opt in because - // their exact chart relationship selection is position-independent. - if include_absolute { - let mut first_absolute = true - for chunk in xml_str.split(" value.checkpoint() - None => () - } - } - let text = chunk.to_owned() - let end = match find_xml_close_tag_from(text, "xdr:absoluteAnchor", 0) { - Some((_, close_end)) => close_end + 1 - None => raise InvalidXml(msg="absoluteAnchor close missing") - } - let slice = text[:end] - let full = " value.checkpoint() @@ -249,12 +390,12 @@ fn charge_drawing_anchor_pass( /// path and relationship-attribute dialect; this pass enforces chart-sheet /// cardinality and keeps relationship lookup tied to that exact payload. fn single_drawing_chart_rel_id( - anchors : ArrayView[(String, PicturePositioning)], + anchors : ArrayView[DrawingAnchor], budget? : ReadBudget, ) -> String raise XlsxError { let mut selected : String? = None - for entry in anchors { - let (anchor_xml, _) = entry + for anchor in anchors { + let anchor_xml = anchor.xml charge_drawing_anchor_pass(budget, anchor_xml) match tag_attributes_in(anchor_xml, "c:chart") { Some(tag) => { @@ -293,7 +434,7 @@ fn drawing_shape_solid_fill_body(sppr_body : StringView) -> String? { ///| fn parse_drawing_images( - anchors : ArrayView[(String, PicturePositioning)], + anchors : ArrayView[DrawingAnchor], drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], @@ -324,92 +465,19 @@ fn parse_drawing_images( cancelled~, ) } - for entry in anchors { - let (anchor_xml, positioning) = entry + for anchor in anchors { + let anchor_xml = anchor.xml + let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) let pic_body = match extract_tag_body_from(anchor_xml, "xdr:pic") { Some(value) => value None => continue } - let from_body = match extract_tag_body_from(anchor_xml, "xdr:from") { - Some(value) => value - None => raise InvalidXml(msg="drawing anchor from missing") - } - let col_idx0 = parse_xml_int_tag( - from_body, "xdr:col", "drawing col invalid", - ) - let row_idx0 = parse_xml_int_tag( - from_body, "xdr:row", "drawing row invalid", - ) - let col_off_emu = parse_xml_int_tag( - from_body, "xdr:colOff", "drawing colOff invalid", - ) - let row_off_emu = parse_xml_int_tag( - from_body, "xdr:rowOff", "drawing rowOff invalid", - ) - let reference = drawing_reference_for_read( - row_idx0, col_idx0, row_off_emu, col_off_emu, - ) - let offset_x = col_off_emu / picture_emu_per_pixel - let offset_y = row_off_emu / picture_emu_per_pixel - let (width_emu, height_emu) = match positioning { - OneCell => { - let ext_tag = match tag_attributes_in(anchor_xml, "xdr:ext") { - Some(value) => value - None => raise InvalidXml(msg="drawing ext missing") - } - let cx = match attr_value(ext_tag, "cx") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing cx invalid") - } - None => raise InvalidXml(msg="drawing cx missing") - } - let cy = match attr_value(ext_tag, "cy") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing cy invalid") - } - None => raise InvalidXml(msg="drawing cy missing") - } - ignore(validate_drawing_extent_for_read(cx, "drawing cx")) - ignore(validate_drawing_extent_for_read(cy, "drawing cy")) - (cx, cy) - } - _ => { - let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { - Some(value) => value - None => raise InvalidXml(msg="drawing anchor to missing") - } - let to_col_idx0 = parse_xml_int_tag( - to_body, "xdr:col", "drawing to col invalid", - ) - let to_row_idx0 = parse_xml_int_tag( - to_body, "xdr:row", "drawing to row invalid", - ) - let to_col_off = parse_xml_int_tag( - to_body, "xdr:colOff", "drawing to colOff invalid", - ) - let to_row_off = parse_xml_int_tag( - to_body, "xdr:rowOff", "drawing to rowOff invalid", - ) - let cx = drawing_axis_span_emu( - metrics.cols, - col_idx0, - col_off_emu, - to_col_idx0, - to_col_off, - ) - let cy = drawing_axis_span_emu( - metrics.rows, - row_idx0, - row_off_emu, - to_row_idx0, - to_row_off, - ) - (cx, cy) - } - } + let origin = anchor.origin_for_read() + let reference = origin.reference + let offset_x = origin.col_off_emu / picture_emu_per_pixel + let offset_y = origin.row_off_emu / picture_emu_per_pixel + let (width_emu, height_emu) = anchor.extent_for_read(metrics, origin) let c_nv_pr_tag = match tag_attributes_in(pic_body, "xdr:cNvPr") { Some(value) => value None => raise InvalidXml(msg="drawing cNvPr missing") @@ -510,7 +578,7 @@ fn parse_drawing_images( ///| fn parse_drawing_charts( - anchors : ArrayView[(String, PicturePositioning)], + anchors : ArrayView[DrawingAnchor], drawing_rels_xml : StringView, drawing_part : StringView, part_names : Map[String, String], @@ -532,8 +600,9 @@ fn parse_drawing_charts( cancelled~, ) } - for entry in anchors { - let (anchor_xml, positioning) = entry + for anchor in anchors { + let anchor_xml = anchor.xml + let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) let chart_tag = match tag_attributes_in(anchor_xml, "c:chart") { Some(value) => value @@ -565,85 +634,11 @@ fn parse_drawing_charts( None => raise MissingPart(path=chart_path) } let xml = decode(chart_bytes) - let from_body = match extract_tag_body_from(anchor_xml, "xdr:from") { - Some(value) => value - None => raise InvalidXml(msg="drawing anchor from missing") - } - let col_idx0 = parse_xml_int_tag( - from_body, "xdr:col", "drawing col invalid", - ) - let row_idx0 = parse_xml_int_tag( - from_body, "xdr:row", "drawing row invalid", - ) - let col_off_emu = parse_xml_int_tag( - from_body, "xdr:colOff", "drawing colOff invalid", - ) - let row_off_emu = parse_xml_int_tag( - from_body, "xdr:rowOff", "drawing rowOff invalid", - ) - let reference = drawing_reference_for_read( - row_idx0, col_idx0, row_off_emu, col_off_emu, - ) - let offset_x = col_off_emu / picture_emu_per_pixel - let offset_y = row_off_emu / picture_emu_per_pixel - let (width_emu, height_emu) = match positioning { - OneCell => { - let ext_tag = match tag_attributes_in(anchor_xml, "xdr:ext") { - Some(value) => value - None => raise InvalidXml(msg="drawing ext missing") - } - let cx = match attr_value(ext_tag, "cx") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing cx invalid") - } - None => raise InvalidXml(msg="drawing cx missing") - } - let cy = match attr_value(ext_tag, "cy") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing cy invalid") - } - None => raise InvalidXml(msg="drawing cy missing") - } - ignore(validate_drawing_extent_for_read(cx, "drawing cx")) - ignore(validate_drawing_extent_for_read(cy, "drawing cy")) - (cx, cy) - } - _ => { - let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { - Some(value) => value - None => raise InvalidXml(msg="drawing anchor to missing") - } - let to_col_idx0 = parse_xml_int_tag( - to_body, "xdr:col", "drawing to col invalid", - ) - let to_row_idx0 = parse_xml_int_tag( - to_body, "xdr:row", "drawing to row invalid", - ) - let to_col_off = parse_xml_int_tag( - to_body, "xdr:colOff", "drawing to colOff invalid", - ) - let to_row_off = parse_xml_int_tag( - to_body, "xdr:rowOff", "drawing to rowOff invalid", - ) - let cx = drawing_axis_span_emu( - metrics.cols, - col_idx0, - col_off_emu, - to_col_idx0, - to_col_off, - ) - let cy = drawing_axis_span_emu( - metrics.rows, - row_idx0, - row_off_emu, - to_row_idx0, - to_row_off, - ) - (cx, cy) - } - } + let origin = anchor.origin_for_read() + let reference = origin.reference + let offset_x = origin.col_off_emu / picture_emu_per_pixel + let offset_y = origin.row_off_emu / picture_emu_per_pixel + let (width_emu, height_emu) = anchor.extent_for_read(metrics, origin) let client_tag = match tag_attributes_in(anchor_xml, "xdr:clientData") { Some(value) => value None => "" @@ -673,12 +668,13 @@ fn parse_drawing_charts( ///| fn parse_drawing_shapes( - anchors : ArrayView[(String, PicturePositioning)], + anchors : ArrayView[DrawingAnchor], budget? : ReadBudget, ) -> Array[Shape] raise XlsxError { let shapes : Array[Shape] = [] - for entry in anchors { - let (anchor_xml, positioning) = entry + for anchor in anchors { + let anchor_xml = anchor.xml + let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) let sp_body = match extract_tag_body_from(anchor_xml, "xdr:sp") { Some(value) => value @@ -687,51 +683,31 @@ fn parse_drawing_shapes( if !sp_body.contains(" value - None => raise InvalidXml(msg="drawing anchor from missing") - } - let col_idx0 = parse_xml_int_tag( - from_body, "xdr:col", "drawing col invalid", - ) - let row_idx0 = parse_xml_int_tag( - from_body, "xdr:row", "drawing row invalid", - ) - let col_off_emu = parse_xml_int_tag( - from_body, "xdr:colOff", "drawing colOff invalid", - ) - let row_off_emu = parse_xml_int_tag( - from_body, "xdr:rowOff", "drawing rowOff invalid", - ) - let reference = drawing_reference_for_read( - row_idx0, col_idx0, row_off_emu, col_off_emu, - ) - match positioning { - OneCell => () - _ => { - let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { - Some(value) => value - None => raise InvalidXml(msg="drawing shape to missing") - } - let to_col_idx0 = parse_xml_int_tag( - to_body, "xdr:col", "drawing shape to col invalid", - ) - let to_row_idx0 = parse_xml_int_tag( - to_body, "xdr:row", "drawing shape to row invalid", - ) - let to_col_off = parse_xml_int_tag( - to_body, "xdr:colOff", "drawing shape to colOff invalid", - ) - let to_row_off = parse_xml_int_tag( - to_body, "xdr:rowOff", "drawing shape to rowOff invalid", - ) - validate_drawing_axis_marker_for_read( - "col", to_col_idx0, to_col_off, true, - ) - validate_drawing_axis_marker_for_read( - "row", to_row_idx0, to_row_off, true, - ) + let origin = anchor.origin_for_read() + let reference = origin.reference + if !anchor.uses_direct_extent() { + let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { + Some(value) => value + None => raise InvalidXml(msg="drawing shape to missing") } + let to_col_idx0 = parse_xml_int_tag( + to_body, "xdr:col", "drawing shape to col invalid", + ) + let to_row_idx0 = parse_xml_int_tag( + to_body, "xdr:row", "drawing shape to row invalid", + ) + let to_col_off = parse_xml_int_tag( + to_body, "xdr:colOff", "drawing shape to colOff invalid", + ) + let to_row_off = parse_xml_int_tag( + to_body, "xdr:rowOff", "drawing shape to rowOff invalid", + ) + validate_drawing_axis_marker_for_read( + "col", to_col_idx0, to_col_off, true, + ) + validate_drawing_axis_marker_for_read( + "row", to_row_idx0, to_row_off, true, + ) } let c_nv_pr_tag = match tag_attributes_in(sp_body, "xdr:cNvPr") { Some(value) => value @@ -765,11 +741,16 @@ fn parse_drawing_shapes( Some(value) => value None => raise InvalidXml(msg="drawing shape spPr missing") } - let xfrm_body = match extract_tag_body_from(sppr_body, "a:xfrm") { - Some(value) => value - None => sppr_body + let maybe_ext_tag = if anchor.native_absolute { + tag_attributes_in(anchor_xml, "xdr:ext") + } else { + let xfrm_body = match extract_tag_body_from(sppr_body, "a:xfrm") { + Some(value) => value + None => sppr_body + } + tag_attributes_in(xfrm_body, "a:ext") } - let ext_tag = match tag_attributes_in(xfrm_body, "a:ext") { + let ext_tag = match maybe_ext_tag { Some(value) => value None => raise InvalidXml(msg="drawing shape ext missing") } @@ -891,6 +872,14 @@ fn parse_drawing_shapes( let format = shape_graphic_options( 1.0, 1.0, name, alt_text, print_object, locked, positioning, ) + let offset_x = origin.col_off_emu / picture_emu_per_pixel + let offset_y = origin.row_off_emu / picture_emu_per_pixel + if offset_x != 0 { + format.offset_x = Some(offset_x) + } + if offset_y != 0 { + format.offset_y = Some(offset_y) + } let fill = shape_fill_from_values(fill_color, fill_transparency) let line = shape_line_from_values(line_color, line_width) shapes.push({ @@ -938,13 +927,14 @@ priv struct SlicerAnchorInfo { ///| fn parse_drawing_slicer_anchors( - anchors : ArrayView[(String, PicturePositioning)], + anchors : ArrayView[DrawingAnchor], metrics : DrawingMetrics, budget? : ReadBudget, ) -> Map[String, SlicerAnchorInfo] raise XlsxError { let out : Map[String, SlicerAnchorInfo] = Map([]) - for entry in anchors { - let (anchor_xml, positioning) = entry + for anchor in anchors { + let anchor_xml = anchor.xml + let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) let slicer_tag = match tag_attributes_in(anchor_xml, "sle:slicer") { Some(value) => value @@ -954,85 +944,15 @@ fn parse_drawing_slicer_anchors( Some(value) => unescape_xml_text(value) None => continue } - let from_body = match extract_tag_body_from(anchor_xml, "xdr:from") { - Some(value) => value - None => raise InvalidXml(msg="drawing slicer from missing") - } - let col_idx0 = parse_xml_int_tag( - from_body, "xdr:col", "drawing slicer col invalid", - ) - let row_idx0 = parse_xml_int_tag( - from_body, "xdr:row", "drawing slicer row invalid", - ) - let col_off_emu = parse_xml_int_tag( - from_body, "xdr:colOff", "drawing slicer colOff invalid", - ) - let row_off_emu = parse_xml_int_tag( - from_body, "xdr:rowOff", "drawing slicer rowOff invalid", - ) - let cell = drawing_reference_for_read( - row_idx0, col_idx0, row_off_emu, col_off_emu, + let origin = anchor.origin_for_read() + let cell = origin.reference + let offset_x = origin.col_off_emu / picture_emu_per_pixel + let offset_y = origin.row_off_emu / picture_emu_per_pixel + let (width_emu, height_emu) = anchor.extent_for_read( + metrics, + origin, + object_name="slicer", ) - let offset_x = col_off_emu / picture_emu_per_pixel - let offset_y = row_off_emu / picture_emu_per_pixel - let (width_emu, height_emu) = match positioning { - OneCell => { - let ext_tag = match tag_attributes_in(anchor_xml, "xdr:ext") { - Some(value) => value - None => raise InvalidXml(msg="drawing slicer ext missing") - } - let cx = match attr_value(ext_tag, "cx") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing slicer cx invalid") - } - None => raise InvalidXml(msg="drawing slicer cx missing") - } - let cy = match attr_value(ext_tag, "cy") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing slicer cy invalid") - } - None => raise InvalidXml(msg="drawing slicer cy missing") - } - ignore(validate_drawing_extent_for_read(cx, "drawing slicer cx")) - ignore(validate_drawing_extent_for_read(cy, "drawing slicer cy")) - (cx, cy) - } - _ => { - let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { - Some(value) => value - None => raise InvalidXml(msg="drawing slicer to missing") - } - let to_col_idx0 = parse_xml_int_tag( - to_body, "xdr:col", "drawing slicer to col invalid", - ) - let to_row_idx0 = parse_xml_int_tag( - to_body, "xdr:row", "drawing slicer to row invalid", - ) - let to_col_off = parse_xml_int_tag( - to_body, "xdr:colOff", "drawing slicer to colOff invalid", - ) - let to_row_off = parse_xml_int_tag( - to_body, "xdr:rowOff", "drawing slicer to rowOff invalid", - ) - let cx = drawing_axis_span_emu( - metrics.cols, - col_idx0, - col_off_emu, - to_col_idx0, - to_col_off, - ) - let cy = drawing_axis_span_emu( - metrics.rows, - row_idx0, - row_off_emu, - to_row_idx0, - to_row_off, - ) - (cx, cy) - } - } let width = (width_emu.to_double() / picture_emu_per_pixel.to_double()) .round() .to_int() @@ -1206,10 +1126,10 @@ test "DrawingML extension payload cannot spoof anchors or drawing features" { assert_false(drawing.contains("extLst")) let anchors = parse_drawing_anchors(drawing) inspect(anchors.length(), content="4") - let (image_anchor, _) = anchors[0] - let (chart_anchor, _) = anchors[1] - let (shape_anchor, _) = anchors[2] - let (slicer_anchor, _) = anchors[3] + let image_anchor = anchors[0].xml + let chart_anchor = anchors[1].xml + let shape_anchor = anchors[2].xml + let slicer_anchor = anchors[3].xml assert_true(image_anchor.contains("image-good")) assert_true(chart_anchor.contains("chart-good")) assert_true(shape_anchor.contains("good-shape")) @@ -1229,9 +1149,9 @@ test "DrawingML ignores valid unsupported anchor objects as complete subtrees" { assert_true(drawing.contains("")) let anchors = parse_drawing_anchors(drawing) inspect(anchors.length(), content="1") - let (anchor, _) = anchors[0] - assert_true(anchor.contains("name=\"direct\"")) - assert_false(anchor.contains("name=\"nested\"")) + let anchor_xml = anchors[0].xml + assert_true(anchor_xml.contains("name=\"direct\"")) + assert_false(anchor_xml.contains("name=\"nested\"")) } ///| @@ -1353,18 +1273,20 @@ test "DrawingML shape fill selection excludes line-only fills" { test "chartsheet drawing selector requires exactly one chart payload" { let anchor = #| - let one : ReadOnlyArray[(String, PicturePositioning)] = [(anchor, OneCell)] + let one : ReadOnlyArray[DrawingAnchor] = [ + { xml: anchor, positioning: OneCell, native_absolute: false }, + ] assert_eq(single_drawing_chart_rel_id(one), "selected") - let empty : ReadOnlyArray[(String, PicturePositioning)] = [] + let empty : ReadOnlyArray[DrawingAnchor] = [] try single_drawing_chart_rel_id(empty) catch { InvalidXml(msg~) => assert_eq(msg, "chartsheet chart payload missing") _ => fail("unexpected empty chartsheet drawing error") } noraise { _ => fail("expected empty chartsheet drawing rejection") } - let two : ReadOnlyArray[(String, PicturePositioning)] = [ - (anchor, OneCell), - (anchor, OneCell), + let two : ReadOnlyArray[DrawingAnchor] = [ + { xml: anchor, positioning: OneCell, native_absolute: false }, + { xml: anchor, positioning: OneCell, native_absolute: false }, ] try single_drawing_chart_rel_id(two) catch { InvalidXml(msg~) => assert_eq(msg, "chartsheet drawing has multiple charts") @@ -1491,7 +1413,7 @@ test "read_drawing_xml wb: drawing spans reject invalid markers and overflow" { ///| test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { let one_missing_close = "" - let one_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + let one_result : Result[Array[DrawingAnchor], Error] = Ok( parse_drawing_anchors(one_missing_close), ) catch { e => Err(e) @@ -1503,7 +1425,7 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { } let two_missing_close = "" - let two_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + let two_result : Result[Array[DrawingAnchor], Error] = Ok( parse_drawing_anchors(two_missing_close), ) catch { e => Err(e) @@ -1515,8 +1437,8 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { } let absolute_missing_close = "" - let absolute_result : Result[Array[(String, PicturePositioning)], Error] = Ok( - parse_drawing_anchors(absolute_missing_close, include_absolute=true), + let absolute_result : Result[Array[DrawingAnchor], Error] = Ok( + parse_drawing_anchors(absolute_missing_close), ) catch { e => Err(e) } @@ -1528,17 +1450,19 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { let xml = #| + #| + #| #| #| - #| #| - let worksheet_anchors = parse_drawing_anchors(xml) - inspect(worksheet_anchors.length(), content="2") - let anchors = parse_drawing_anchors(xml, include_absolute=true) - inspect(anchors.length(), content="3") - debug_inspect(anchors[0].1, content="Absolute") - debug_inspect(anchors[1].1, content="TwoCell") - debug_inspect(anchors[2].1, content="Absolute") + let anchors = parse_drawing_anchors(xml) + inspect(anchors.length(), content="4") + debug_inspect(anchors[0].positioning, content="Absolute") + debug_inspect(anchors[1].positioning, content="OneCell") + debug_inspect(anchors[2].positioning, content="Absolute") + debug_inspect(anchors[3].positioning, content="TwoCell") + inspect(anchors[0].native_absolute, content="true") + inspect(anchors[2].native_absolute, content="false") } ///| @@ -1547,7 +1471,7 @@ test "read_drawing_xml wb: anchor parsing checks limits before copying source" { let limited = ReadBudget::new( ReadLimits::with_values(max_parser_work_units=1), ) - let limit_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + let limit_result : Result[Array[DrawingAnchor], Error] = Ok( parse_drawing_anchors(drawing_xml, budget=limited), ) catch { error => Err(error) @@ -1566,7 +1490,7 @@ test "read_drawing_xml wb: anchor parsing checks limits before copying source" { checks += 1 true }) - let cancel_result : Result[Array[(String, PicturePositioning)], Error] = Ok( + let cancel_result : Result[Array[DrawingAnchor], Error] = Ok( parse_drawing_anchors(drawing_xml, budget=cancelled), ) catch { error => Err(error) diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 718bd753..a36d2489 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -180,6 +180,7 @@ fn validate_xlsx_drawing_reader_structure( let mut marker_row_off_seen = false let mut from_seen = false let mut to_seen = false + let mut anchor_pos_seen = false let mut anchor_ext_seen = false let mut client_data_seen = false let mut non_visual_container_seen = false @@ -226,6 +227,7 @@ fn validate_xlsx_drawing_reader_structure( reader_payload_seen = false from_seen = false to_seen = false + anchor_pos_seen = false anchor_ext_seen = false client_data_seen = false } else if depth <= 3 { @@ -290,6 +292,7 @@ fn validate_xlsx_drawing_reader_structure( reader_payload_seen = false from_seen = false to_seen = false + anchor_pos_seen = false anchor_ext_seen = false client_data_seen = false } else if namespace_uri == drawing_namespace && @@ -345,6 +348,7 @@ fn validate_xlsx_drawing_reader_structure( ( local_name == "from" || local_name == "to" || + local_name == "pos" || local_name == "ext" || local_name == "clientData" ) { @@ -355,6 +359,9 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing anchor reader target is misplaced") } if local_name == "from" { + if anchor_name == "absoluteAnchor" { + raise InvalidXml(msg="drawing anchor from target is misplaced") + } if from_seen { raise InvalidXml(msg="drawing anchor from target is duplicated") } @@ -365,6 +372,9 @@ fn validate_xlsx_drawing_reader_structure( marker_row_seen = false marker_row_off_seen = false } else if local_name == "to" { + if anchor_name == "absoluteAnchor" { + raise InvalidXml(msg="drawing anchor to target is misplaced") + } if to_seen { raise InvalidXml(msg="drawing anchor to target is duplicated") } @@ -374,6 +384,14 @@ fn validate_xlsx_drawing_reader_structure( marker_col_off_seen = false marker_row_seen = false marker_row_off_seen = false + } else if local_name == "pos" { + if anchor_name != "absoluteAnchor" { + raise InvalidXml(msg="drawing anchor pos target is misplaced") + } + if anchor_pos_seen { + raise InvalidXml(msg="drawing anchor pos target is duplicated") + } + anchor_pos_seen = true } else if local_name == "ext" { if anchor_ext_seen { raise InvalidXml(msg="drawing anchor ext target is duplicated") diff --git a/xlsx/shape_read_test.mbt b/xlsx/shape_read_test.mbt index e40c9053..c25c2054 100644 --- a/xlsx/shape_read_test.mbt +++ b/xlsx/shape_read_test.mbt @@ -51,3 +51,64 @@ test "shape parsed on read" { inspect(got.paragraph.length() > 0, content="true") inspect(got.paragraph[0].text, content="Hello") } + +///| +test "native absolute shape geometry survives normalized roundtrip" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + workbook.add_shape("Sheet1", @xlsx.Shape::new("A1", "rect")) + let archive = @zip.read(@xlsx.write(workbook)) catch { + err => fail("zip read failed: \{repr(err)}") + } + let drawing_path = "xl/drawings/drawing1.xml" + let absolute_drawing = + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + #| + assert_true( + archive.replace(drawing_path, @encoding/utf8.encode(absolute_drawing)), + ) + let parsed = @xlsx.read(@zip.write(archive)) + let sheet = parsed.sheet("Sheet1").unwrap() + inspect(sheet.shapes().length(), content="1") + let shape = sheet.shapes()[0] + inspect(shape.reference, content="A1") + inspect(shape.width, content="10") + inspect(shape.height, content="20") + debug_inspect(shape.format.offset_x, content="Some(2)") + debug_inspect(shape.format.offset_y, content="Some(3)") + debug_inspect(shape.positioning, content="Absolute") + + let normalized = @xlsx.write(parsed) + let normalized_archive = @zip.read(normalized) catch { + err => fail("normalized zip read failed: \{repr(err)}") + } + let normalized_drawing = match normalized_archive.get(drawing_path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("normalized drawing utf8 decode failed") + } + None => fail("normalized drawing missing") + } + assert_true( + normalized_drawing.contains(""), + ) + assert_true(normalized_drawing.contains("19050")) + assert_true(normalized_drawing.contains("28575")) + let reparsed = @xlsx.read(normalized) + let reparsed_shape = reparsed.sheet("Sheet1").unwrap().shapes()[0] + inspect(reparsed_shape.width, content="10") + inspect(reparsed_shape.height, content="20") + debug_inspect(reparsed_shape.format.offset_x, content="Some(2)") + debug_inspect(reparsed_shape.format.offset_y, content="Some(3)") + debug_inspect(reparsed_shape.positioning, content="Absolute") +} diff --git a/xlsx/write.mbt b/xlsx/write.mbt index 4ca7ad3a..22dd62d3 100644 --- a/xlsx/write.mbt +++ b/xlsx/write.mbt @@ -663,6 +663,14 @@ fn write_drawing_xml( let (row, col) = cell_ref_to_rc(shape.reference) let col_idx = col - 1 let row_idx = row - 1 + let offset_x = shape.format.offset_x.unwrap_or(0) + let offset_y = shape.format.offset_y.unwrap_or(0) + let col_off = drawing_pixel_offset_for_write( + offset_x, "drawing shape column offset", + ) + let row_off = drawing_pixel_offset_for_write( + offset_y, "drawing shape row offset", + ) let shape_id = obj_id obj_id = obj_id + 1 let cx = drawing_scaled_extent_for_write( @@ -680,20 +688,20 @@ fn write_drawing_xml( sb.write_view(" \n") sb.write_view(" \n") sb.write_view(" \{col_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{col_off}\n") sb.write_view(" \{row_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{row_off}\n") sb.write_view(" \n") let (to_col_idx, to_col_off) = two_cell_axis_to( drawing_metrics.cols, col_idx, - 0, + col_off, cx, ) let (to_row_idx, to_row_off) = two_cell_axis_to( drawing_metrics.rows, row_idx, - 0, + row_off, cy, ) sb.write_view(" \n") @@ -707,20 +715,20 @@ fn write_drawing_xml( sb.write_view(" \n") sb.write_view(" \n") sb.write_view(" \{col_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{col_off}\n") sb.write_view(" \{row_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{row_off}\n") sb.write_view(" \n") let (to_col_idx, to_col_off) = two_cell_axis_to( drawing_metrics.cols, col_idx, - 0, + col_off, cx, ) let (to_row_idx, to_row_off) = two_cell_axis_to( drawing_metrics.rows, row_idx, - 0, + row_off, cy, ) sb.write_view(" \n") @@ -734,9 +742,9 @@ fn write_drawing_xml( sb.write_view(" \n") sb.write_view(" \n") sb.write_view(" \{col_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{col_off}\n") sb.write_view(" \{row_idx}\n") - sb.write_view(" 0\n") + sb.write_view(" \{row_off}\n") sb.write_view(" \n") sb.write_view(" \n") } From 514d6d6998aac5f8500aaa4f1dd645e49ff8860f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 15:38:13 +0800 Subject: [PATCH 144/150] fix(xlsx): preserve drawing anchor geometry --- xlsx/cell_images.mbt | 4 + xlsx/pkg.generated.mbti | 4 + xlsx/read.mbt | 17 +++ xlsx/read_drawing_xml.mbt | 187 +++++++++++++++---------------- xlsx/read_sheet_rel_parts.mbt | 4 + xlsx/read_xml_namespaces.mbt | 116 ++++++++++++++++++- xlsx/rich_value_images.mbt | 4 + xlsx/shape.mbt | 8 ++ xlsx/shape_read_test.mbt | 89 ++++++++++++++- xlsx/slicer.mbt | 8 ++ xlsx/workbook.mbt | 4 + xlsx/worksheet.mbt | 20 ++++ xlsx/worksheet_types.mbt | 8 ++ xlsx/write.mbt | 204 ++++++++++++++++++++++++++-------- 14 files changed, 525 insertions(+), 152 deletions(-) diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index 4fd7d983..4520a6e9 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -160,6 +160,10 @@ fn CellImage::to_image(self : CellImage, reference : String) -> Image { alt_text: self.alt_text, lock_aspect_ratio: false, positioning: OneCell, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } } diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 23f1ab17..4e91fd7b 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -274,6 +274,7 @@ pub struct Chart { print_object : Bool locked : Bool positioning : PicturePositioning + // private fields } pub(all) struct ChartAxis { @@ -1052,6 +1053,7 @@ pub struct Image { alt_text : String lock_aspect_ratio : Bool positioning : PicturePositioning + // private fields } pub struct MergeCell { @@ -1336,6 +1338,7 @@ pub struct Shape { print_object : Bool locked : Bool positioning : PicturePositioning + // private fields } derive(@debug.Debug) pub fn Shape::new(String, String, macro_name? : String, text? : String, paragraph? : Array[RichTextRun], width? : Int, height? : Int, scale_x? : Double, scale_y? : Double, fill_color? : String, fill_transparency? : Int, line_color? : String, line_width? : Double, name? : String, alt_text? : String, print_object? : Bool, locked? : Bool, positioning? : PicturePositioning) -> Self raise XlsxError #deprecated @@ -1511,6 +1514,7 @@ pub struct Slicer { display_header : Bool? item_desc : Bool format : GraphicOptions + // private fields } derive(@debug.Debug) pub fn Slicer::new(String) -> Self raise XlsxError #deprecated diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 0bf41f01..61a48c92 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4875,6 +4875,7 @@ fn read_zip_archive_core_unchecked( sheet.charts.append(drawing_charts) let drawing_shapes = parse_drawing_shapes( drawing_anchors, + drawing_metrics, budget=read_budget, ) sheet.shapes.append(drawing_shapes) @@ -4944,6 +4945,22 @@ fn read_zip_archive_core_unchecked( display_header: entry.display_header, item_desc, format, + drawing_offset_x_emu: match slicer_anchor_info.get(entry.name) { + Some(info) => Some(info.offset_x_emu) + None => None + }, + drawing_offset_y_emu: match slicer_anchor_info.get(entry.name) { + Some(info) => Some(info.offset_y_emu) + None => None + }, + drawing_width_emu: match slicer_anchor_info.get(entry.name) { + Some(info) => Some(info.width_emu) + None => None + }, + drawing_height_emu: match slicer_anchor_info.get(entry.name) { + Some(info) => Some(info.height_emu) + None => None + }, }) } workbook.sheets.push(sheet) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 4f5c661d..0f8b80c5 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -115,11 +115,18 @@ fn drawing_anchor_has_unsupported_reader_object(xml : StringView) -> Bool { xml.contains(" DrawingAnchorOrigin raise XlsxError { - if self.native_absolute { + if self.form == AbsoluteAnchor { let pos_tag = match tag_attributes_in(self.xml, "xdr:pos") { Some(value) => value None => raise InvalidXml(msg="drawing absolute position missing") @@ -200,13 +207,9 @@ fn DrawingAnchor::origin_for_read( ///| fn DrawingAnchor::uses_direct_extent(self : DrawingAnchor) -> Bool { - if self.native_absolute { - true - } else { - match self.positioning { - OneCell => true - TwoCell | Absolute => false - } + match self.form { + OneCellAnchor | AbsoluteAnchor => true + TwoCellAnchor => false } } @@ -342,23 +345,30 @@ fn parse_drawing_anchors( } } let full = xml_str[start:close_end + 1].to_owned() - let native_absolute = next_name == "xdr:absoluteAnchor" - let positioning = if native_absolute { - Absolute + let form = if next_name == "xdr:absoluteAnchor" { + AbsoluteAnchor } else if next_name == "xdr:oneCellAnchor" { - OneCell + OneCellAnchor } else { - match tag_attributes_in(full, "xdr:twoCellAnchor") { - Some(tag) => - match attr_value(tag, "editAs") { - Some(value) if value == "absolute" => Absolute - _ => TwoCell - } - None => TwoCell - } + TwoCellAnchor + } + let positioning = match form { + AbsoluteAnchor => Absolute + OneCellAnchor => OneCell + TwoCellAnchor => + match tag_attributes_in(full, "xdr:twoCellAnchor") { + Some(tag) => + match attr_value(tag, "editAs") { + Some("absolute") => Absolute + Some("oneCell") => OneCell + Some("twoCell") | None => TwoCell + Some(_) => raise InvalidXml(msg="drawing editAs value is invalid") + } + None => TwoCell + } } if !drawing_anchor_has_unsupported_reader_object(full) { - anchors.push({ xml: full, positioning, native_absolute }) + anchors.push({ xml: full, positioning, form }) } scanned += 1 cursor = close_end + 1 @@ -571,6 +581,10 @@ fn parse_drawing_images( alt_text, lock_aspect_ratio, positioning, + drawing_offset_x_emu: Some(origin.col_off_emu), + drawing_offset_y_emu: Some(origin.row_off_emu), + drawing_width_emu: Some(width_emu), + drawing_height_emu: Some(height_emu), }) } images @@ -661,6 +675,10 @@ fn parse_drawing_charts( print_object, locked, positioning, + drawing_offset_x_emu: Some(origin.col_off_emu), + drawing_offset_y_emu: Some(origin.row_off_emu), + drawing_width_emu: Some(width_emu), + drawing_height_emu: Some(height_emu), }) } charts @@ -669,6 +687,7 @@ fn parse_drawing_charts( ///| fn parse_drawing_shapes( anchors : ArrayView[DrawingAnchor], + metrics : DrawingMetrics, budget? : ReadBudget, ) -> Array[Shape] raise XlsxError { let shapes : Array[Shape] = [] @@ -685,30 +704,7 @@ fn parse_drawing_shapes( } let origin = anchor.origin_for_read() let reference = origin.reference - if !anchor.uses_direct_extent() { - let to_body = match extract_tag_body_from(anchor_xml, "xdr:to") { - Some(value) => value - None => raise InvalidXml(msg="drawing shape to missing") - } - let to_col_idx0 = parse_xml_int_tag( - to_body, "xdr:col", "drawing shape to col invalid", - ) - let to_row_idx0 = parse_xml_int_tag( - to_body, "xdr:row", "drawing shape to row invalid", - ) - let to_col_off = parse_xml_int_tag( - to_body, "xdr:colOff", "drawing shape to colOff invalid", - ) - let to_row_off = parse_xml_int_tag( - to_body, "xdr:rowOff", "drawing shape to rowOff invalid", - ) - validate_drawing_axis_marker_for_read( - "col", to_col_idx0, to_col_off, true, - ) - validate_drawing_axis_marker_for_read( - "row", to_row_idx0, to_row_off, true, - ) - } + let (cx, cy) = anchor.extent_for_read(metrics, origin, object_name="shape") let c_nv_pr_tag = match tag_attributes_in(sp_body, "xdr:cNvPr") { Some(value) => value None => raise InvalidXml(msg="drawing shape cNvPr missing") @@ -741,35 +737,6 @@ fn parse_drawing_shapes( Some(value) => value None => raise InvalidXml(msg="drawing shape spPr missing") } - let maybe_ext_tag = if anchor.native_absolute { - tag_attributes_in(anchor_xml, "xdr:ext") - } else { - let xfrm_body = match extract_tag_body_from(sppr_body, "a:xfrm") { - Some(value) => value - None => sppr_body - } - tag_attributes_in(xfrm_body, "a:ext") - } - let ext_tag = match maybe_ext_tag { - Some(value) => value - None => raise InvalidXml(msg="drawing shape ext missing") - } - let cx = match attr_value(ext_tag, "cx") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing shape cx invalid") - } - None => raise InvalidXml(msg="drawing shape cx missing") - } - let cy = match attr_value(ext_tag, "cy") { - Some(value) => - @string.parse_int(value, base=10) catch { - _ => raise InvalidXml(msg="drawing shape cy invalid") - } - None => raise InvalidXml(msg="drawing shape cy missing") - } - ignore(validate_drawing_extent_for_read(cx, "drawing shape cx")) - ignore(validate_drawing_extent_for_read(cy, "drawing shape cy")) let width = (cx.to_double() / picture_emu_per_pixel.to_double()) .round() .to_int() @@ -906,6 +873,10 @@ fn parse_drawing_shapes( print_object, locked, positioning, + drawing_offset_x_emu: Some(origin.col_off_emu), + drawing_offset_y_emu: Some(origin.row_off_emu), + drawing_width_emu: Some(cx), + drawing_height_emu: Some(cy), }) } shapes @@ -923,6 +894,10 @@ priv struct SlicerAnchorInfo { alt_text : String locked : Bool print_object : Bool + offset_x_emu : Int + offset_y_emu : Int + width_emu : Int + height_emu : Int } ///| @@ -999,6 +974,10 @@ fn parse_drawing_slicer_anchors( alt_text, locked, print_object, + offset_x_emu: origin.col_off_emu, + offset_y_emu: origin.row_off_emu, + width_emu, + height_emu, } } out @@ -1006,7 +985,7 @@ fn parse_drawing_slicer_anchors( ///| fn wb_from_xml() -> String { - "0000" + "0000" } ///| @@ -1017,13 +996,13 @@ fn wb_pic_xml(embed_id : String) -> String { ///| fn wb_one_cell_anchor(xml_from : String, xml_ext : String) -> String { let pic = wb_pic_xml("rId1") - "\{xml_from}\{xml_ext}\{pic}" + "\{xml_from}\{xml_ext}\{pic}" } ///| fn wb_two_cell_anchor(xml_from : String, xml_to : String) -> String { let pic = wb_pic_xml("rId1") - "\{xml_from}\{xml_to}\{pic}" + "\{xml_from}\{xml_to}\{pic}" } ///| @@ -1037,10 +1016,10 @@ fn wb_rels_image_target(target : String) -> String { test "DrawingML MCE fallback feeds image chart shape and slicer readers" { let source = #| - #| 0000 - #| 1000 - #| 2000 - #| 3000 + #| 0000 + #| 1000 + #| 2000 + #| 3000 #| let drawing = canonicalize_xlsx_drawing_xml( source, default_max_xml_part_bytes, @@ -1092,7 +1071,7 @@ test "DrawingML MCE fallback feeds image chart shape and slicer readers" { ) inspect(charts.length(), content="1") inspect(charts[0].xml, content="good-chart") - let shapes = parse_drawing_shapes(anchors) + let shapes = parse_drawing_shapes(anchors, metrics) inspect(shapes.length(), content="1") inspect(shapes[0].name, content="good-shape") inspect(shapes[0].shape_type, content="rect") @@ -1110,10 +1089,10 @@ test "DrawingML extension payload cannot spoof anchors or drawing features" { let source = #| #| 999900 - #| 0000 - #| 1000 - #| 2000 - #| 3000 + #| 0000 + #| 1000 + #| 2000 + #| 3000 #| let drawing = canonicalize_xlsx_drawing_xml( source, default_max_xml_part_bytes, @@ -1140,8 +1119,8 @@ test "DrawingML extension payload cannot spoof anchors or drawing features" { test "DrawingML ignores valid unsupported anchor objects as complete subtrees" { let source = #| - #| 0000 - #| 1000 + #| 0000 + #| 1000 #| let drawing = canonicalize_xlsx_drawing_xml( source, default_max_xml_part_bytes, @@ -1274,7 +1253,7 @@ test "chartsheet drawing selector requires exactly one chart payload" { let anchor = #| let one : ReadOnlyArray[DrawingAnchor] = [ - { xml: anchor, positioning: OneCell, native_absolute: false }, + { xml: anchor, positioning: OneCell, form: OneCellAnchor }, ] assert_eq(single_drawing_chart_rel_id(one), "selected") let empty : ReadOnlyArray[DrawingAnchor] = [] @@ -1285,8 +1264,8 @@ test "chartsheet drawing selector requires exactly one chart payload" { _ => fail("expected empty chartsheet drawing rejection") } let two : ReadOnlyArray[DrawingAnchor] = [ - { xml: anchor, positioning: OneCell, native_absolute: false }, - { xml: anchor, positioning: OneCell, native_absolute: false }, + { xml: anchor, positioning: OneCell, form: OneCellAnchor }, + { xml: anchor, positioning: OneCell, form: OneCellAnchor }, ] try single_drawing_chart_rel_id(two) catch { InvalidXml(msg~) => assert_eq(msg, "chartsheet drawing has multiple charts") @@ -1299,7 +1278,7 @@ test "chartsheet drawing selector requires exactly one chart payload" { ///| test "DrawingML validates Strict chart dialect and payload URI together" { let source = - #| + #|0000 let canonical = canonicalize_xlsx_drawing_xml( source, default_max_xml_part_bytes, ) @@ -1453,16 +1432,28 @@ test "read_drawing_xml wb: parse_drawing_anchors validation and positioning" { #| #| #| - #| + #| + #| #| let anchors = parse_drawing_anchors(xml) - inspect(anchors.length(), content="4") + inspect(anchors.length(), content="5") debug_inspect(anchors[0].positioning, content="Absolute") debug_inspect(anchors[1].positioning, content="OneCell") debug_inspect(anchors[2].positioning, content="Absolute") - debug_inspect(anchors[3].positioning, content="TwoCell") - inspect(anchors[0].native_absolute, content="true") - inspect(anchors[2].native_absolute, content="false") + debug_inspect(anchors[3].positioning, content="OneCell") + debug_inspect(anchors[4].positioning, content="TwoCell") + debug_inspect(anchors[0].form, content="AbsoluteAnchor") + debug_inspect(anchors[2].form, content="TwoCellAnchor") + try + parse_drawing_anchors( + "", + ) + catch { + InvalidXml(msg~) => inspect(msg, content="drawing editAs value is invalid") + _ => fail("unexpected drawing editAs error") + } noraise { + _ => fail("expected invalid drawing editAs rejection") + } } ///| diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index 53fd867a..fba7022f 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -480,6 +480,10 @@ fn resolve_slicer_sources_after_read( display_header: slicer.display_header, item_desc, format: slicer.format, + drawing_offset_x_emu: slicer.drawing_offset_x_emu, + drawing_offset_y_emu: slicer.drawing_offset_y_emu, + drawing_width_emu: slicer.drawing_width_emu, + drawing_height_emu: slicer.drawing_height_emu, } } } diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index a36d2489..8ad83388 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -98,6 +98,60 @@ fn xlsx_drawing_unsupported_reader_object_name(local_name : StringView) -> Bool local_name == "grpSp" || local_name == "cxnSp" || local_name == "contentPart" } +///| +fn drawing_reader_sequence_matches( + actual : ArrayView[String], + expected : ArrayView[String], +) -> Bool { + if actual.length() != expected.length() { + return false + } + for i in 0.. Unit raise XlsxError { + if marker_parent != "" && + !drawing_reader_sequence_matches(sequence, [ + "col", "colOff", "row", "rowOff", + ]) { + raise InvalidXml(msg="drawing marker children are invalid") + } +} + +///| +fn validate_drawing_anchor_grammar( + anchor_name : StringView, + sequence : ArrayView[String], +) -> Unit raise XlsxError { + let valid = if anchor_name == "oneCellAnchor" { + drawing_reader_sequence_matches(sequence, [ + "from", "ext", "object", "clientData", + ]) + } else if anchor_name == "twoCellAnchor" { + drawing_reader_sequence_matches(sequence, [ + "from", "to", "object", "clientData", + ]) + } else if anchor_name == "absoluteAnchor" { + drawing_reader_sequence_matches(sequence, [ + "pos", "ext", "object", "clientData", + ]) + } else { + false + } + if !valid { + raise InvalidXml(msg="drawing anchor children are invalid") + } +} + ///| fn drawing_reader_relationship_attribute( scanner : @ooxml.XmlStartTagScanner, @@ -203,10 +257,21 @@ fn validate_xlsx_drawing_reader_structure( let mut graphic_seen = false let mut graphic_data_seen = false let mut unsupported_reader_object_depth : Int? = None + let anchor_sequence : Array[String] = [] + let marker_sequence : Array[String] = [] while workbook_scanner_next(scanner) { let depth = scanner.depth() let local_name = scanner.local_name() let namespace_uri = scanner.namespace_uri().to_owned() + if marker_parent != "" && depth <= 3 { + validate_drawing_marker_grammar(marker_parent, marker_sequence) + marker_parent = "" + marker_sequence.clear() + } + if in_reader_anchor && depth <= 2 { + validate_drawing_anchor_grammar(anchor_name, anchor_sequence) + anchor_sequence.clear() + } match text_body_depth { Some(start) if depth <= start => text_body_depth = None _ => () @@ -295,6 +360,7 @@ fn validate_xlsx_drawing_reader_structure( anchor_pos_seen = false anchor_ext_seen = false client_data_seen = false + anchor_sequence.clear() } else if namespace_uri == drawing_namespace && xlsx_drawing_reader_object_name(local_name) { if depth != 3 || @@ -307,6 +373,7 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing anchor has multiple reader objects") } reader_object_seen = true + anchor_sequence.push("object") current_reader_object = local_name.to_owned() non_visual_container_seen = false c_nv_pr_seen = false @@ -342,6 +409,7 @@ fn validate_xlsx_drawing_reader_structure( // not model them. Ignore the complete subtree so nested `sp`, `pic`, or // `graphicFrame` elements cannot be mistaken for direct anchor objects. reader_object_seen = true + anchor_sequence.push("object") current_reader_object = "" unsupported_reader_object_depth = Some(depth) } else if namespace_uri == drawing_namespace && @@ -358,6 +426,7 @@ fn validate_xlsx_drawing_reader_structure( scanner.parent_namespace_uri() != Some(drawing_namespace) { raise InvalidXml(msg="drawing anchor reader target is misplaced") } + anchor_sequence.push(local_name.to_owned()) if local_name == "from" { if anchor_name == "absoluteAnchor" { raise InvalidXml(msg="drawing anchor from target is misplaced") @@ -371,6 +440,7 @@ fn validate_xlsx_drawing_reader_structure( marker_col_off_seen = false marker_row_seen = false marker_row_off_seen = false + marker_sequence.clear() } else if local_name == "to" { if anchor_name == "absoluteAnchor" { raise InvalidXml(msg="drawing anchor to target is misplaced") @@ -384,6 +454,7 @@ fn validate_xlsx_drawing_reader_structure( marker_col_off_seen = false marker_row_seen = false marker_row_off_seen = false + marker_sequence.clear() } else if local_name == "pos" { if anchor_name != "absoluteAnchor" { raise InvalidXml(msg="drawing anchor pos target is misplaced") @@ -416,6 +487,7 @@ fn validate_xlsx_drawing_reader_structure( scanner.parent_namespace_uri() != Some(drawing_namespace) { raise InvalidXml(msg="drawing marker target is misplaced") } + marker_sequence.push(local_name.to_owned()) if local_name == "col" { if marker_col_seen { raise InvalidXml(msg="drawing marker col target is duplicated") @@ -739,8 +811,20 @@ fn validate_xlsx_drawing_reader_structure( raise InvalidXml(msg="drawing anchor has multiple reader payloads") } reader_payload_seen = true + } else if namespace_uri == drawing_namespace && + depth == 3 && + in_reader_anchor && + scanner.parent_local_name() == Some(anchor_name) && + scanner.parent_namespace_uri() == Some(drawing_namespace) { + raise InvalidXml(msg="drawing anchor content is invalid") } } + if marker_parent != "" { + validate_drawing_marker_grammar(marker_parent, marker_sequence) + } + if in_reader_anchor { + validate_drawing_anchor_grammar(anchor_name, anchor_sequence) + } } ///| @@ -1235,7 +1319,7 @@ test "XLSX XML canonicalization follows extension namespace identity" { ///| test "DrawingML canonicalization follows aliases and removes foreign subtrees" { let valid = - #| + #|0000 let canonical = canonicalize_xlsx_drawing_xml( valid, default_max_xml_part_bytes, ) @@ -1256,6 +1340,36 @@ test "DrawingML canonicalization follows aliases and removes foreign subtrees" { assert_false(spoofed_canonical.contains("0000", + "drawing anchor children are invalid", + ), + ( + "0000", + "drawing marker children are invalid", + ), + ( + "0000", + "drawing anchor children are invalid", + ), + ] + for case in cases { + let (source, expected) = case + try + canonicalize_xlsx_drawing_xml(source, default_max_xml_part_bytes) + catch { + InvalidXml(msg~) => inspect(msg, content=expected) + _ => fail("unexpected drawing grammar error") + } noraise { + _ => fail("expected drawing grammar rejection") + } + } +} + ///| test "XLSX XML canonicalization propagates cancellation from a long pass" { let checks = [0] diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index 1df61d4b..fb2dbeb2 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -529,6 +529,10 @@ fn Workbook::rich_value_image_for_cell( alt_text, lock_aspect_ratio: false, positioning: OneCell, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) } None => None diff --git a/xlsx/shape.mbt b/xlsx/shape.mbt index 638b62ec..77940629 100644 --- a/xlsx/shape.mbt +++ b/xlsx/shape.mbt @@ -81,6 +81,10 @@ pub struct Shape { print_object : Bool locked : Bool positioning : PicturePositioning + priv drawing_offset_x_emu : Int? + priv drawing_offset_y_emu : Int? + priv drawing_width_emu : Int? + priv drawing_height_emu : Int? } derive(Debug) ///| @@ -202,5 +206,9 @@ pub fn Shape::new( print_object, locked, positioning, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } } diff --git a/xlsx/shape_read_test.mbt b/xlsx/shape_read_test.mbt index c25c2054..4475f27b 100644 --- a/xlsx/shape_read_test.mbt +++ b/xlsx/shape_read_test.mbt @@ -65,11 +65,11 @@ test "native absolute shape geometry survives normalized roundtrip" { #| #| #| - #| - #| + #| + #| #| #| - #| + #| #| #| #| @@ -102,8 +102,8 @@ test "native absolute shape geometry survives normalized roundtrip" { assert_true( normalized_drawing.contains(""), ) - assert_true(normalized_drawing.contains("19050")) - assert_true(normalized_drawing.contains("28575")) + assert_true(normalized_drawing.contains("19051")) + assert_true(normalized_drawing.contains("28576")) let reparsed = @xlsx.read(normalized) let reparsed_shape = reparsed.sheet("Sheet1").unwrap().shapes()[0] inspect(reparsed_shape.width, content="10") @@ -111,4 +111,83 @@ test "native absolute shape geometry survives normalized roundtrip" { debug_inspect(reparsed_shape.format.offset_x, content="Some(2)") debug_inspect(reparsed_shape.format.offset_y, content="Some(3)") debug_inspect(reparsed_shape.positioning, content="Absolute") + let rewritten_archive = @zip.read(@xlsx.write(reparsed)) catch { + err => fail("rewritten zip read failed: \{repr(err)}") + } + let rewritten_drawing = match rewritten_archive.get(drawing_path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("rewritten drawing utf8 decode failed") + } + None => fail("rewritten drawing missing") + } + inspect(rewritten_drawing == normalized_drawing, content="true") +} + +///| +test "shape geometry follows anchor form and editAs oneCell normalizes" { + let workbook = @xlsx.Workbook::new() + ignore(workbook.add_sheet("Sheet1")) + workbook.add_shape("Sheet1", @xlsx.Shape::new("A1", "rect")) + let archive = @zip.read(@xlsx.write(workbook)) catch { + err => fail("zip read failed: \{repr(err)}") + } + let drawing_path = "xl/drawings/drawing1.xml" + let drawing = + #| + #| + #| + #| 019051028576 + #| + #| + #| + #| + #| + #| 119052128577 + #| 11143031219078 + #| + #| + #| + #| + assert_true(archive.replace(drawing_path, @encoding/utf8.encode(drawing))) + let parsed = @xlsx.read(@zip.write(archive)) + let shapes = parsed.sheet("Sheet1").unwrap().shapes() + inspect(shapes.length(), content="2") + inspect(shapes[0].width, content="10") + inspect(shapes[0].height, content="20") + debug_inspect(shapes[0].positioning, content="OneCell") + inspect(shapes[1].width, content="10") + inspect(shapes[1].height, content="20") + debug_inspect(shapes[1].positioning, content="OneCell") + + let normalized = @xlsx.write(parsed) + let normalized_archive = @zip.read(normalized) catch { + err => fail("normalized zip read failed: \{repr(err)}") + } + let normalized_drawing = match normalized_archive.get(drawing_path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("normalized drawing utf8 decode failed") + } + None => fail("normalized drawing missing") + } + inspect(normalized_drawing.split("").count(), content="3") + assert_false(normalized_drawing.contains("19051")) + assert_true(normalized_drawing.contains("19052")) + assert_true( + normalized_drawing.contains(""), + ) + let reparsed = @xlsx.read(normalized) + let rewritten_archive = @zip.read(@xlsx.write(reparsed)) catch { + err => fail("rewritten zip read failed: \{repr(err)}") + } + let rewritten_drawing = match rewritten_archive.get(drawing_path) { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("rewritten drawing utf8 decode failed") + } + None => fail("rewritten drawing missing") + } + inspect(rewritten_drawing == normalized_drawing, content="true") } diff --git a/xlsx/slicer.mbt b/xlsx/slicer.mbt index 17d52439..bb5ac5f0 100644 --- a/xlsx/slicer.mbt +++ b/xlsx/slicer.mbt @@ -13,6 +13,10 @@ pub struct Slicer { display_header : Bool? item_desc : Bool format : GraphicOptions + priv drawing_offset_x_emu : Int? + priv drawing_offset_y_emu : Int? + priv drawing_width_emu : Int? + priv drawing_height_emu : Int? } derive(Debug) ///| @@ -53,6 +57,10 @@ pub fn Slicer::new(name : String) -> Slicer raise XlsxError { display_header: None, item_desc: false, format: GraphicOptions::new(), + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } } diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 1d2564b4..02d48ed7 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -2687,6 +2687,10 @@ pub fn Workbook::add_slicer( display_header: opts.display_header, item_desc: opts.item_desc, format, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } sheet.add_slicer(slicer) } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 4074962a..6fd429b3 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2044,6 +2044,10 @@ pub fn Worksheet::add_image( alt_text, lock_aspect_ratio, positioning, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) } @@ -2068,6 +2072,10 @@ fn image_with_reference(image : Image, reference : String) -> Image { alt_text: image.alt_text, lock_aspect_ratio: image.lock_aspect_ratio, positioning: image.positioning, + drawing_offset_x_emu: image.drawing_offset_x_emu, + drawing_offset_y_emu: image.drawing_offset_y_emu, + drawing_width_emu: image.drawing_width_emu, + drawing_height_emu: image.drawing_height_emu, } } @@ -2083,6 +2091,10 @@ fn chart_with_reference(chart : Chart, reference : String) -> Chart { print_object: chart.print_object, locked: chart.locked, positioning: chart.positioning, + drawing_offset_x_emu: chart.drawing_offset_x_emu, + drawing_offset_y_emu: chart.drawing_offset_y_emu, + drawing_width_emu: chart.drawing_width_emu, + drawing_height_emu: chart.drawing_height_emu, } } @@ -2155,6 +2167,10 @@ pub fn Worksheet::add_chart( print_object: true, locked: true, positioning: TwoCell, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) } @@ -2222,6 +2238,10 @@ pub fn Worksheet::add_chart_with_options( print_object, locked, positioning, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) } diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index 48a7dc68..f8adf0c7 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -52,6 +52,10 @@ pub struct Image { alt_text : String lock_aspect_ratio : Bool positioning : PicturePositioning + priv drawing_offset_x_emu : Int? + priv drawing_offset_y_emu : Int? + priv drawing_width_emu : Int? + priv drawing_height_emu : Int? } ///| @@ -65,6 +69,10 @@ pub struct Chart { print_object : Bool locked : Bool positioning : PicturePositioning + priv drawing_offset_x_emu : Int? + priv drawing_offset_y_emu : Int? + priv drawing_width_emu : Int? + priv drawing_height_emu : Int? } ///| diff --git a/xlsx/write.mbt b/xlsx/write.mbt index 22dd62d3..2314be99 100644 --- a/xlsx/write.mbt +++ b/xlsx/write.mbt @@ -248,6 +248,10 @@ priv struct DrawingChart { locked : Bool positioning : PicturePositioning rel_id : Int + drawing_offset_x_emu : Int? + drawing_offset_y_emu : Int? + drawing_width_emu : Int? + drawing_height_emu : Int? } ///| @@ -413,16 +417,36 @@ fn write_drawing_xml( let row_idx = row - 1 let pic_id = obj_id obj_id = obj_id + 1 - let col_off = drawing_pixel_offset_for_write( - image.offset_x, - "drawing image column offset", + let col_off = match image.drawing_offset_x_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write( + image.offset_x, + "drawing image column offset", + ) + } + let row_off = match image.drawing_offset_y_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write( + image.offset_y, + "drawing image row offset", + ) + } + let cx = drawing_extent_for_write( + match image.drawing_width_emu { + Some(value) => value + None => image.width_emu + }, + "drawing image width", ) - let row_off = drawing_pixel_offset_for_write( - image.offset_y, - "drawing image row offset", + let cy = drawing_extent_for_write( + match image.drawing_height_emu { + Some(value) => value + None => image.height_emu + }, + "drawing image height", ) - let cx = drawing_extent_for_write(image.width_emu, "drawing image width") - let cy = drawing_extent_for_write(image.height_emu, "drawing image height") let picture_name = if image.name == "" { "Image \{pic_id}" } else { @@ -553,16 +577,36 @@ fn write_drawing_xml( let row_idx = row - 1 let frame_id = obj_id obj_id = obj_id + 1 - let col_off = drawing_pixel_offset_for_write( - chart.offset_x, - "drawing chart column offset", + let col_off = match chart.drawing_offset_x_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write( + chart.offset_x, + "drawing chart column offset", + ) + } + let row_off = match chart.drawing_offset_y_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write( + chart.offset_y, + "drawing chart row offset", + ) + } + let cx = drawing_extent_for_write( + match chart.drawing_width_emu { + Some(value) => value + None => chart.width_emu + }, + "drawing chart width", ) - let row_off = drawing_pixel_offset_for_write( - chart.offset_y, - "drawing chart row offset", + let cy = drawing_extent_for_write( + match chart.drawing_height_emu { + Some(value) => value + None => chart.height_emu + }, + "drawing chart height", ) - let cx = drawing_extent_for_write(chart.width_emu, "drawing chart width") - let cy = drawing_extent_for_write(chart.height_emu, "drawing chart height") match chart.positioning { TwoCell => { sb.write_view(" \n") @@ -665,24 +709,36 @@ fn write_drawing_xml( let row_idx = row - 1 let offset_x = shape.format.offset_x.unwrap_or(0) let offset_y = shape.format.offset_y.unwrap_or(0) - let col_off = drawing_pixel_offset_for_write( - offset_x, "drawing shape column offset", - ) - let row_off = drawing_pixel_offset_for_write( - offset_y, "drawing shape row offset", - ) + let col_off = match shape.drawing_offset_x_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write(offset_x, "drawing shape column offset") + } + let row_off = match shape.drawing_offset_y_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write(offset_y, "drawing shape row offset") + } let shape_id = obj_id obj_id = obj_id + 1 - let cx = drawing_scaled_extent_for_write( - shape.width.to_double(), - shape.scale_x, - "drawing shape width", - ) - let cy = drawing_scaled_extent_for_write( - shape.height.to_double(), - shape.scale_y, - "drawing shape height", - ) + let cx = match shape.drawing_width_emu { + Some(value) => drawing_extent_for_write(value, "drawing shape width") + None => + drawing_scaled_extent_for_write( + shape.width.to_double(), + shape.scale_x, + "drawing shape width", + ) + } + let cy = match shape.drawing_height_emu { + Some(value) => drawing_extent_for_write(value, "drawing shape height") + None => + drawing_scaled_extent_for_write( + shape.height.to_double(), + shape.scale_y, + "drawing shape height", + ) + } match shape.positioning { TwoCell => { sb.write_view(" \n") @@ -911,12 +967,16 @@ fn write_drawing_xml( Some(v) => v None => 0 } - let col_off = drawing_pixel_offset_for_write( - offset_x, "drawing slicer column offset", - ) - let row_off = drawing_pixel_offset_for_write( - offset_y, "drawing slicer row offset", - ) + let col_off = match slicer.drawing_offset_x_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write(offset_x, "drawing slicer column offset") + } + let row_off = match slicer.drawing_offset_y_emu { + Some(value) => value + None => + drawing_pixel_offset_for_write(offset_y, "drawing slicer row offset") + } let scale_x = match slicer.format.scale_x { Some(v) => if v <= 0.0 { 1.0 } else { v } None => 1.0 @@ -925,16 +985,24 @@ fn write_drawing_xml( Some(v) => if v <= 0.0 { 1.0 } else { v } None => 1.0 } - let cx = drawing_scaled_extent_for_write( - slicer.width.to_double(), - scale_x, - "drawing slicer width", - ) - let cy = drawing_scaled_extent_for_write( - slicer.height.to_double(), - scale_y, - "drawing slicer height", - ) + let cx = match slicer.drawing_width_emu { + Some(value) => drawing_extent_for_write(value, "drawing slicer width") + None => + drawing_scaled_extent_for_write( + slicer.width.to_double(), + scale_x, + "drawing slicer width", + ) + } + let cy = match slicer.drawing_height_emu { + Some(value) => drawing_extent_for_write(value, "drawing slicer height") + None => + drawing_scaled_extent_for_write( + slicer.height.to_double(), + scale_y, + "drawing slicer height", + ) + } let positioning = match slicer.format.positioning { Some(v) => v None => TwoCell @@ -2958,6 +3026,10 @@ fn write_with_io( locked: chart.locked, positioning: chart.positioning, rel_id, + drawing_offset_x_emu: chart.drawing_offset_x_emu, + drawing_offset_y_emu: chart.drawing_offset_y_emu, + drawing_width_emu: chart.drawing_width_emu, + drawing_height_emu: chart.drawing_height_emu, }) rel_id = rel_id + 1 manifest.add_content_override( @@ -3054,6 +3126,10 @@ fn write_with_io( locked: true, positioning: OneCell, rel_id: 1, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }, ] let drawing_xml = write_drawing_xml( @@ -3244,6 +3320,10 @@ fn wb_drawing_image_with_offset( alt_text: "", lock_aspect_ratio: false, positioning, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } { image, rel_id: 1, hyperlink_rel_id: None } } @@ -3265,6 +3345,10 @@ fn wb_drawing_chart(positioning : PicturePositioning) -> DrawingChart { locked: true, positioning, rel_id: 1, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } } @@ -3336,6 +3420,10 @@ fn wb_drawing_slicer( display_header: None, item_desc: false, format, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } { slicer, is_table_slicer } } @@ -3526,6 +3614,10 @@ test "write hardening wb: two-cell anchors enforce the worksheet boundary" { locked: true, positioning: TwoCell, rel_id: 1, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } let result : Result[String, Error] = Ok( write_drawing_xml(Some(Worksheet::new("Sheet1")), [], [chart], [], []), @@ -3555,6 +3647,10 @@ test "write hardening wb: signed offsets and exact end markers stay readable" { locked: true, positioning: TwoCell, rel_id: 1, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, } let boundary_xml = write_drawing_xml( Some(Worksheet::new("Sheet1")), @@ -4044,6 +4140,10 @@ test "write hardening wb: write fails for slicer source missing" { display_header: None, item_desc: false, format: GraphicOptions::new(), + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::InvalidSheetOperation(_)), content="true") @@ -4068,6 +4168,10 @@ test "write hardening wb: write fails for slicer table field missing" { display_header: None, item_desc: false, format: GraphicOptions::new(), + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::InvalidSheetOperation(_)), content="true") @@ -4135,6 +4239,10 @@ test "write hardening wb: default zip writer failure maps to UnsupportedFeature" alt_text: "", lock_aspect_ratio: false, positioning: TwoCell, + drawing_offset_x_emu: None, + drawing_offset_y_emu: None, + drawing_width_emu: None, + drawing_height_emu: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::UnsupportedFeature(_)), content="true") From 60eb76158e0bf19454b9195830d991cb73f4b3bd Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 15:56:21 +0800 Subject: [PATCH 145/150] fix(xlsx): preserve shape rich text --- xlsx/read_drawing_xml.mbt | 229 +++++++++++++++++++++--- xlsx/read_xml_namespaces.mbt | 69 ++++++- xlsx/shape_form_control_slicer_test.mbt | 48 ++++- 3 files changed, 320 insertions(+), 26 deletions(-) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index 0f8b80c5..ccb7f797 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -442,6 +442,179 @@ fn drawing_shape_solid_fill_body(sppr_body : StringView) -> String? { extract_tag_body_from(sppr_body, "a:solidFill") } +///| +fn drawing_text_bool_attribute( + tag : StringView, + name : StringView, +) -> Bool raise XlsxError { + match attr_value(tag, name) { + Some(value) => parse_bool_attr(value) + None => false + } +} + +///| +fn parse_drawing_text_font( + container_body : StringView, +) -> RichTextFont? raise XlsxError { + let rpr_tag = match tag_attributes_in(container_body, "a:rPr") { + Some(value) => value + None => return None + } + let bold = drawing_text_bool_attribute(rpr_tag, "b") + let italic = drawing_text_bool_attribute(rpr_tag, "i") + let strike = match attr_value(rpr_tag, "strike") { + Some("sngStrike") | Some("dblStrike") => true + Some("noStrike") | None => false + Some(_) => raise InvalidXml(msg="drawing text strike is invalid") + } + let underline = match attr_value(rpr_tag, "u") { + Some("none") | None => None + Some(value) => Some(unescape_xml_text(value)) + } + let size = match attr_value(rpr_tag, "sz") { + Some(value) => { + let parsed = @string.parse_int(value, base=10) catch { + _ => raise InvalidXml(msg="drawing text size is invalid") + } + if parsed < 0 { + raise InvalidXml(msg="drawing text size is invalid") + } + Some(parsed.to_double() / 100.0) + } + None => None + } + let rpr_body = extract_tag_body_from(container_body, "a:rPr").unwrap_or("") + let source = "" + + rpr_body + + "" + let scanner = @ooxml.XmlStartTagScanner::new(source) + let mut family : String? = None + let mut color : String? = None + let mut direct_solid_fill = false + while workbook_scanner_next(scanner) { + let depth = scanner.depth() + if depth <= 2 { + direct_solid_fill = false + } + if depth == 2 && + scanner.namespace_uri() == transitional_drawing_main_namespace && + scanner.parent_local_name() == Some("root") { + if scanner.local_name() == "latin" && family is None { + family = workbook_scanner_attribute(scanner, "", "typeface") + } else if scanner.local_name() == "solidFill" { + direct_solid_fill = true + } + } else if direct_solid_fill && + depth == 3 && + scanner.namespace_uri() == transitional_drawing_main_namespace && + scanner.local_name() == "srgbClr" && + scanner.parent_local_name() == Some("solidFill") && + scanner.parent_namespace_uri() == + Some(transitional_drawing_main_namespace) && + color is None { + color = match workbook_scanner_attribute(scanner, "", "val") { + Some(value) if value != "" => Some(value) + _ => raise InvalidXml(msg="drawing text color is invalid") + } + } + } + if bold || + italic || + strike || + underline is Some(_) || + size is Some(_) || + family is Some(_) || + color is Some(_) { + Some({ + bold, + italic, + strike, + outline: false, + shadow: false, + condense: false, + extended: false, + underline, + size, + color, + color_theme: None, + color_indexed: None, + color_tint: None, + charset: None, + family_number: None, + scheme: None, + vert_align: None, + family, + }) + } else { + None + } +} + +///| +fn parse_drawing_shape_text_runs( + tx_body : StringView, +) -> Array[RichTextRun] raise XlsxError { + let runs : Array[RichTextRun] = [] + let source = tx_body.to_owned() + let container_names : ReadOnlyArray[String] = ["a:r", "a:fld", "a:br"] + let mut cursor = 0 + for ;; { + let mut next_start : Int? = None + let mut next_name = "" + for candidate in container_names { + match find_xml_open_tag_start_from(source, candidate, cursor) { + Some(start) => + match next_start { + Some(current) if current <= start => () + _ => { + next_start = Some(start) + next_name = candidate + } + } + None => () + } + } + let start = match next_start { + Some(value) => value + None => break + } + let open_end = match xml_open_tag_end_from(source, start + 1) { + Some(value) => value + None => raise InvalidXml(msg="drawing text tag is not closed") + } + let self_closing = source[start + 1:open_end].trim().has_suffix("/") + let (container_body, close_end) = if self_closing { + ("", open_end) + } else { + let (close_start, end) = match + find_xml_close_tag_from(source, next_name, open_end + 1) { + Some(value) => value + None => raise InvalidXml(msg="drawing text close is missing") + } + (source[open_end + 1:close_start].to_owned(), end) + } + if next_name == "a:br" { + runs.push({ text: "\n", font: parse_drawing_text_font(container_body) }) + } else { + let text = match extract_tag_body_from(container_body, "a:t") { + Some(value) => unescape_xml_text(value) + None => + if tag_attributes_in(container_body, "a:t") is Some(_) { + "" + } else { + raise InvalidXml(msg="drawing text value is missing") + } + } + runs.push({ text, font: parse_drawing_text_font(container_body) }) + } + cursor = close_end + 1 + } + runs +} + ///| fn parse_drawing_images( anchors : ArrayView[DrawingAnchor], @@ -814,26 +987,9 @@ fn parse_drawing_shapes( } None => () } - let paragraph : Array[RichTextRun] = [] - match extract_tag_body_from(sp_body, "xdr:txBody") { - Some(tx_body) => { - let tx = tx_body.to_string() - let mut first = true - for chunk in tx.split("") { - if first { - first = false - continue - } - let part = chunk.to_owned() - let end = match part.find("") { - Some(pos) => pos - None => continue - } - let text = part[:end] - paragraph.push({ text: unescape_xml_text(text), font: None }) - } - } - None => () + let paragraph = match extract_tag_body_from(sp_body, "xdr:txBody") { + Some(tx_body) => parse_drawing_shape_text_runs(tx_body) + None => [] } let text = "" let format = shape_graphic_options( @@ -1221,6 +1377,12 @@ test "DrawingML rejects supported-namespace reader decoys" { ), "drawing text target is misplaced", ), + ( + ( + #|decoyreal + ), + "drawing text target is misplaced", + ), ] for case in cases { let (source, expected) = case @@ -1235,6 +1397,33 @@ test "DrawingML rejects supported-namespace reader decoys" { } } +///| +test "DrawingML rich text ignores unmodeled paragraph defaults" { + let source = + #| + #| + #| 0000 + #| + #| + #| + #| + #| real + #| + #| + #| + #| + let canonical = canonicalize_xlsx_drawing_xml( + source, default_max_xml_part_bytes, + ) + let tx_body = extract_tag_body_from(canonical, "xdr:txBody").unwrap_or("") + let runs = parse_drawing_shape_text_runs(tx_body) + inspect(runs.length(), content="1") + inspect(runs[0].text, content=" real ") + let font = runs[0].font.unwrap() + inspect(font.color.unwrap_or(""), content="A1B2C3") + inspect(font.family.unwrap_or(""), content="") +} + ///| test "DrawingML shape fill selection excludes line-only fills" { let line_only = diff --git a/xlsx/read_xml_namespaces.mbt b/xlsx/read_xml_namespaces.mbt index 8ad83388..aac55136 100644 --- a/xlsx/read_xml_namespaces.mbt +++ b/xlsx/read_xml_namespaces.mbt @@ -225,6 +225,9 @@ fn validate_xlsx_drawing_reader_structure( let mut graphic_data_uri : String? = None let mut extension_depth : Int? = None let mut text_body_depth : Int? = None + let mut text_container = "" + let mut text_run_properties_seen = false + let mut text_value_seen = false let mut solid_fill_depth : Int? = None let mut color_depth : Int? = None let mut marker_parent = "" @@ -263,6 +266,14 @@ fn validate_xlsx_drawing_reader_structure( let depth = scanner.depth() let local_name = scanner.local_name() let namespace_uri = scanner.namespace_uri().to_owned() + if text_container != "" && depth <= 6 { + if (text_container == "r" || text_container == "fld") && !text_value_seen { + raise InvalidXml(msg="drawing text run is invalid") + } + text_container = "" + text_run_properties_seen = false + text_value_seen = false + } if marker_parent != "" && depth <= 3 { validate_drawing_marker_grammar(marker_parent, marker_sequence) marker_parent = "" @@ -326,15 +337,60 @@ fn validate_xlsx_drawing_reader_structure( continue } // Rich-text run properties use the same DrawingML element names as shape - // styling (for example a:rPr/a:solidFill/a:srgbClr), but they are not - // inputs to the shape-style reader. Keep dialect checks above active and - // ignore only same-dialect DrawingML descendants of the validated txBody; - // xdr/c/sle descendants must still be rejected as reader decoys. + // styling. Prove every name consumed by the text reader occurs on its + // exact txBody/paragraph/run path before tolerating unmodeled DrawingML + // properties. Other dialects and xdr/c/sle descendants still fall through + // to the normal decoy checks below. let inside_shape_text_body = match text_body_depth { Some(start) => depth > start None => false } if inside_shape_text_body && namespace_uri == main_namespace { + let parent_name = scanner.parent_local_name().unwrap_or("") + let parent_namespace = scanner.parent_namespace_uri() + if local_name == "p" { + if depth != 5 || + parent_name != "txBody" || + parent_namespace != Some(drawing_namespace) { + raise InvalidXml(msg="drawing text paragraph is misplaced") + } + } else if local_name == "r" || local_name == "fld" || local_name == "br" { + if depth != 6 || + parent_name != "p" || + parent_namespace != Some(main_namespace) { + raise InvalidXml(msg="drawing text run is misplaced") + } + text_container = local_name.to_owned() + text_run_properties_seen = false + text_value_seen = false + } else if local_name == "rPr" { + if depth != 7 || + parent_name != text_container || + parent_namespace != Some(main_namespace) || + text_container == "" { + raise InvalidXml(msg="drawing text properties are misplaced") + } + if text_run_properties_seen || text_value_seen { + raise InvalidXml(msg="drawing text run is invalid") + } + text_run_properties_seen = true + } else if local_name == "t" { + if depth != 7 || + (text_container != "r" && text_container != "fld") || + parent_name != text_container || + parent_namespace != Some(main_namespace) { + raise InvalidXml(msg="drawing text target is misplaced") + } + if text_value_seen { + raise InvalidXml(msg="drawing text run is invalid") + } + text_value_seen = true + } + // Run-property parsing below is namespace-aware and only consumes + // direct children of the validated rPr above. Other DrawingML text + // properties (for example pPr/defRPr defaults and nested underline + // fills) are valid but deliberately unmodeled, so they can be ignored + // without becoming lexical decoys. continue } if local_name == "ext" && @@ -825,6 +881,11 @@ fn validate_xlsx_drawing_reader_structure( if in_reader_anchor { validate_drawing_anchor_grammar(anchor_name, anchor_sequence) } + if text_container != "" && + (text_container == "r" || text_container == "fld") && + !text_value_seen { + raise InvalidXml(msg="drawing text run is invalid") + } } ///| diff --git a/xlsx/shape_form_control_slicer_test.mbt b/xlsx/shape_form_control_slicer_test.mbt index 6d6329ac..a92f6f6d 100644 --- a/xlsx/shape_form_control_slicer_test.mbt +++ b/xlsx/shape_form_control_slicer_test.mbt @@ -29,7 +29,7 @@ test "shape size and rich text paragraphs" { let sheet = workbook.add_sheet("Sheet1") let runs : Array[@xlsx.RichTextRun] = [ { - text: "Rectangle", + text: " Rectangle ", font: Some({ bold: false, italic: false, @@ -102,6 +102,10 @@ test "shape size and rich text paragraphs" { inspect(drawing_xml.contains("sz=\"1800\""), content="true") inspect(drawing_xml.contains("typeface=\"Times New Roman\""), content="true") inspect(drawing_xml.contains("val=\"2980B9\""), content="true") + inspect( + drawing_xml.contains(" Rectangle "), + content="true", + ) let parsed = @xlsx.read(bytes) let parsed_sheet = match parsed.sheet("Sheet1") { Some(value) => value @@ -110,8 +114,48 @@ test "shape size and rich text paragraphs" { inspect(parsed_sheet.shapes().length(), content="1") let parsed_shape = parsed_sheet.shapes()[0] inspect(parsed_shape.paragraph.length(), content="2") - inspect(parsed_shape.paragraph[0].text, content="Rectangle") + inspect(parsed_shape.paragraph[0].text, content=" Rectangle ") inspect(parsed_shape.paragraph[1].text, content="Shape") + let first_font = match parsed_shape.paragraph[0].font { + Some(value) => value + None => fail("missing first shape run font") + } + debug_inspect(first_font.color, content="Some(\"CD5C5C\")") + let second_font = match parsed_shape.paragraph[1].font { + Some(value) => value + None => fail("missing second shape run font") + } + inspect(second_font.bold, content="true") + inspect(second_font.italic, content="true") + debug_inspect(second_font.underline, content="Some(\"sng\")") + inspect(second_font.size == Some(18.0), content="true") + debug_inspect(second_font.family, content="Some(\"Times New Roman\")") + debug_inspect(second_font.color, content="Some(\"2980B9\")") + + let rewritten_archive = @zip.read(@xlsx.write(parsed)) catch { + err => fail("rewritten zip read failed: \{repr(err)}") + } + let rewritten_drawing = match + rewritten_archive.get("xl/drawings/drawing1.xml") { + Some(value) => + @encoding/utf8.decode(value) catch { + _ => fail("rewritten drawing1.xml utf8 decode failed") + } + None => fail("missing rewritten drawing1.xml") + } + inspect(rewritten_drawing.contains("b=\"1\""), content="true") + inspect(rewritten_drawing.contains("i=\"1\""), content="true") + inspect(rewritten_drawing.contains("u=\"sng\""), content="true") + inspect(rewritten_drawing.contains("sz=\"1800\""), content="true") + inspect( + rewritten_drawing.contains("typeface=\"Times New Roman\""), + content="true", + ) + inspect(rewritten_drawing.contains("val=\"2980B9\""), content="true") + inspect( + rewritten_drawing.contains(" Rectangle "), + content="true", + ) } ///| From 3563202f88e71853561010e6fe29762803226fe1 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 16:06:35 +0800 Subject: [PATCH 146/150] feat(ooxml): enumerate package relationships --- ooxml/pkg.generated.mbti | 10 +++++ ooxml/read_parse.mbt | 88 +++++++++++++++++++++++++++++++++++++ ooxml/read_parse_test.mbt | 30 +++++++++++++ ooxml/start_tag_scanner.mbt | 43 ++++++++++++++++++ 4 files changed, 171 insertions(+) diff --git a/ooxml/pkg.generated.mbti b/ooxml/pkg.generated.mbti index 6d380d68..debdbf5d 100644 --- a/ooxml/pkg.generated.mbti +++ b/ooxml/pkg.generated.mbti @@ -44,6 +44,8 @@ pub fn parse_internal_relationship_targets_by_types(StringView, ArrayView[String pub fn parse_package_content_types(StringView, cancelled? : () -> Bool) -> PackageContentTypes raise ParseXmlError +pub fn parse_package_relationships(StringView, cancelled? : () -> Bool) -> Array[PackageRelationship] raise ParseXmlError + pub fn project_xml_markup_compatibility(StringView, ArrayView[String], max_output_chars? : Int, cancelled? : () -> Bool, extension_elements? : ArrayView[(String, String)]) -> String raise ParseXmlError pub fn remove_xml_expanded_name_subtrees(StringView, ArrayView[String], String, cancelled? : () -> Bool) -> String? raise ParseXmlError @@ -78,6 +80,13 @@ pub struct PackageContentTypes { } pub fn PackageContentTypes::content_type_for(Self, StringView, cancelled? : () -> Bool) -> String? raise ParseXmlError +pub struct PackageRelationship { + id : String + rel_type : String + target : String + external : Bool +} + type WorkbookManifest pub fn WorkbookManifest::add_content_default(Self, String, String) -> Unit pub fn WorkbookManifest::add_content_override(Self, String, String) -> Unit @@ -94,6 +103,7 @@ pub struct XmlStartTagScanner { // private fields } pub fn XmlStartTagScanner::attribute(Self, StringView, StringView) -> String? raise ParseXmlError +pub fn XmlStartTagScanner::attribute_values_in_namespace(Self, StringView) -> Array[String] raise ParseXmlError pub fn XmlStartTagScanner::attribute_view(Self, StringView, StringView) -> StringView? raise ParseXmlError pub fn XmlStartTagScanner::depth(Self) -> Int pub fn XmlStartTagScanner::local_name(Self) -> StringView diff --git a/ooxml/read_parse.mbt b/ooxml/read_parse.mbt index 30a30d55..a2a2a129 100644 --- a/ooxml/read_parse.mbt +++ b/ooxml/read_parse.mbt @@ -150,6 +150,15 @@ priv enum ParsedRelationshipTargetMode { ExternalTarget } +///| +/// One validated relationship from an OPC `.rels` part. +pub struct PackageRelationship { + id : String + rel_type : String + target : String + external : Bool +} + ///| fn relationship_target_mode( scanner : XmlStartTagScanner, @@ -872,6 +881,85 @@ fn parse_relationship_targets_by_types_selected( targets } +///| +/// Parses every effective relationship record in source order. The same +/// namespace, MCE, schema, IRI, uniqueness, cancellation, and resource-limit +/// checks used by the type-selecting readers apply here. +pub fn parse_package_relationships( + xml : StringView, + cancelled? : () -> Bool = () => false, +) -> Array[PackageRelationship] raise ParseXmlError { + let relationships : Array[PackageRelationship] = [] + let ids : Set[String] = Set([]) + let scanner = XmlStartTagScanner::new(xml, cancelled~) + let mce_states : Array[RelationshipMceState] = [] + let mce_context = RelationshipMceContext::new() + let mut records = 0 + let mut retained_chars = 0 + while scanner.next() { + let mce_state = relationship_mce_state_for_current_element( + scanner, mce_states, mce_context, + ) + mce_states.push(mce_state) + if mce_state.element_effective_depth == 1 { + continue + } + if !mce_state.active || !mce_state.included_in_effective_tree { + continue + } + if scanner.namespace_uri() != package_relationships_namespace || + scanner.local_name() != "Relationship" || + mce_state.element_effective_depth != 2 { + raise InvalidXml(msg="relationships effective content is invalid") + } + relationship_schema_attribute_allowed( + scanner, + mce_context, + ["Id", "Type", "Target", "TargetMode"], + "record", + ) + records = records + 1 + if records > max_relationship_records { + raise InvalidXml(msg="relationship record limit exceeded") + } + let rel_type = relationship_attribute(scanner, "Type") + let id = relationship_attribute(scanner, "Id") + let target = relationship_attribute(scanner, "Target") + let parsed_target_mode = relationship_target_mode(scanner) + if rel_type.length() > max_relationship_type_chars { + raise InvalidXml(msg="relationship type is too long") + } + if !relationship_type_iri_valid(rel_type) { + raise InvalidXml(msg="relationship type is invalid") + } + if id.length() > max_relationship_id_chars { + raise InvalidXml(msg="relationship id is too long") + } + if !is_xml_ncname(id) { + raise InvalidXml(msg="relationship id is invalid") + } + if target.length() > max_relationship_target_chars { + raise InvalidXml(msg="relationship target is too long") + } + let external_target = parsed_target_mode is ExternalTarget + if !relationship_target_iri_valid(target, external_target) { + raise InvalidXml(msg="relationship target is invalid") + } + if ids.contains(id) { + raise InvalidXml(msg="duplicate relationship id") + } + ids.add(id) + let retained = id.length() + rel_type.length() + target.length() + if retained > max_relationship_retained_chars - retained_chars { + raise InvalidXml(msg="relationship retained text limit exceeded") + } + retained_chars = retained_chars + retained + relationships.push({ id, rel_type, target, external: external_target }) + } + trim_relationship_mce_states(mce_states, 1, mce_context) + relationships +} + ///| /// Reads internal relationship targets whose decoded `Type` is in /// `rel_types`. Explicitly external records are validated but excluded. diff --git a/ooxml/read_parse_test.mbt b/ooxml/read_parse_test.mbt index 9cf303e7..fa52364d 100644 --- a/ooxml/read_parse_test.mbt +++ b/ooxml/read_parse_test.mbt @@ -129,6 +129,36 @@ test "ooxml read_parse: relationship identifiers and URIs fail closed" { } } +///| +test "ooxml read_parse: enumerates validated relationship records" { + let xml = + #| + #| + #| + #| + let records = @ooxml.parse_package_relationships(xml) + inspect(records.length(), content="2") + inspect(records[0].id, content="inside") + inspect(records[0].rel_type, content="urn:inside") + inspect(records[0].target, content="parts/a.xml") + inspect(records[0].external, content="false") + inspect(records[1].id, content="outside") + inspect(records[1].target, content="https://example.invalid/a?x=1&y=2") + inspect(records[1].external, content="true") +} + +///| +test "ooxml scanner enumerates relationship namespace attributes" { + let xml = + #| + let scanner = @ooxml.XmlStartTagScanner::new(xml) + assert_true(scanner.next()) + debug_inspect( + scanner.attribute_values_in_namespace("urn:relationships"), + content="[\"first\", \"second&value\"]", + ) +} + ///| test "ooxml read_parse: relationship target modes are explicit and filtered" { let xml = diff --git a/ooxml/start_tag_scanner.mbt b/ooxml/start_tag_scanner.mbt index 9aabeb7d..2adf915c 100644 --- a/ooxml/start_tag_scanner.mbt +++ b/ooxml/start_tag_scanner.mbt @@ -1914,6 +1914,49 @@ pub fn XmlStartTagScanner::attribute( } } +///| +/// Returns decoded values for every attribute in `namespace_uri` on the +/// current element. Namespace declarations are excluded and source order is +/// retained. This is useful for extensible XML vocabularies whose relationship +/// attribute local names are not known ahead of time. +pub fn XmlStartTagScanner::attribute_values_in_namespace( + self : XmlStartTagScanner, + namespace_uri : StringView, +) -> Array[String] raise ParseXmlError { + let values : Array[String] = [] + let mut next = self.current_name_end + while next_xml_attribute( + self.xml, + next, + self.current_tag_end, + cancelled=self.cancelled, + ) + is Some(attribute) { + next = attribute.next + if xml_namespace_declaration_prefix(self.xml, attribute) is Some(_) { + continue + } + let colon = validate_xml_qname( + self.xml, + attribute.name_start, + attribute.name_end, + ) + let prefix = match colon { + Some(value) => self.xml[attribute.name_start:value] + None => self.xml[attribute.name_start:attribute.name_start] + } + if xml_string_equals_view(self.resolve_prefix(prefix, true), namespace_uri) { + values.push( + decode_xml_attribute( + self.xml[attribute.value_start:attribute.value_end], + cancelled=self.cancelled, + ), + ) + } + } + values +} + ///| priv struct XmlExpandedAttributeName { namespace_uri : String From 026784758995832ef3c88965965401f24cde5e55 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 16:34:42 +0800 Subject: [PATCH 147/150] fix(xlsx): preserve opaque drawing anchors --- xlsx/cell_images.mbt | 1 + xlsx/drawing_preservation.mbt | 462 +++++++++++++++++++++++++++ xlsx/drawing_preservation_wbtest.mbt | 263 +++++++++++++++ xlsx/read.mbt | 23 ++ xlsx/read_drawing_xml.mbt | 71 +++- xlsx/read_sheet_rel_parts.mbt | 1 + xlsx/rich_value_images.mbt | 1 + xlsx/shape.mbt | 2 + xlsx/slicer.mbt | 2 + xlsx/workbook.mbt | 34 ++ xlsx/worksheet.mbt | 14 +- xlsx/worksheet_types.mbt | 12 + xlsx/write.mbt | 239 ++++++++++++-- xlsx/write_rels_xml.mbt | 54 +++- 14 files changed, 1140 insertions(+), 39 deletions(-) create mode 100644 xlsx/drawing_preservation.mbt create mode 100644 xlsx/drawing_preservation_wbtest.mbt diff --git a/xlsx/cell_images.mbt b/xlsx/cell_images.mbt index 4520a6e9..b87d003c 100644 --- a/xlsx/cell_images.mbt +++ b/xlsx/cell_images.mbt @@ -164,6 +164,7 @@ fn CellImage::to_image(self : CellImage, reference : String) -> Image { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } } diff --git a/xlsx/drawing_preservation.mbt b/xlsx/drawing_preservation.mbt new file mode 100644 index 00000000..8ad5f22b --- /dev/null +++ b/xlsx/drawing_preservation.mbt @@ -0,0 +1,462 @@ +///| +/// An opaque, validated drawing anchor retained because the semantic XLSX API +/// does not model its direct object payload yet. +priv struct PreservedDrawingAnchor { + order : Int + xml : String +} + +///| +/// A relationship needed by an opaque drawing anchor or one of its retained +/// package parts. `target_part` is the normalized source-package path for an +/// internal target and absent for an external target. +priv struct PreservedDrawingRelationship { + id : String + rel_type : String + target : String + external : Bool + target_part : String? +} + +///| +/// One internal OPC part in the transitive relationship closure of an opaque +/// drawing anchor. +priv struct PreservedDrawingPart { + source_path : String + data : Bytes + content_type : String + relationships : Array[PreservedDrawingRelationship] +} + +///| +/// An isolated output part used by an opaque drawing anchor. Rehoming retained +/// parts under a writer-owned directory prevents hostile or stale source paths +/// from replacing generated workbook parts. +priv struct PreservedDrawingWritePart { + path : String + data : Bytes + content_type : String + relationships : Array[PreservedDrawingRelationship] +} + +///| +/// The complete write-time projection for one worksheet drawing. +priv struct PreservedDrawingWritePlan { + anchors : Array[PreservedDrawingAnchor] + drawing_relationships : Array[PreservedDrawingRelationship] + parts : Array[PreservedDrawingWritePart] + relationship_ids : Array[String] + object_ids : Array[String] +} + +///| +fn Worksheet::allocate_drawing_order(self : Worksheet) -> Int raise XlsxError { + if self.next_drawing_order == 0x7fff_ffff { + raise InvalidSheetOperation(msg="drawing order exhausted") + } + let order = self.next_drawing_order + self.next_drawing_order = order + 1 + order +} + +///| +fn preserved_drawing_relationships_from_xml( + xml : StringView, + source_part : StringView, + part_names : Map[String, String], + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, +) -> Array[PreservedDrawingRelationship] raise XlsxError { + match budget { + Some(value) => { + value.checkpoint() + value.charge_work(xml.length()) + } + None => () + } + let parsed = @ooxml.parse_package_relationships(xml, cancelled~) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + match budget { + Some(value) => value.charge_items(parsed.length()) + None => () + } + let relationships : Array[PreservedDrawingRelationship] = [] + for record in parsed { + let target_part = if record.external { + None + } else { + Some( + logical_archive_part_path( + actual_relationship_target_path( + source_part, + record.target, + part_names, + cancelled~, + ), + ), + ) + } + relationships.push({ + id: record.id, + rel_type: record.rel_type, + target: record.target, + external: record.external, + target_part, + }) + } + relationships +} + +///| +fn opaque_drawing_relationship_ids( + anchor_xml : StringView, + cancelled? : () -> Bool = () => false, +) -> Array[String] raise XlsxError { + // Canonical drawing anchors retain only supported core element namespaces; + // supply their root bindings so every relationship attribute can be + // enumerated by expanded name even though the retained substring no longer + // contains the original wsDr start tag. + let wrapped = "" + + anchor_xml.to_owned() + + "" + let scanner = @ooxml.XmlStartTagScanner::new(wrapped, cancelled~) + let ids : Array[String] = [] + let seen : Set[String] = Set([]) + while workbook_scanner_next(scanner) { + for + namespace_uri in [ + transitional_relationship_attribute_namespace, strict_relationship_attribute_namespace, + ] { + let values = scanner.attribute_values_in_namespace(namespace_uri) catch { + InvalidXml(msg~) => raise InvalidXml(msg~) + ReadCancelled => raise ReadCancelled + } + for value in values { + if !seen.contains(value) { + seen.add(value) + ids.push(value) + } + } + } + } + ids +} + +///| +fn normalized_drawing_object_id(value : StringView) -> String { + if value == "" { + return "" + } + let mut first_nonzero = 0 + let mut index = 0 + while index < value.length() { + let unit = value[index] + if unit < ('0' : UInt16) || unit > ('9' : UInt16) { + return value.to_owned() + } + if unit == ('0' : UInt16) && first_nonzero == index { + first_nonzero = first_nonzero + 1 + } + index = index + 1 + } + if first_nonzero == value.length() { + "0" + } else { + value[first_nonzero:].to_owned() + } +} + +///| +/// Reserve every numeric drawing object identifier used inside an opaque +/// anchor. Leading zeroes are normalized because `ST_DrawingElementId` is a +/// number even though it is represented as XML text. +fn opaque_drawing_object_ids( + anchors : ArrayView[PreservedDrawingAnchor], +) -> Array[String] raise XlsxError { + let ids : Array[String] = [] + let seen : Set[String] = Set([]) + for anchor in anchors { + let wrapped = "" + + anchor.xml + + "" + let scanner = @ooxml.XmlStartTagScanner::new(wrapped) + while workbook_scanner_next(scanner) { + if scanner.namespace_uri() != transitional_spreadsheet_drawing_namespace || + scanner.local_name() != "cNvPr" { + continue + } + match workbook_scanner_attribute(scanner, "", "id") { + Some(value) => { + let key = normalized_drawing_object_id(value) + if key != "" && !seen.contains(key) { + seen.add(key) + ids.push(key) + } + } + None => () + } + } + } + ids +} + +///| +fn remap_preserved_drawing_relationship( + relationship : PreservedDrawingRelationship, + output_paths : Map[String, String], +) -> PreservedDrawingRelationship raise XlsxError { + if relationship.external { + return relationship + } + let source_target = match relationship.target_part { + Some(value) => value + None => + raise InvalidSheetOperation( + msg="preserved drawing relationship target missing", + ) + } + let output_target = match + output_paths.get(@ooxml.package_part_name_key(source_target)) { + Some(value) => value + None => + raise InvalidSheetOperation( + msg="preserved drawing relationship closure incomplete", + ) + } + { + ..relationship + // A package-absolute target is valid OPC and avoids depending on the + // source part's former directory depth after isolation. + , + target: "/" + output_target, + target_part: Some(output_target), + } +} + +///| +fn preserved_drawing_write_plan( + sheet : Worksheet, + sheet_id : Int, +) -> PreservedDrawingWritePlan raise XlsxError { + let output_paths : Map[String, String] = Map([]) + for index, part in sheet.preserved_drawing_parts { + let key = @ooxml.package_part_name_key(part.source_path) + if output_paths.contains(key) { + raise InvalidSheetOperation(msg="duplicate preserved drawing part") + } + output_paths[key] = "xl/preservedDrawingParts/drawing\{sheet_id}/part\{index + 1}.bin" + } + let parts : Array[PreservedDrawingWritePart] = [] + for part in sheet.preserved_drawing_parts { + let path = output_paths + .get(@ooxml.package_part_name_key(part.source_path)) + .unwrap() + let relationships : Array[PreservedDrawingRelationship] = [] + for relationship in part.relationships { + relationships.push( + remap_preserved_drawing_relationship(relationship, output_paths), + ) + } + parts.push({ + path, + data: part.data, + content_type: part.content_type, + relationships, + }) + } + let drawing_relationships : Array[PreservedDrawingRelationship] = [] + let relationship_ids : Array[String] = [] + let relationship_ids_seen : Set[String] = Set([]) + for relationship in sheet.preserved_drawing_relationships { + if relationship_ids_seen.contains(relationship.id) { + raise InvalidSheetOperation( + msg="duplicate preserved drawing relationship id", + ) + } + relationship_ids_seen.add(relationship.id) + relationship_ids.push(relationship.id) + drawing_relationships.push( + remap_preserved_drawing_relationship(relationship, output_paths), + ) + } + { + anchors: sheet.preserved_drawing_anchors.copy(), + drawing_relationships, + parts, + relationship_ids, + object_ids: opaque_drawing_object_ids(sheet.preserved_drawing_anchors), + } +} + +///| +fn collect_preserved_drawing_parts( + initial_paths : ArrayView[String], + archive : @zip.Archive, + part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, + decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, +) -> Array[PreservedDrawingPart] raise XlsxError { + let parts : Array[PreservedDrawingPart] = [] + let queue : Array[String] = [] + queue.append(initial_paths) + let seen : Set[String] = Set([]) + let mut cursor = 0 + while cursor < queue.length() { + match budget { + Some(value) => value.checkpoint() + None => () + } + let source_path = queue[cursor] + cursor = cursor + 1 + let key = @ooxml.package_part_name_key(source_path) + if seen.contains(key) { + continue + } + seen.add(key) + let actual_path = actual_archive_part_path(part_names, source_path) + let data = match archive.get(actual_path) { + Some(value) => value.to_owned() + None => raise MissingPart(path=actual_path) + } + let content_type = declared_part_content_type( + content_types, + source_path, + "preserved drawing part", + cancelled~, + ) + let relationships = match + load_optional_relationship_part( + archive, + content_types, + actual_relationship_part_path(actual_path, part_names), + "preserved drawing relationships", + cancelled~, + ) { + Some(bytes) => + preserved_drawing_relationships_from_xml( + decode(bytes), + source_path, + part_names, + budget?, + cancelled~, + ) + None => [] + } + for relationship in relationships { + match relationship.target_part { + Some(target_part) => queue.push(target_part) + None => () + } + } + match budget { + Some(value) => value.charge_items(1) + None => () + } + parts.push({ source_path, data, content_type, relationships }) + } + parts +} + +///| +fn preserved_unsupported_drawing_state( + anchors : ArrayView[DrawingAnchor], + drawing_rels_xml : StringView, + drawing_part : StringView, + archive : @zip.Archive, + part_names : Map[String, String], + content_types : @ooxml.PackageContentTypes, + decode : (BytesView) -> String raise XlsxError, + budget? : ReadBudget, + cancelled? : () -> Bool = () => false, +) -> ( + Array[PreservedDrawingAnchor], + Array[PreservedDrawingRelationship], + Array[PreservedDrawingPart], +) raise XlsxError { + let opaque_anchors : Array[PreservedDrawingAnchor] = [] + let required_ids : Array[String] = [] + let required_ids_seen : Set[String] = Set([]) + for anchor in anchors { + if !anchor.unsupported { + continue + } + opaque_anchors.push({ order: anchor.order, xml: anchor.xml }) + for id in opaque_drawing_relationship_ids(anchor.xml, cancelled~) { + if !required_ids_seen.contains(id) { + required_ids_seen.add(id) + required_ids.push(id) + } + } + } + if opaque_anchors.length() == 0 || required_ids.length() == 0 { + return (opaque_anchors, [], []) + } + if drawing_rels_xml == "" { + raise InvalidXml(msg="unsupported drawing relationship missing") + } + let all_relationships = preserved_drawing_relationships_from_xml( + drawing_rels_xml, + drawing_part, + part_names, + budget?, + cancelled~, + ) + let relationship_indices : Map[String, Int] = Map([]) + for index, relationship in all_relationships { + relationship_indices[relationship.id] = index + } + let direct_relationships : Array[PreservedDrawingRelationship] = [] + let initial_paths : Array[String] = [] + let initial_seen : Set[String] = Set([]) + for id in required_ids { + let relationship = match relationship_indices.get(id) { + Some(index) => all_relationships[index] + None => raise InvalidXml(msg="unsupported drawing relationship missing") + } + direct_relationships.push(relationship) + match relationship.target_part { + Some(path) => { + let key = @ooxml.package_part_name_key(path) + if !initial_seen.contains(key) { + initial_seen.add(key) + initial_paths.push(path) + } + } + None => () + } + } + let parts = collect_preserved_drawing_parts( + initial_paths, + archive, + part_names, + content_types, + decode, + budget?, + cancelled~, + ) + (opaque_anchors, direct_relationships, parts) +} diff --git a/xlsx/drawing_preservation_wbtest.mbt b/xlsx/drawing_preservation_wbtest.mbt new file mode 100644 index 00000000..21b1f3bd --- /dev/null +++ b/xlsx/drawing_preservation_wbtest.mbt @@ -0,0 +1,263 @@ +///| +fn drawing_preservation_test_utf8( + archive : @zip.Archive, + path : String, +) -> String raise Error { + let bytes = match archive.get(path) { + Some(value) => value + None => fail("missing archive part: \{path}") + } + @encoding/utf8.decode(bytes) catch { + _ => fail("archive part is not UTF-8: \{path}") + } +} + +///| +fn drawing_preservation_test_position( + xml : String, + marker : String, +) -> Int raise Error { + match xml.find(marker) { + Some(value) => value + None => fail("missing drawing marker: \{marker}") + } +} + +///| +fn drawing_preservation_test_count( + value : StringView, + needle : StringView, +) -> Int { + if needle == "" { + return 0 + } + let mut count = 0 + let mut cursor = 0 + while cursor + needle.length() <= value.length() { + match value[cursor:].find(needle) { + Some(offset) => { + count = count + 1 + cursor = cursor + offset + needle.length() + } + None => break + } + } + count +} + +///| +fn drawing_preservation_test_fixture( + include_opaque_relationship : Bool, +) -> Bytes raise Error { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("Sheet1") + sheet.add_shape(Shape::new("A1", "rect", text="known-shape")) + sheet.add_image( + "B1", + b"synthetic image", + "png", + "image/png", + name="Known Image", + ) + sheet.add_chart( + "C1", + ( + #| + #| + #| + ), + ) + let archive = @zip.read(write(workbook)) + let drawing_path = "xl/drawings/drawing1.xml" + let drawing_rels_path = "xl/drawings/_rels/drawing1.xml.rels" + let drawing = drawing_preservation_test_utf8(archive, drawing_path) + let generated = parse_drawing_anchors(drawing) + let mut image_anchor : String? = None + let mut chart_anchor : String? = None + let mut shape_anchor : String? = None + for anchor in generated { + if anchor.xml.contains("") { + image_anchor = Some(anchor.xml) + } else if anchor.xml.contains("") { + shape_anchor = Some(anchor.xml) + } + } + let shape = shape_anchor.unwrap_or_else(() => fail("missing shape anchor")) + let image = image_anchor + .unwrap_or_else(() => fail("missing image anchor")) + .replace_all(old="rId1", new="rId5") + .replace(old="id=\"1\"", new="id=\"5\"") + let chart = chart_anchor.unwrap_or_else(() => fail("missing chart anchor")) + // The public writer must honor insertion order even though its individual + // serializers are organized by object type. + assert_true( + drawing_preservation_test_position(drawing, "known-shape") < + drawing_preservation_test_position(drawing, ""), + ) + assert_true( + drawing_preservation_test_position(drawing, "") < + drawing_preservation_test_position(drawing, "3000 + let opaque_group = + #|4000 + let patched_drawing = "\n" + + "\n" + + shape + + "\n" + + opaque_content + + image + + "\n" + + opaque_group + + chart + + "\n" + assert_true( + archive.replace(drawing_path, @encoding/utf8.encode(patched_drawing)), + ) + let original_rels = drawing_preservation_test_utf8(archive, drawing_rels_path) + let opaque_relationship = if include_opaque_relationship { + " \n" + } else { + "" + } + let patched_rels = original_rels + .replace(old="Id=\"rId1\"", new="Id=\"rId5\"") + .replace( + old="", + new=opaque_relationship + "", + ) + assert_true( + archive.replace(drawing_rels_path, @encoding/utf8.encode(patched_rels)), + ) + archive.add("custom/opaque.xml", b"payload") + archive.add("custom/nested/data.bin", b"opaque nested payload") + archive.add( + "custom/_rels/opaque.xml.rels", + @encoding/utf8.encode( + ( + #| + #| + #| + #| + #| + #| + ), + ), + ) + let content_types_path = "[Content_Types].xml" + let content_types = drawing_preservation_test_utf8( + archive, content_types_path, + ) + let patched_content_types = content_types.replace( + old="", + new="" + + "" + + "", + ) + assert_true( + archive.replace( + content_types_path, + @encoding/utf8.encode(patched_content_types), + ), + ) + @zip.write(archive) +} + +///| +fn assert_drawing_preservation_test_output( + archive : @zip.Archive, +) -> Unit raise Error { + let drawing = drawing_preservation_test_utf8( + archive, "xl/drawings/drawing1.xml", + ) + let shape_pos = drawing_preservation_test_position(drawing, "known-shape") + let content_pos = drawing_preservation_test_position( + drawing, "") + let group_pos = drawing_preservation_test_position(drawing, "Opaque Group") + let chart_pos = drawing_preservation_test_position(drawing, "payload", + ) + assert_true( + archive.get(nested_part_path).unwrap_or(b"") == b"opaque nested payload", + ) + let opaque_rels = drawing_preservation_test_utf8( + archive, "xl/preservedDrawingParts/drawing1/_rels/part1.bin.rels", + ) + assert_true( + opaque_rels.contains( + "Id=\"child\" Type=\"urn:example:child\" Target=\"/xl/preservedDrawingParts/drawing1/part2.bin\"", + ), + ) + assert_true( + opaque_rels.contains( + "Id=\"external\" Type=\"urn:example:external\" Target=\"https://example.invalid/resource\" TargetMode=\"External\"", + ), + ) + let content_types = drawing_preservation_test_utf8( + archive, "[Content_Types].xml", + ) + assert_true( + content_types.contains( + "PartName=\"/xl/preservedDrawingParts/drawing1/part1.bin\" ContentType=\"application/vnd.example.opaque+xml\"", + ), + ) + assert_true( + content_types.contains( + "PartName=\"/xl/preservedDrawingParts/drawing1/part2.bin\" ContentType=\"application/vnd.example.opaque-data\"", + ), + ) + assert_true(archive.get("custom/opaque.xml") is None) +} + +///| +test "opaque drawing anchors preserve order ids and relationship closure" { + let parsed = read(drawing_preservation_test_fixture(true)) + let first = @zip.read(write(parsed)) + assert_drawing_preservation_test_output(first) + // A second read/write cycle proves the isolated projection is itself a + // complete, readable OPC subgraph rather than a one-shot copy operation. + let second = @zip.read(write(read(@zip.write(first)))) + assert_drawing_preservation_test_output(second) +} + +///| +test "opaque drawing anchors reject missing relationships" { + try read(drawing_preservation_test_fixture(false)) catch { + InvalidXml(msg~) => + assert_eq(msg, "unsupported drawing relationship missing") + _ => fail("unexpected opaque drawing relationship error") + } noraise { + _ => fail("expected missing opaque drawing relationship rejection") + } +} diff --git a/xlsx/read.mbt b/xlsx/read.mbt index 61a48c92..fc89172f 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -4779,6 +4779,10 @@ fn read_zip_archive_core_unchecked( col_dimensions, state, stream_state: Idle, + next_drawing_order: 0, + preserved_drawing_anchors: [], + preserved_drawing_relationships: [], + preserved_drawing_parts: [], cell_vm: if rich_value_images is Some(_) { parse_cell_vm_map(sheet_xml) } else { @@ -4830,6 +4834,7 @@ fn read_zip_archive_core_unchecked( drawing_xml, budget=read_budget, ) + sheet.next_drawing_order = drawing_anchors.length() if slicer_part_entries.length() > 0 { slicer_anchor_info = parse_drawing_slicer_anchors( drawing_anchors, @@ -4848,6 +4853,20 @@ fn read_zip_archive_core_unchecked( Some(value) => decode(value) None => "" } + let (preserved_anchors, preserved_relationships, preserved_parts) = preserved_unsupported_drawing_state( + drawing_anchors, + drawing_rels_xml, + drawing_path, + archive, + part_names, + content_types, + decode, + budget=read_budget, + cancelled~, + ) + sheet.preserved_drawing_anchors.append(preserved_anchors) + sheet.preserved_drawing_relationships.append(preserved_relationships) + sheet.preserved_drawing_parts.append(preserved_parts) let drawing_images = parse_drawing_images( drawing_anchors, drawing_rels_xml, @@ -4961,6 +4980,10 @@ fn read_zip_archive_core_unchecked( Some(info) => Some(info.height_emu) None => None }, + drawing_order: match slicer_anchor_info.get(entry.name) { + Some(info) => Some(info.order) + None => None + }, }) } workbook.sheets.push(sheet) diff --git a/xlsx/read_drawing_xml.mbt b/xlsx/read_drawing_xml.mbt index ccb7f797..b11be17a 100644 --- a/xlsx/read_drawing_xml.mbt +++ b/xlsx/read_drawing_xml.mbt @@ -127,6 +127,8 @@ priv struct DrawingAnchor { xml : String positioning : PicturePositioning form : DrawingAnchorForm + order : Int + unsupported : Bool } ///| @@ -367,9 +369,13 @@ fn parse_drawing_anchors( None => TwoCell } } - if !drawing_anchor_has_unsupported_reader_object(full) { - anchors.push({ xml: full, positioning, form }) - } + anchors.push({ + xml: full, + positioning, + form, + order: scanned, + unsupported: drawing_anchor_has_unsupported_reader_object(full), + }) scanned += 1 cursor = close_end + 1 } @@ -405,6 +411,9 @@ fn single_drawing_chart_rel_id( ) -> String raise XlsxError { let mut selected : String? = None for anchor in anchors { + if anchor.unsupported { + continue + } let anchor_xml = anchor.xml charge_drawing_anchor_pass(budget, anchor_xml) match tag_attributes_in(anchor_xml, "c:chart") { @@ -649,6 +658,9 @@ fn parse_drawing_images( ) } for anchor in anchors { + if anchor.unsupported { + continue + } let anchor_xml = anchor.xml let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) @@ -758,6 +770,7 @@ fn parse_drawing_images( drawing_offset_y_emu: Some(origin.row_off_emu), drawing_width_emu: Some(width_emu), drawing_height_emu: Some(height_emu), + drawing_order: Some(anchor.order), }) } images @@ -788,6 +801,9 @@ fn parse_drawing_charts( ) } for anchor in anchors { + if anchor.unsupported { + continue + } let anchor_xml = anchor.xml let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) @@ -852,6 +868,7 @@ fn parse_drawing_charts( drawing_offset_y_emu: Some(origin.row_off_emu), drawing_width_emu: Some(width_emu), drawing_height_emu: Some(height_emu), + drawing_order: Some(anchor.order), }) } charts @@ -865,6 +882,9 @@ fn parse_drawing_shapes( ) -> Array[Shape] raise XlsxError { let shapes : Array[Shape] = [] for anchor in anchors { + if anchor.unsupported { + continue + } let anchor_xml = anchor.xml let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) @@ -1033,6 +1053,7 @@ fn parse_drawing_shapes( drawing_offset_y_emu: Some(origin.row_off_emu), drawing_width_emu: Some(cx), drawing_height_emu: Some(cy), + drawing_order: Some(anchor.order), }) } shapes @@ -1054,6 +1075,7 @@ priv struct SlicerAnchorInfo { offset_y_emu : Int width_emu : Int height_emu : Int + order : Int } ///| @@ -1064,6 +1086,9 @@ fn parse_drawing_slicer_anchors( ) -> Map[String, SlicerAnchorInfo] raise XlsxError { let out : Map[String, SlicerAnchorInfo] = Map([]) for anchor in anchors { + if anchor.unsupported { + continue + } let anchor_xml = anchor.xml let positioning = anchor.positioning charge_drawing_anchor_pass(budget, anchor_xml) @@ -1134,6 +1159,7 @@ fn parse_drawing_slicer_anchors( offset_y_emu: origin.row_off_emu, width_emu, height_emu, + order: anchor.order, } } out @@ -1272,7 +1298,7 @@ test "DrawingML extension payload cannot spoof anchors or drawing features" { } ///| -test "DrawingML ignores valid unsupported anchor objects as complete subtrees" { +test "DrawingML retains valid unsupported anchors without reading nested objects" { let source = #| #| 0000 @@ -1283,10 +1309,19 @@ test "DrawingML ignores valid unsupported anchor objects as complete subtrees" { ) assert_true(drawing.contains("")) let anchors = parse_drawing_anchors(drawing) - inspect(anchors.length(), content="1") - let anchor_xml = anchors[0].xml + inspect(anchors.length(), content="2") + assert_true(anchors[0].unsupported) + assert_true(anchors[0].xml.contains("name=\"nested\"")) + assert_false(anchors[1].unsupported) + let anchor_xml = anchors[1].xml assert_true(anchor_xml.contains("name=\"direct\"")) assert_false(anchor_xml.contains("name=\"nested\"")) + let shapes = parse_drawing_shapes( + anchors, + drawing_metrics_for_sheet(Worksheet::new("Sheet1")), + ) + assert_eq(shapes.length(), 1) + assert_eq(shapes[0].name, "direct") } ///| @@ -1442,7 +1477,13 @@ test "chartsheet drawing selector requires exactly one chart payload" { let anchor = #| let one : ReadOnlyArray[DrawingAnchor] = [ - { xml: anchor, positioning: OneCell, form: OneCellAnchor }, + { + xml: anchor, + positioning: OneCell, + form: OneCellAnchor, + order: 0, + unsupported: false, + }, ] assert_eq(single_drawing_chart_rel_id(one), "selected") let empty : ReadOnlyArray[DrawingAnchor] = [] @@ -1453,8 +1494,20 @@ test "chartsheet drawing selector requires exactly one chart payload" { _ => fail("expected empty chartsheet drawing rejection") } let two : ReadOnlyArray[DrawingAnchor] = [ - { xml: anchor, positioning: OneCell, form: OneCellAnchor }, - { xml: anchor, positioning: OneCell, form: OneCellAnchor }, + { + xml: anchor, + positioning: OneCell, + form: OneCellAnchor, + order: 0, + unsupported: false, + }, + { + xml: anchor, + positioning: OneCell, + form: OneCellAnchor, + order: 1, + unsupported: false, + }, ] try single_drawing_chart_rel_id(two) catch { InvalidXml(msg~) => assert_eq(msg, "chartsheet drawing has multiple charts") diff --git a/xlsx/read_sheet_rel_parts.mbt b/xlsx/read_sheet_rel_parts.mbt index fba7022f..a40247a0 100644 --- a/xlsx/read_sheet_rel_parts.mbt +++ b/xlsx/read_sheet_rel_parts.mbt @@ -484,6 +484,7 @@ fn resolve_slicer_sources_after_read( drawing_offset_y_emu: slicer.drawing_offset_y_emu, drawing_width_emu: slicer.drawing_width_emu, drawing_height_emu: slicer.drawing_height_emu, + drawing_order: slicer.drawing_order, } } } diff --git a/xlsx/rich_value_images.mbt b/xlsx/rich_value_images.mbt index fb2dbeb2..85548e11 100644 --- a/xlsx/rich_value_images.mbt +++ b/xlsx/rich_value_images.mbt @@ -533,6 +533,7 @@ fn Workbook::rich_value_image_for_cell( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, }) } None => None diff --git a/xlsx/shape.mbt b/xlsx/shape.mbt index 77940629..121508f0 100644 --- a/xlsx/shape.mbt +++ b/xlsx/shape.mbt @@ -85,6 +85,7 @@ pub struct Shape { priv drawing_offset_y_emu : Int? priv drawing_width_emu : Int? priv drawing_height_emu : Int? + priv drawing_order : Int? } derive(Debug) ///| @@ -210,5 +211,6 @@ pub fn Shape::new( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } } diff --git a/xlsx/slicer.mbt b/xlsx/slicer.mbt index bb5ac5f0..0afe8227 100644 --- a/xlsx/slicer.mbt +++ b/xlsx/slicer.mbt @@ -17,6 +17,7 @@ pub struct Slicer { priv drawing_offset_y_emu : Int? priv drawing_width_emu : Int? priv drawing_height_emu : Int? + priv drawing_order : Int? } derive(Debug) ///| @@ -61,6 +62,7 @@ pub fn Slicer::new(name : String) -> Slicer raise XlsxError { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } } diff --git a/xlsx/workbook.mbt b/xlsx/workbook.mbt index 02d48ed7..b42d5058 100644 --- a/xlsx/workbook.mbt +++ b/xlsx/workbook.mbt @@ -1266,6 +1266,31 @@ fn clone_slicers(values : ArrayView[Slicer]) -> Array[Slicer] { out } +///| +fn clone_preserved_drawing_relationships( + values : ArrayView[PreservedDrawingRelationship], +) -> Array[PreservedDrawingRelationship] { + let out : Array[PreservedDrawingRelationship] = [] + for value in values { + out.push(value) + } + out +} + +///| +fn clone_preserved_drawing_parts( + values : ArrayView[PreservedDrawingPart], +) -> Array[PreservedDrawingPart] { + let out : Array[PreservedDrawingPart] = [] + for value in values { + out.push({ + ..value, + relationships: clone_preserved_drawing_relationships(value.relationships), + }) + } + out +} + ///| fn clone_ignored_errors( values : ArrayView[IgnoredError], @@ -1394,6 +1419,14 @@ fn clone_worksheet( stream_state: Idle, vml_drawing_xml: sheet.vml_drawing_xml, vml_drawing_hf_xml: sheet.vml_drawing_hf_xml, + next_drawing_order: sheet.next_drawing_order, + preserved_drawing_anchors: sheet.preserved_drawing_anchors.copy(), + preserved_drawing_relationships: clone_preserved_drawing_relationships( + sheet.preserved_drawing_relationships, + ), + preserved_drawing_parts: clone_preserved_drawing_parts( + sheet.preserved_drawing_parts, + ), cell_vm: { let copy : Map[String, Int] = Map([]) for reference, vm in sheet.cell_vm { @@ -2691,6 +2724,7 @@ pub fn Workbook::add_slicer( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } sheet.add_slicer(slicer) } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 6fd429b3..2b4d18db 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -2025,6 +2025,7 @@ pub fn Worksheet::add_image( } else { reference } + let drawing_order = self.allocate_drawing_order() self.images.push({ reference: stored_reference, data, @@ -2048,6 +2049,7 @@ pub fn Worksheet::add_image( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: Some(drawing_order), }) } @@ -2076,6 +2078,7 @@ fn image_with_reference(image : Image, reference : String) -> Image { drawing_offset_y_emu: image.drawing_offset_y_emu, drawing_width_emu: image.drawing_width_emu, drawing_height_emu: image.drawing_height_emu, + drawing_order: image.drawing_order, } } @@ -2095,6 +2098,7 @@ fn chart_with_reference(chart : Chart, reference : String) -> Chart { drawing_offset_y_emu: chart.drawing_offset_y_emu, drawing_width_emu: chart.drawing_width_emu, drawing_height_emu: chart.drawing_height_emu, + drawing_order: chart.drawing_order, } } @@ -2157,6 +2161,7 @@ pub fn Worksheet::add_chart( xml : String, ) -> Unit raise XlsxError { let (_row, _col) = cell_ref_to_rc(reference) + let drawing_order = self.allocate_drawing_order() self.charts.push({ reference, xml, @@ -2171,6 +2176,7 @@ pub fn Worksheet::add_chart( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: Some(drawing_order), }) } @@ -2228,6 +2234,7 @@ pub fn Worksheet::add_chart_with_options( "drawing chart height", ) let xml = chart_options_to_xml(opts) + let drawing_order = self.allocate_drawing_order() self.charts.push({ reference, xml, @@ -2242,6 +2249,7 @@ pub fn Worksheet::add_chart_with_options( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: Some(drawing_order), }) } @@ -2270,7 +2278,8 @@ pub fn Worksheet::add_shape( shape : Shape, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.shapes.push(shape) + let drawing_order = self.allocate_drawing_order() + self.shapes.push({ ..shape, drawing_order: Some(drawing_order) }) } ///| @@ -2341,7 +2350,8 @@ pub fn Worksheet::add_slicer( slicer : Slicer, ) -> Unit raise XlsxError { self.ensure_stream_idle() - self.slicers.push(slicer) + let drawing_order = self.allocate_drawing_order() + self.slicers.push({ ..slicer, drawing_order: Some(drawing_order) }) } ///| diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index f8adf0c7..ed7641b3 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -56,6 +56,7 @@ pub struct Image { priv drawing_offset_y_emu : Int? priv drawing_width_emu : Int? priv drawing_height_emu : Int? + priv drawing_order : Int? } ///| @@ -73,6 +74,7 @@ pub struct Chart { priv drawing_offset_y_emu : Int? priv drawing_width_emu : Int? priv drawing_height_emu : Int? + priv drawing_order : Int? } ///| @@ -144,6 +146,12 @@ pub struct Worksheet { mut stream_state : StreamState mut vml_drawing_xml : String? mut vml_drawing_hf_xml : String? + /// Next stable cross-type drawing order. Source anchors occupy the initial + /// range; newly added objects append after them regardless of object kind. + priv mut next_drawing_order : Int + priv preserved_drawing_anchors : Array[PreservedDrawingAnchor] + priv preserved_drawing_relationships : Array[PreservedDrawingRelationship] + priv preserved_drawing_parts : Array[PreservedDrawingPart] /// Cell value-metadata (`vm`) indices captured on read, keyed by /// canonical cell reference. Links a cell to its rich-value embedded /// image; lives on the worksheet so it survives rename/copy/delete. @@ -197,6 +205,10 @@ pub fn Worksheet::new(name : String) -> Worksheet { stream_state: Idle, vml_drawing_xml: None, vml_drawing_hf_xml: None, + next_drawing_order: 0, + preserved_drawing_anchors: [], + preserved_drawing_relationships: [], + preserved_drawing_parts: [], cell_vm: Map([]), } } diff --git a/xlsx/write.mbt b/xlsx/write.mbt index 2314be99..95f5e1ad 100644 --- a/xlsx/write.mbt +++ b/xlsx/write.mbt @@ -252,6 +252,7 @@ priv struct DrawingChart { drawing_offset_y_emu : Int? drawing_width_emu : Int? drawing_height_emu : Int? + drawing_order : Int? } ///| @@ -267,6 +268,13 @@ priv struct DrawingSlicer { is_table_slicer : Bool } +///| +priv struct OrderedDrawingAnchor { + xml : String + order : Int? + fallback_order : Int +} + ///| priv struct ResolvedSlicerCache { cache_id : Int @@ -392,6 +400,81 @@ fn two_cell_axis_to( ) } +///| +fn write_drawing_open(sb : StringBuilder) -> Unit { + sb.write_view("\n") + sb.write_view( + "\n", + ) +} + +///| +fn allocate_drawing_numeric_id( + next : Int, + reserved : Set[String], +) -> (Int, Int) raise XlsxError { + let mut candidate = next + while reserved.contains("\{candidate}") { + if candidate == 0x7fff_ffff { + raise InvalidSheetOperation(msg="drawing object id exhausted") + } + candidate = candidate + 1 + } + if candidate == 0x7fff_ffff { + raise InvalidSheetOperation(msg="drawing object id exhausted") + } + (candidate, candidate + 1) +} + +///| +fn allocate_drawing_relationship_id( + next : Int, + reserved : Set[String], +) -> (Int, Int) raise XlsxError { + let mut candidate = next + while reserved.contains("rId\{candidate}") { + if candidate == 0x7fff_ffff { + raise InvalidSheetOperation(msg="drawing relationship id exhausted") + } + candidate = candidate + 1 + } + if candidate == 0x7fff_ffff { + raise InvalidSheetOperation(msg="drawing relationship id exhausted") + } + (candidate, candidate + 1) +} + +///| +fn compare_ordered_drawing_anchor( + left : OrderedDrawingAnchor, + right : OrderedDrawingAnchor, +) -> Int { + match (left.order, right.order) { + (Some(a), Some(b)) => + if a < b { + -1 + } else if a > b { + 1 + } else if left.fallback_order < right.fallback_order { + -1 + } else if left.fallback_order > right.fallback_order { + 1 + } else { + 0 + } + (Some(_), None) => -1 + (None, Some(_)) => 1 + (None, None) => + if left.fallback_order < right.fallback_order { + -1 + } else if left.fallback_order > right.fallback_order { + 1 + } else { + 0 + } + } +} + ///| fn write_drawing_xml( sheet : Worksheet?, @@ -399,24 +482,29 @@ fn write_drawing_xml( charts : Array[DrawingChart], shapes : Array[Shape], slicers : Array[DrawingSlicer], + preserved_anchors? : ArrayView[PreservedDrawingAnchor] = [], + reserved_object_ids? : ArrayView[String] = [], ) -> String raise XlsxError { let drawing_metrics = match sheet { Some(value) => drawing_metrics_for_sheet(value) None => drawing_metrics_for_sheet(Worksheet::new("Drawing")) } let sb = StringBuilder::new() - sb.write_view("\n") - sb.write_view( - "\n", - ) + write_drawing_open(sb) + let reserved_object_id_set : Set[String] = Set([]) + for id in reserved_object_ids { + reserved_object_id_set.add(id) + } let mut obj_id = 1 for item in images { let image = item.image let (row, col) = cell_ref_to_rc(image.reference) let col_idx = col - 1 let row_idx = row - 1 - let pic_id = obj_id - obj_id = obj_id + 1 + let (pic_id, next_obj_id) = allocate_drawing_numeric_id( + obj_id, reserved_object_id_set, + ) + obj_id = next_obj_id let col_off = match image.drawing_offset_x_emu { Some(value) => value None => @@ -575,8 +663,10 @@ fn write_drawing_xml( let (row, col) = cell_ref_to_rc(chart.reference) let col_idx = col - 1 let row_idx = row - 1 - let frame_id = obj_id - obj_id = obj_id + 1 + let (frame_id, next_obj_id) = allocate_drawing_numeric_id( + obj_id, reserved_object_id_set, + ) + obj_id = next_obj_id let col_off = match chart.drawing_offset_x_emu { Some(value) => value None => @@ -719,8 +809,10 @@ fn write_drawing_xml( None => drawing_pixel_offset_for_write(offset_y, "drawing shape row offset") } - let shape_id = obj_id - obj_id = obj_id + 1 + let (shape_id, next_obj_id) = allocate_drawing_numeric_id( + obj_id, reserved_object_id_set, + ) + obj_id = next_obj_id let cx = match shape.drawing_width_emu { Some(value) => drawing_extent_for_write(value, "drawing shape width") None => @@ -957,8 +1049,10 @@ fn write_drawing_xml( let (row, col) = cell_ref_to_rc(slicer.cell) let col_idx = col - 1 let row_idx = row - 1 - let frame_id = obj_id - obj_id = obj_id + 1 + let (frame_id, next_obj_id) = allocate_drawing_numeric_id( + obj_id, reserved_object_id_set, + ) + obj_id = next_obj_id let offset_x = match slicer.format.offset_x { Some(v) => v None => 0 @@ -1200,7 +1294,65 @@ fn write_drawing_xml( } } sb.write_view("") - sb.to_string() + let grouped_xml = sb.to_string() + let known_orders : Array[Int?] = [] + let mut needs_ordering = preserved_anchors.length() > 0 + for item in images { + known_orders.push(item.image.drawing_order) + if item.image.drawing_order is Some(_) { + needs_ordering = true + } + } + for item in charts { + known_orders.push(item.drawing_order) + if item.drawing_order is Some(_) { + needs_ordering = true + } + } + for shape in shapes { + known_orders.push(shape.drawing_order) + if shape.drawing_order is Some(_) { + needs_ordering = true + } + } + for item in slicers { + known_orders.push(item.slicer.drawing_order) + if item.slicer.drawing_order is Some(_) { + needs_ordering = true + } + } + if !needs_ordering { + return grouped_xml + } + let generated_anchors = parse_drawing_anchors(grouped_xml) + if generated_anchors.length() != known_orders.length() { + raise InvalidSheetOperation(msg="generated drawing anchor count mismatch") + } + let ordered : Array[OrderedDrawingAnchor] = [] + for index, anchor in generated_anchors { + ordered.push({ + xml: anchor.xml, + order: known_orders[index], + fallback_order: index, + }) + } + for index, anchor in preserved_anchors { + ordered.push({ + xml: anchor.xml, + order: Some(anchor.order), + fallback_order: generated_anchors.length() + index, + }) + } + ordered.sort_by(compare_ordered_drawing_anchor) + let output = StringBuilder::new() + write_drawing_open(output) + for anchor in ordered { + output.write_view(" ") + output.write_view(anchor.xml) + output.write_view("\n") + } + output.write_view("") + output.to_string() } ///| @@ -2611,12 +2763,29 @@ fn write_with_io( let mut slicer_part_index = 1 for i, sheet in workbook.sheets() { let sheet_id = i + 1 + let preserved_drawing = preserved_drawing_write_plan(sheet, sheet_id) + for part in preserved_drawing.parts { + archive.add(part.path, part.data) + manifest.add_content_override( + content_override_path(part.path), + part.content_type, + ) + if part.relationships.length() > 0 { + archive.add( + rels_path_for_part(part.path), + @encoding/utf8.encode( + write_preserved_drawing_rels_xml(part.relationships), + ), + ) + } + } let sheet_tables = tables_by_sheet[i] let sheet_pivots = pivots_by_sheet[i] let has_drawing = sheet.images().length() > 0 || sheet.charts().length() > 0 || sheet.shapes().length() > 0 || - sheet.slicers().length() > 0 + sheet.slicers().length() > 0 || + preserved_drawing.anchors.length() > 0 let mut rel_id = 1 let drawing_rel_id = if has_drawing { let id = rel_id @@ -2986,17 +3155,25 @@ fn write_with_io( let drawing_charts : Array[DrawingChart] = [] let drawing_shapes : Array[Shape] = [] let drawing_slicers : Array[DrawingSlicer] = [] + let reserved_relationship_ids : Set[String] = Set([]) + for id in preserved_drawing.relationship_ids { + reserved_relationship_ids.add(id) + } let mut rel_id = 1 for image in sheet.images() { let file_name = "image\{image_index}.\{image.extension}" image_index = image_index + 1 archive.add(media_part_path(file_name), image.data) - let image_rel_id = rel_id - rel_id = rel_id + 1 + let (image_rel_id, next_rel_id) = allocate_drawing_relationship_id( + rel_id, reserved_relationship_ids, + ) + rel_id = next_rel_id let mut hyperlink_rel_id : Int? = None if image.hyperlink != "" && image.hyperlink_type != Unset { - let link_rel_id = rel_id - rel_id = rel_id + 1 + let (link_rel_id, next_rel_id) = allocate_drawing_relationship_id( + rel_id, reserved_relationship_ids, + ) + rel_id = next_rel_id drawing_hyperlinks.push({ rel_id: link_rel_id, target: image.hyperlink, @@ -3015,7 +3192,11 @@ fn write_with_io( let file_name = "chart\{chart_index}.xml" chart_index = chart_index + 1 archive.add(chart_part_path(file_name), @encoding/utf8.encode(chart.xml)) - chart_targets.push((rel_id, chart_rel_target(file_name))) + let (chart_rel_id, next_rel_id) = allocate_drawing_relationship_id( + rel_id, reserved_relationship_ids, + ) + rel_id = next_rel_id + chart_targets.push((chart_rel_id, chart_rel_target(file_name))) drawing_charts.push({ reference: chart.reference, offset_x: chart.offset_x, @@ -3025,13 +3206,13 @@ fn write_with_io( print_object: chart.print_object, locked: chart.locked, positioning: chart.positioning, - rel_id, + rel_id: chart_rel_id, drawing_offset_x_emu: chart.drawing_offset_x_emu, drawing_offset_y_emu: chart.drawing_offset_y_emu, drawing_width_emu: chart.drawing_width_emu, drawing_height_emu: chart.drawing_height_emu, + drawing_order: chart.drawing_order, }) - rel_id = rel_id + 1 manifest.add_content_override( content_override_path(chart_part_path(file_name)), ct_chart, @@ -3056,10 +3237,15 @@ fn write_with_io( drawing_charts, drawing_shapes, drawing_slicers, + preserved_anchors=preserved_drawing.anchors, + reserved_object_ids=preserved_drawing.object_ids, ) archive.add(drawing_part_path(sheet_id), @encoding/utf8.encode(drawing_xml)) let drawing_rels_xml = write_drawing_rels_xml( - image_targets, chart_targets, drawing_hyperlinks, + image_targets, + chart_targets, + drawing_hyperlinks, + preserved_relationships=preserved_drawing.drawing_relationships, ) archive.add( drawing_rels_part_path(sheet_id), @@ -3130,6 +3316,7 @@ fn write_with_io( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, }, ] let drawing_xml = write_drawing_xml( @@ -3324,6 +3511,7 @@ fn wb_drawing_image_with_offset( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } { image, rel_id: 1, hyperlink_rel_id: None } } @@ -3349,6 +3537,7 @@ fn wb_drawing_chart(positioning : PicturePositioning) -> DrawingChart { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } } @@ -3424,6 +3613,7 @@ fn wb_drawing_slicer( drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } { slicer, is_table_slicer } } @@ -3618,6 +3808,7 @@ test "write hardening wb: two-cell anchors enforce the worksheet boundary" { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } let result : Result[String, Error] = Ok( write_drawing_xml(Some(Worksheet::new("Sheet1")), [], [chart], [], []), @@ -3651,6 +3842,7 @@ test "write hardening wb: signed offsets and exact end markers stay readable" { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, } let boundary_xml = write_drawing_xml( Some(Worksheet::new("Sheet1")), @@ -4144,6 +4336,7 @@ test "write hardening wb: write fails for slicer source missing" { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::InvalidSheetOperation(_)), content="true") @@ -4172,6 +4365,7 @@ test "write hardening wb: write fails for slicer table field missing" { drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::InvalidSheetOperation(_)), content="true") @@ -4243,6 +4437,7 @@ test "write hardening wb: default zip writer failure maps to UnsupportedFeature" drawing_offset_y_emu: None, drawing_width_emu: None, drawing_height_emu: None, + drawing_order: None, }) let result : Result[Bytes, Error] = Ok(write(workbook)) catch { e => Err(e) } inspect(result is Err(XlsxError::UnsupportedFeature(_)), content="true") diff --git a/xlsx/write_rels_xml.mbt b/xlsx/write_rels_xml.mbt index fa33ab0e..cf529f8f 100644 --- a/xlsx/write_rels_xml.mbt +++ b/xlsx/write_rels_xml.mbt @@ -11,15 +11,17 @@ fn write_rels_open(sb : StringBuilder) -> Unit { /// Emit one `` element (two-space indented, newline-terminated). /// `external` adds `TargetMode="External"`. Centralizing the element format /// keeps every rels writer byte-identical. -fn write_relationship( +fn write_relationship_with_id( sb : StringBuilder, - rel_id : Int, - rel_type : String, - target : String, + rel_id : StringView, + rel_type : StringView, + target : StringView, external? : Bool = false, ) -> Unit { - sb.write_view(" Unit { + write_relationship_with_id(sb, "rId\{rel_id}", rel_type, target, external~) +} + ///| fn write_sheet_rels_xml( drawing_target : String?, @@ -115,6 +128,7 @@ fn write_drawing_rels_xml( image_targets : Array[(Int, String)], chart_targets : Array[(Int, String)], hyperlink_targets : Array[DrawingHyperlink], + preserved_relationships? : ArrayView[PreservedDrawingRelationship] = [], ) -> String { let sb = StringBuilder::new() write_rels_open(sb) @@ -135,6 +149,34 @@ fn write_drawing_rels_xml( external=entry.link_type == External, ) } + for relationship in preserved_relationships { + write_relationship_with_id( + sb, + relationship.id, + relationship.rel_type, + relationship.target, + external=relationship.external, + ) + } + sb.write_view("") + sb.to_string() +} + +///| +fn write_preserved_drawing_rels_xml( + relationships : ArrayView[PreservedDrawingRelationship], +) -> String { + let sb = StringBuilder::new() + write_rels_open(sb) + for relationship in relationships { + write_relationship_with_id( + sb, + relationship.id, + relationship.rel_type, + relationship.target, + external=relationship.external, + ) + } sb.write_view("") sb.to_string() } From 2a3fe674ab18bef5be0a20077377a9e3daee1b2e Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 17:43:11 +0800 Subject: [PATCH 148/150] fix(xlsx): preserve numeric and style boundaries --- xlsx/pkg.generated.mbti | 1 + xlsx/read_worksheet_xml.mbt | 4 +- xlsx/stream.mbt | 1 + xlsx/value_format.mbt | 97 +++++++++++++++++++++++++++-------- xlsx/value_format_test.mbt | 13 +++++ xlsx/workbook_compat.mbt | 1 + xlsx/worksheet.mbt | 37 +++++++++++-- xlsx/worksheet_cell_index.mbt | 30 +++++++++++ xlsx/worksheet_types.mbt | 3 ++ xlsx/write.mbt | 6 ++- 10 files changed, 166 insertions(+), 27 deletions(-) diff --git a/xlsx/pkg.generated.mbti b/xlsx/pkg.generated.mbti index 4e91fd7b..40feeea7 100644 --- a/xlsx/pkg.generated.mbti +++ b/xlsx/pkg.generated.mbti @@ -233,6 +233,7 @@ pub struct Cell { formula_ref : String? formula_shared_index : UInt? formula_value_present : Bool + style_explicit : Bool style_id : Int } diff --git a/xlsx/read_worksheet_xml.mbt b/xlsx/read_worksheet_xml.mbt index 31938e6e..df671f0e 100644 --- a/xlsx/read_worksheet_xml.mbt +++ b/xlsx/read_worksheet_xml.mbt @@ -74,7 +74,8 @@ fn parse_worksheet( Some(value) => unescape_xml_text(value) None => raise InvalidXml(msg="cell reference missing") } - let style_id = match attr_value(tag, "s") { + let style_attribute = attr_value(tag, "s") + let style_id = match style_attribute { Some(value) => @string.parse_int(value, base=10) catch { _ => raise InvalidXml(msg="style id invalid") @@ -279,6 +280,7 @@ fn parse_worksheet( formula_ref, formula_shared_index, formula_value_present: formula is Some(_) && raw_value_opt is Some(_), + style_explicit: style_attribute is Some(_), style_id, }) } diff --git a/xlsx/stream.mbt b/xlsx/stream.mbt index 60a52fc5..2a9c33c9 100644 --- a/xlsx/stream.mbt +++ b/xlsx/stream.mbt @@ -447,6 +447,7 @@ pub fn StreamWriter::set_row_cells( formula_ref: None, formula_shared_index: None, formula_value_present: cell.formula is Some(_) && cell.value != "", + style_explicit: style_id != 0, style_id, }) col = col + 1 diff --git a/xlsx/value_format.mbt b/xlsx/value_format.mbt index 734c9b03..52512a63 100644 --- a/xlsx/value_format.mbt +++ b/xlsx/value_format.mbt @@ -1013,15 +1013,16 @@ fn format_scientific_number(value : Double, info : NumberPatternInfo) -> String ///| fn format_fraction_number(value : Double, info : NumberPatternInfo) -> String { - // `Double::to_int64` saturates rather than reporting overflow. Fraction - // formats cannot faithfully render an integer part outside Int64, so retain - // the finite numeric value instead of manufacturing the saturation bound. - if value > 0x7fff_ffff_ffff_ffffL.to_double() { + // The integer part is non-negative here. Keep the full UInt64 range instead + // of passing through Double::to_int64, whose saturating conversion would + // manufacture Int64::MAX at the signed boundary. + if value >= uint64_exclusive_upper_bound_double { return value.to_string() } - let int_part = Double::to_int64(Double::floor(value)) - let frac = value - int_part.to_double() - let integer_text = format_int64_padded( + let integer_double = Double::floor(value) + let int_part = nonnegative_double_to_uint64(integer_double) + let frac = value - integer_double + let integer_text = format_uint64_padded( int_part, info.min_int_digits, info.use_group, @@ -1098,7 +1099,7 @@ fn float_to_frac_use_continued_fraction( return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } } let floored = Double::floor(r) - if floored > maximum_int64.to_double() { + if floored >= int64_exclusive_upper_bound_double { return if lastb > 0L { (lasta, lastb) } else { (0L, 1L) } } let a = Double::to_int64(floored) @@ -1215,8 +1216,19 @@ fn trim_optional_decimals(text : String, required_decimals : Int) -> String { match dot { None => text Some(pos) => { + let exponent_pos = match text.find("e") { + Some(value) => value + None => text.find("E").unwrap_or(text.length()) + } + // A fallback from fixed-point formatting can be scientific notation. + // Trim only the significand's fractional zeros and retain the exponent + // verbatim (especially its trailing zero). + if exponent_pos < pos { + return text + } let int_part = text[:pos].to_owned() - let frac_part = text[pos + 1:].to_owned() + let frac_part = text[pos + 1:exponent_pos].to_owned() + let exponent = text[exponent_pos:].to_owned() let mut end = frac_part.length() while end > required_decimals { if frac_part[end - 1] == '0' { @@ -1226,14 +1238,33 @@ fn trim_optional_decimals(text : String, required_decimals : Int) -> String { } } if end == 0 && required_decimals == 0 { - int_part + int_part + exponent } else { - int_part + "." + frac_part[:end].to_owned() + int_part + "." + frac_part[:end].to_owned() + exponent } } } } +///| +let int64_exclusive_upper_bound_double : Double = 9_223_372_036_854_775_808.0 + +///| +let uint64_exclusive_upper_bound_double : Double = 18_446_744_073_709_551_616.0 + +///| +fn nonnegative_double_to_uint64(value : Double) -> UInt64 { + // Some MoonBit backends currently saturate Double::to_uint64 at the signed + // boundary. Split off bit 63 explicitly so behavior is identical across + // backends for every finite integral Double in [0, 2^64). + if value >= int64_exclusive_upper_bound_double { + 0x8000_0000_0000_0000UL + + Double::to_int64(value - int64_exclusive_upper_bound_double).reinterpret_as_uint64() + } else { + Double::to_int64(value).reinterpret_as_uint64() + } +} + ///| fn format_fixed_number( value : Double, @@ -1251,22 +1282,30 @@ fn format_fixed_number( decimals } let scale = pow10_int(places) - let scale64 = Int64::from_int(scale) - if abs_value > 0x7fff_ffff_ffff_ffffL.to_double() / Double::from_int(scale) { + let integer_double = Double::floor(abs_value) + if integer_double >= uint64_exclusive_upper_bound_double { return abs_value.to_string() } - let scaled = Double::round(abs_value * Double::from_int(scale)) - let scaled_int = Double::to_int64(scaled) - let int_part = if scale == 0 { scaled_int } else { scaled_int / scale64 } - let frac_part = if scale == 0 { 0L } else { scaled_int % scale64 } + let scale64 = scale.to_uint64() + let mut int_part = nonnegative_double_to_uint64(integer_double) + let mut frac_part = nonnegative_double_to_uint64( + Double::round((abs_value - integer_double) * Double::from_int(scale)), + ) + if frac_part >= scale64 { + if int_part == 0xffff_ffff_ffff_ffffUL { + return abs_value.to_string() + } + int_part = int_part + 1UL + frac_part = 0UL + } let result = StringBuilder::new() if negative { result.write_char('-') } - result.write_view(format_int64_padded(int_part, min_int_digits, use_group)) + result.write_view(format_uint64_padded(int_part, min_int_digits, use_group)) if places > 0 { result.write_char('.') - result.write_view(pad_left_int64(frac_part, places)) + result.write_view(pad_left_uint64(frac_part, places)) } result.to_string() } @@ -1316,8 +1355,24 @@ fn pad_left_int64(value : Int64, width : Int) -> String { } ///| -fn format_int64_padded( - value : Int64, +fn pad_left_uint64(value : UInt64, width : Int) -> String { + let text = value.to_string() + if text.length() >= width { + return text + } + let sb = StringBuilder::new() + let mut i = text.length() + while i < width { + sb.write_char('0') + i = i + 1 + } + sb.write_view(text) + sb.to_string() +} + +///| +fn format_uint64_padded( + value : UInt64, min_digits : Int, use_group : Bool, ) -> String { diff --git a/xlsx/value_format_test.mbt b/xlsx/value_format_test.mbt index a52661d8..5dbf8a07 100644 --- a/xlsx/value_format_test.mbt +++ b/xlsx/value_format_test.mbt @@ -131,6 +131,19 @@ test "fraction, scientific, percent" { inspect(format_numeric(0.0, "# ?/?"), content="0 ") inspect(format_numeric(123.456, "0.00E+00"), content="1.23E+02") inspect(format_numeric(0.256, "0.00%"), content="25.60%") + inspect(format_numeric(1.2345e30, "0.##"), content="1.2345e+30") + inspect( + format_numeric(9_223_372_036_854_775_808.0, "0"), + content="9223372036854775808", + ) + inspect( + format_numeric(-9_223_372_036_854_775_808.0, "0.0"), + content="-9223372036854775808.0", + ) + inspect( + format_numeric(9_223_372_036_854_775_808.0, "# ?/?"), + content="9223372036854775808 ", + ) let large_fraction = format_numeric(1.0e20, "# ?/?") assert_false(large_fraction.contains("9223372036854775807")) } diff --git a/xlsx/workbook_compat.mbt b/xlsx/workbook_compat.mbt index 14e8124d..fb495d44 100644 --- a/xlsx/workbook_compat.mbt +++ b/xlsx/workbook_compat.mbt @@ -272,6 +272,7 @@ pub fn Workbook::update_linked_value(self : Workbook) -> Unit raise XlsxError { formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: false, + style_explicit: cell.style_explicit, style_id: cell.style_id, } } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 2b4d18db..912f8f03 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -134,6 +134,7 @@ pub fn Worksheet::set_cell( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -151,6 +152,7 @@ pub fn Worksheet::set_cell( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -211,6 +213,7 @@ pub fn Worksheet::set_cell_value( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -228,6 +231,7 @@ pub fn Worksheet::set_cell_value( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -315,6 +319,7 @@ pub fn Worksheet::set_cell_rich_text( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -332,6 +337,7 @@ pub fn Worksheet::set_cell_rich_text( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -574,6 +580,7 @@ fn normal_formula_cell_for_structural_edit( formula_ref: None, formula_shared_index: None, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, } } @@ -700,6 +707,7 @@ pub fn Worksheet::set_cell_formula( formula_ref: None, formula_shared_index: None, formula_value_present: value != "", + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -717,6 +725,7 @@ pub fn Worksheet::set_cell_formula( formula_ref: None, formula_shared_index: None, formula_value_present: value != "", + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -823,6 +832,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_ref: Some(formula_ref), formula_shared_index: None, formula_value_present: value != "", + style_explicit: sheet.cells[i].style_explicit, style_id, } } @@ -840,6 +850,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_ref: Some(formula_ref), formula_shared_index: None, formula_value_present: value != "", + style_explicit: false, style_id: 0, }) sheet.record_cell_index_if_valid(row, col, index) @@ -911,6 +922,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_ref, formula_shared_index: Some(shared_index), formula_value_present: value != "", + style_explicit: sheet.cells[i].style_explicit, style_id, } } @@ -928,6 +940,7 @@ pub fn Worksheet::set_cell_formula_opts( formula_ref, formula_shared_index: Some(shared_index), formula_value_present: value != "", + style_explicit: false, style_id: 0, }) sheet.record_cell_index_if_valid(row, col, index) @@ -1184,6 +1197,7 @@ pub fn Worksheet::set_cell_style( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: true, style_id, } } @@ -1201,6 +1215,7 @@ pub fn Worksheet::set_cell_style( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: true, style_id, }) self.record_cell_index_if_valid(row, col, index) @@ -1247,6 +1262,7 @@ pub fn Worksheet::set_cell_style_rc( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: true, style_id, } } @@ -1264,6 +1280,7 @@ pub fn Worksheet::set_cell_style_rc( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: true, style_id, }) self.record_cell_index_if_valid(row, col, index) @@ -1288,8 +1305,9 @@ pub fn Worksheet::get_cell_style_rc( ///| /// Returns the style that formats a resolved coordinate without performing a -/// workbook sheet-name lookup. Precedence matches the write path: a non-zero -/// cell style wins, then the row style, then the column style, then style 0. +/// workbook sheet-name lookup. Precedence matches the write path: an explicit +/// cell style (including style 0) wins, then the row style, then the column +/// style, then style 0. /// The row and column are 1-based and must fit the XLSX grid. pub fn Worksheet::effective_style_id_rc( self : Worksheet, @@ -1297,8 +1315,8 @@ pub fn Worksheet::effective_style_id_rc( col : Int, ) -> Int raise XlsxError { ignore(cell_ref_from(row, col)) - match self.get_cell_style_rc(row, col) { - Some(style_id) if style_id != 0 => style_id + match self.cell_index_of(row, col) { + Some(i) if self.cells[i].style_explicit => self.cells[i].style_id _ => match self.get_row_style(row) { Some(style_id) if style_id != 0 => style_id @@ -2903,6 +2921,7 @@ pub fn Worksheet::set_cell_rc( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -2920,6 +2939,7 @@ pub fn Worksheet::set_cell_rc( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -2954,6 +2974,7 @@ pub fn Worksheet::set_cell_value_rc( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -2971,6 +2992,7 @@ pub fn Worksheet::set_cell_value_rc( formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -3004,6 +3026,7 @@ pub fn Worksheet::set_cell_formula_rc( formula_ref: None, formula_shared_index: None, formula_value_present: value != "", + style_explicit: self.cells[i].style_explicit, style_id, } } @@ -3021,6 +3044,7 @@ pub fn Worksheet::set_cell_formula_rc( formula_ref: None, formula_shared_index: None, formula_value_present: value != "", + style_explicit: false, style_id: 0, }) self.record_cell_index_if_valid(row, col, index) @@ -3687,6 +3711,7 @@ fn Worksheet::insert_rows( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, }) } else { @@ -3934,6 +3959,7 @@ fn Worksheet::remove_rows( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, }) } @@ -4607,6 +4633,7 @@ fn Worksheet::duplicate_row_to( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, }) } @@ -4729,6 +4756,7 @@ fn Worksheet::insert_cols( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, }) } else { @@ -4972,6 +5000,7 @@ fn Worksheet::remove_cols( formula_ref: cell.formula_ref, formula_shared_index: cell.formula_shared_index, formula_value_present: cell.formula_value_present, + style_explicit: cell.style_explicit, style_id: cell.style_id, }) } diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index 9bd6127a..219cfcb8 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -98,6 +98,7 @@ fn raw_cell(reference : String, row : Int, col : Int, value : String) -> Cell { formula_ref: None, formula_shared_index: None, formula_value_present: false, + style_explicit: false, style_id: 0, } } @@ -141,6 +142,35 @@ test "resolved worksheet reads avoid repeated names and preserve style precedenc assert_eq(sheet.effective_style_id_rc(3, 2), 0) } +///| +test "explicit cell style zero overrides inherited row style after roundtrip" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("S") + let row_style = workbook.add_style(Style::builtin_number_format(2)) + workbook.set_row_style("S", 1, row_style) + sheet.set_cell_value_rc(1, 1, Numeric(1.0)) + sheet.set_cell_style_rc(1, 1, 0) + assert_eq(sheet.effective_style_id_rc(1, 1), 0) + assert_eq(workbook.get_cell_value("S", "A1"), Some("1")) + let parsed = read(write(workbook)) + guard parsed.sheet("S") is Some(parsed_sheet) else { + fail("expected parsed worksheet") + } + guard parsed_sheet.cell_index_of(1, 1) is Some(cell_index) else { + fail("expected parsed cell") + } + assert_true(parsed_sheet.cells[cell_index].style_explicit) + assert_eq(parsed_sheet.cells[cell_index].style_id, 0) + assert_eq(parsed_sheet.effective_style_id_rc(1, 1), 0) + assert_eq(parsed.get_cell_value("S", "A1"), Some("1")) + let reparsed = read(write(parsed)) + guard reparsed.sheet("S") is Some(reparsed_sheet) else { + fail("expected reparsed worksheet") + } + assert_eq(reparsed_sheet.effective_style_id_rc(1, 1), 0) + assert_eq(reparsed.get_cell_value("S", "A1"), Some("1")) +} + ///| fn expect_unowned_worksheet_error( action : () -> Unit raise XlsxError, diff --git a/xlsx/worksheet_types.mbt b/xlsx/worksheet_types.mbt index ed7641b3..6572dcd3 100644 --- a/xlsx/worksheet_types.mbt +++ b/xlsx/worksheet_types.mbt @@ -13,6 +13,9 @@ pub struct Cell { /// Whether this formula cell contained an OOXML `` cached-result node. /// Empty cached strings are semantically distinct from a missing cache. formula_value_present : Bool + /// Whether the OOXML cell explicitly specifies its style, including `s="0"`. + /// An omitted cell style may inherit from its row or column. + style_explicit : Bool style_id : Int } diff --git a/xlsx/write.mbt b/xlsx/write.mbt index 95f5e1ad..4f600a1b 100644 --- a/xlsx/write.mbt +++ b/xlsx/write.mbt @@ -2021,7 +2021,7 @@ fn write_worksheet_xml( sb.write_view(" Date: Sun, 19 Jul 2026 18:41:55 +0800 Subject: [PATCH 149/150] fix(xlsx): preserve row style boundaries --- xlsx/cell_time.mbt | 13 +---- xlsx/read.mbt | 22 +++++++-- xlsx/worksheet.mbt | 8 ++-- xlsx/worksheet_cell_index.mbt | 89 +++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 20 deletions(-) diff --git a/xlsx/cell_time.mbt b/xlsx/cell_time.mbt index 7e27edf8..1af66a16 100644 --- a/xlsx/cell_time.mbt +++ b/xlsx/cell_time.mbt @@ -143,18 +143,7 @@ fn Workbook::resolve_cell_base_style( reference : String, ) -> Int raise XlsxError { let (row, col) = cell_ref_to_rc(reference) - match self.get_cell_style(sheet_name, reference) { - Some(idx) if idx != 0 => idx - _ => - match self.get_row_style(sheet_name, row) { - Some(idx) if idx != 0 => idx - _ => - match self.get_col_style(sheet_name, col) { - Some(idx) if idx != 0 => idx - _ => 0 - } - } - } + self.require_sheet(sheet_name).effective_style_id_rc(row, col) } ///| diff --git a/xlsx/read.mbt b/xlsx/read.mbt index fc89172f..0d0aa9aa 100644 --- a/xlsx/read.mbt +++ b/xlsx/read.mbt @@ -2960,7 +2960,8 @@ fn parse_row_dimensions( } None => 0 } - let style_id = match attr_value(tag, "s") { + let style_attribute = attr_value(tag, "s") + let style_id = match style_attribute { Some(value) => @string.parse_int(value, base=10) catch { _ => raise InvalidXml(msg="row style id invalid") @@ -2970,7 +2971,14 @@ fn parse_row_dimensions( if style_id < 0 || style_id >= style_count { raise InvalidStyleId(index=style_id) } - let style = if style_id > 0 { Some(style_id) } else { None } + // `s` is only an active row format when `customFormat` is true. Keep + // `Some(0)` for that case: an explicit default row format is a real + // inheritance boundary and must block a nonzero column style. + let custom_format = match attr_value(tag, "customFormat") { + Some(value) => parse_bool_attr(value) + None => false + } + let style = if custom_format { Some(style_id) } else { None } if height is None && !hidden && outline_level == 0 && style is None { continue } @@ -3042,7 +3050,8 @@ fn parse_col_dimensions( } None => 0 } - let style_id = match attr_value(tag, "style") { + let style_attribute = attr_value(tag, "style") + let style_id = match style_attribute { Some(value) => @string.parse_int(value, base=10) catch { _ => raise InvalidXml(msg="col style id invalid") @@ -3052,7 +3061,12 @@ fn parse_col_dimensions( if style_id < 0 || style_id >= style_count { raise InvalidStyleId(index=style_id) } - let style = if style_id > 0 { Some(style_id) } else { None } + // Unlike rows, columns have no `customFormat` gate. Attribute presence is + // therefore the semantic distinction, including an explicit style 0. + let style = match style_attribute { + Some(_) => Some(style_id) + None => None + } if width is None && !hidden && outline_level == 0 && style is None { continue } diff --git a/xlsx/worksheet.mbt b/xlsx/worksheet.mbt index 912f8f03..639d310a 100644 --- a/xlsx/worksheet.mbt +++ b/xlsx/worksheet.mbt @@ -1319,11 +1319,11 @@ pub fn Worksheet::effective_style_id_rc( Some(i) if self.cells[i].style_explicit => self.cells[i].style_id _ => match self.get_row_style(row) { - Some(style_id) if style_id != 0 => style_id - _ => + Some(style_id) => style_id + None => match self.get_col_style(col) { - Some(style_id) if style_id != 0 => style_id - _ => 0 + Some(style_id) => style_id + None => 0 } } } diff --git a/xlsx/worksheet_cell_index.mbt b/xlsx/worksheet_cell_index.mbt index 219cfcb8..7d9188c5 100644 --- a/xlsx/worksheet_cell_index.mbt +++ b/xlsx/worksheet_cell_index.mbt @@ -171,6 +171,95 @@ test "explicit cell style zero overrides inherited row style after roundtrip" { assert_eq(reparsed.get_cell_value("S", "A1"), Some("1")) } +///| +test "row customFormat boundaries control style inheritance after roundtrip" { + let workbook = Workbook::new() + let sheet = workbook.add_sheet("S") + let column_style = workbook.add_style( + Style::font(Font::with_values(bold=true)), + ) + let ignored_row_style = workbook.add_style( + Style::font(Font::with_values(italic=true)), + ) + workbook.set_col_style("S", 1, column_style) + workbook.set_col_style("S", 2, column_style) + for row in 1..<=3 { + sheet.set_cell_value_rc(row, 1, Numeric(row.to_double())) + } + let archive = @zip.read(write(workbook)) + let path = "xl/worksheets/sheet1.xml" + guard archive.get(path) is Some(sheet_bytes) else { + fail("expected worksheet part") + } + let source = @encoding/utf8.decode(sheet_bytes) + assert_true(source.contains("")) + assert_true(source.contains("")) + assert_true(source.contains("")) + let patched = source + .replace( + old="", + new="", + ) + .replace( + old="", + new="", + ) + .replace(old="", new="") + assert_true(archive.replace(path, @encoding/utf8.encode(patched))) + + let parsed = read(@zip.write(archive)) + guard parsed.sheet("S") is Some(parsed_sheet) else { + fail("expected parsed worksheet") + } + assert_eq(parsed_sheet.get_row_style(1), Some(0)) + assert_eq(parsed_sheet.get_row_style(2), None) + assert_eq(parsed_sheet.get_row_style(3), Some(0)) + assert_eq(parsed_sheet.effective_style_id_rc(1, 1), 0) + assert_eq(parsed_sheet.effective_style_id_rc(2, 1), column_style) + assert_eq(parsed_sheet.effective_style_id_rc(3, 1), 0) + + // Time writes must consult the same effective-style resolver. The explicit + // row-zero boundary keeps B1 from inheriting the bold column style. + parsed.set_cell_time("S", "B1", @time.date_time(2024, 7, 3)) + guard parsed.get_cell_style("S", "B1") is Some(time_style_id) else { + fail("expected time-cell style") + } + assert_true(parsed.get_style(time_style_id).font is None) + + let rewritten = @zip.read(write(parsed)) + guard rewritten.get(path) is Some(rewritten_bytes) else { + fail("expected rewritten worksheet part") + } + let rewritten_xml = @encoding/utf8.decode(rewritten_bytes) + assert_true( + rewritten_xml.contains(""), + ) + assert_false(rewritten_xml.contains(""), + ) + let reparsed = read(@zip.write(rewritten)) + guard reparsed.sheet("S") is Some(reparsed_sheet) else { + fail("expected reparsed worksheet") + } + assert_eq(reparsed_sheet.get_row_style(1), Some(0)) + assert_eq(reparsed_sheet.get_row_style(2), None) + assert_eq(reparsed_sheet.get_row_style(3), Some(0)) + assert_eq(reparsed_sheet.effective_style_id_rc(1, 1), 0) + assert_eq(reparsed_sheet.effective_style_id_rc(2, 1), column_style) + assert_eq(reparsed_sheet.effective_style_id_rc(3, 1), 0) +} + +///| +test "explicit column style zero remains distinct from an omitted style" { + let dimensions = parse_col_dimensions( + "", + 1, + ) + assert_eq(dimensions.get(1).map(dim => dim.style_id), Some(Some(0))) + assert_true(dimensions.get(2) is None) +} + ///| fn expect_unowned_worksheet_error( action : () -> Unit raise XlsxError, From f02b0937741182b986b899a393fd360a4954bc49 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sun, 19 Jul 2026 18:41:55 +0800 Subject: [PATCH 150/150] fix(xlsx): preserve custom geometry anchors --- xlsx/drawing_preservation_wbtest.mbt | 10 +++++++++- xlsx/read_drawing_xml.mbt | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/xlsx/drawing_preservation_wbtest.mbt b/xlsx/drawing_preservation_wbtest.mbt index 21b1f3bd..cc6048f7 100644 --- a/xlsx/drawing_preservation_wbtest.mbt +++ b/xlsx/drawing_preservation_wbtest.mbt @@ -104,6 +104,8 @@ fn drawing_preservation_test_fixture( #|3000 let opaque_group = #|4000 + let opaque_custom_shape = + #|5000 let patched_drawing = "\n" + "\n" + shape + @@ -112,6 +114,7 @@ fn drawing_preservation_test_fixture( image + "\n" + opaque_group + + opaque_custom_shape + chart + "\n" assert_true( @@ -179,11 +182,16 @@ fn assert_drawing_preservation_test_output( ) let image_pos = drawing_preservation_test_position(drawing, "") let group_pos = drawing_preservation_test_position(drawing, "Opaque Group") + let custom_shape_pos = drawing_preservation_test_position( + drawing, "Opaque Custom Geometry", + ) let chart_pos = drawing_preservation_test_position(drawing, "")) assert_eq(drawing_preservation_test_count(drawing, " Bool { // descendant of its group subtree), so lexical filtering is safe here. xml.contains("