fix: sync parser common and utility compatibility

tests/unit/test_caption_extraction.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+from uniparser_tools.common.constant import LayoutType
+from uniparser_tools.common.dataclass import BBox, TextualResult
+from uniparser_tools.tools.caption_extraction.main import clean_flatten_pages
+
+
+def _textual_result(
+    *,
+    types: list[LayoutType],
+    bboxes: list[BBox] | None = None,
+) -> TextualResult:
+    item = TextualResult(
+        token="token",
+        page=0,
+        block=1,
+        bbox=BBox(0, 0, 1, 1),
+        conf=1.0,
+        page_size=(100, 100),
+        type=LayoutType.Reference,
+        bboxes=([BBox(0, 0, 0.2, 0.1), BBox(0.2, 0, 0.8, 0.1), BBox(0.8, 0, 1, 0.1)] if bboxes is None else bboxes),
+        contents=["prefix ", "https://doi.org/10.1/example ", "suffix"],
+        types=types,
+    )
+    return item
+
+
+def test_clean_flatten_pages_filters_types_with_doi_content() -> None:
+    item = _textual_result(types=[LayoutType.Text, LayoutType.Text, LayoutType.Molecule])
+
+    cleaned = clean_flatten_pages([[item]])[0][0]
+
+    assert cleaned.contents == ["prefix ", "suffix"]
+    assert cleaned.bboxes == [BBox(0, 0, 0.2, 0.1), BBox(0.8, 0, 1, 0.1)]
+    assert cleaned.types == [LayoutType.Text, LayoutType.Molecule]
+
+
+def test_clean_flatten_pages_preserves_text_when_bboxes_are_empty() -> None:
+    item = _textual_result(
+        types=[LayoutType.Text, LayoutType.Text, LayoutType.Molecule],
+        bboxes=[],
+    )
+
+    cleaned = clean_flatten_pages([[item]])[0][0]
+
+    assert cleaned.text == "prefix suffix"
+    assert cleaned.contents == ["prefix ", "suffix"]
+    assert cleaned.bboxes == []
+    assert cleaned.types == [LayoutType.Text, LayoutType.Molecule]
+
+
+def test_clean_flatten_pages_supports_legacy_text_only_result() -> None:
+    text = "Smith et al. https://doi.org/10.1/example Nature 2020."
+    item = TextualResult(
+        token="token",
+        page=0,
+        block=1,
+        bbox=BBox(0, 0, 1, 1),
+        conf=1.0,
+        page_size=(100, 100),
+        type=LayoutType.Reference,
+        text=text,
+    )
+
+    cleaned = clean_flatten_pages([[item]])[0][0]
+
+    assert cleaned.text == text
+    assert cleaned.contents == [text]
+    assert cleaned.bboxes == []
+    assert cleaned.types == [LayoutType.Text]
+
+
+def test_clean_flatten_pages_supports_structured_contents_without_bboxes() -> None:
+    contents = [
+        "在软件管理的领域里存在着被称作“依赖地狱”的死亡之谷。  ",
+        r"\frac{dy}{dx}=\frac{-b\pm\sqrt{b^{2}-4ac}}{2a}",
+        "   开放源码软件所广泛使用的  ",
+        r"c^{2}=a^{2}+b^{2}-2ab\cos C",
+        "  公共 <b>not html</b><img src=x onerror=test> literal @@MATH_TOKEN_0@@ and  ",
+        r"x^{2}",
+        " ; literal @@CODE_TOKEN_0@@ and `code`",
+    ]
+    types = [
+        LayoutType.Text,
+        LayoutType.EquationInline,
+        LayoutType.Text,
+        LayoutType.EquationInline,
+        LayoutType.Text,
+        LayoutType.EquationInline,
+        LayoutType.Text,
+    ]
+    rich_text = (
+        "在软件管理的领域里存在着被称作“依赖地狱”的死亡之谷。  "
+        r"\(\frac{dy}{dx}=\frac{-b\pm\sqrt{b^{2}-4ac}}{2a}\)"
+        "   开放源码软件所广泛使用的  "
+        r"\(c^{2}=a^{2}+b^{2}-2ab\cos C\)"
+        "  公共 <b>not html</b><img src=x onerror=test> literal @@MATH_TOKEN_0@@ and  "
+        r"\(x^{2}\)"
+        " ; literal @@CODE_TOKEN_0@@ and `code`"
+    )
+    item = TextualResult(
+        token="token",
+        page=0,
+        block=1,
+        bbox=BBox(0, 0, 1, 1),
+        conf=1.0,
+        page_size=(100, 100),
+        type=LayoutType.Paragraph,
+        bboxes=[],
+        contents=contents,
+        types=types,
+        text=rich_text,
+    )
+
+    cleaned = clean_flatten_pages([[item]])[0][0]
+
+    assert cleaned.contents == contents
+    assert cleaned.bboxes == []
+    assert cleaned.types == types
+    assert r"\(\frac{dy}{dx}=\frac{-b\pm\sqrt{b^{2}-4ac}}{2a}\)" in cleaned.text
+    assert r"\(c^{2}=a^{2}+b^{2}-2ab\cos C\)" in cleaned.text
+    assert "<b>not html</b><img src=x onerror=test>" in cleaned.text
+    assert "@@MATH_TOKEN_0@@" in cleaned.text
+    assert "@@CODE_TOKEN_0@@ and `code`" in cleaned.text

tests/unit/test_common_constant.py

@@ -0,0 +1,14 @@
+from uniparser_tools.common.constant import PPOCR_LANG, SUPPORTED_LANG, Language, to_ocr_lang
+
+
+def test_ppocr_language_mapping_matches_parser_models() -> None:
+    assert PPOCR_LANG[Language.German] == "de"
+    assert PPOCR_LANG[Language.Serbian] == "rs_cyrillic"
+    assert PPOCR_LANG[Language.Serbian_Latin] == "rs_latin"
+    assert Language.Persian in SUPPORTED_LANG
+
+
+def test_to_ocr_lang_handles_aliases_and_unknown_values() -> None:
+    assert to_ocr_lang(Language.Cantonese) == Language.Chinese_Traditional
+    assert to_ocr_lang(Language.Western_Mari) == Language.Eastern_Mari
+    assert to_ocr_lang("not-a-language") == Language.Unknown

tests/unit/test_dataclass_results.py

@@ -84,10 +84,36 @@ def test_title_markdown_gets_heading(self) -> None:
         r = TextualResult(**kw, bboxes=[], contents=[], text="Chapter 1")
         assert r.markdown == "## Chapter 1"
 
-    def test_title_html_uses_h2(self) -> None:
+    def test_title_html_matches_server_tag(self) -> None:
         kw = {**ITEM_KWARGS, "type": LayoutType.Title}
         r = TextualResult(**kw, bboxes=[], contents=[], text="Chapter 1")
-        assert r.html == '<h2 class="title">Chapter 1</h2>'
+        assert r.html == "<h2>Chapter 1</h2>"
+
+    @pytest.mark.parametrize(
+        ("item_type", "expected"),
+        [
+            (LayoutType.Caption, "<caption>content</caption>"),
+            (LayoutType.PageNote, "<aside><em>content</em></aside>"),
+            (LayoutType.Algorithm, "<pre><code>content</code></pre>"),
+            (LayoutType.DocumentTitle, "<h1>content</h1>"),
+        ],
+    )
+    def test_html_tags_match_server_output(self, item_type: LayoutType, expected: str) -> None:
+        kw = {**ITEM_KWARGS, "type": item_type}
+        r = TextualResult(**kw, bboxes=[], contents=["content"], types=[LayoutType.Text])
+
+        assert r.html == expected
+
+    def test_algorithm_markdown_preserves_raw_contents(self) -> None:
+        kw = {**ITEM_KWARGS, "type": LayoutType.Algorithm}
+        r = TextualResult(
+            **kw,
+            bboxes=[],
+            contents=["print(", "x^2", ")"],
+            types=[LayoutType.Text, LayoutType.EquationInline, LayoutType.Text],
+        )
+
+        assert r.markdown == "```\nprint(x^2)\n```"
 
     def test_document_title_latex_uses_title_macro(self) -> None:
         kw = {**ITEM_KWARGS, "type": LayoutType.DocumentTitle}
@@ -154,6 +180,20 @@ def test_markdown_and_latex_are_empty_for_unusable_tables(self, structure: str)
         assert r.markdown == ""
         assert r.latex == ""
 
+    def test_format_failure_does_not_fall_back_to_plain_dataframe(self, monkeypatch) -> None:
+        payload = make_tabular_payload(structure="<table></table>", placeholders=[], contents=[])
+        r = TabularResult(**payload)
+
+        def format_sensitive_df(item_format: FormatFlag) -> pd.DataFrame:
+            if item_format == FormatFlag.Plain:
+                return pd.DataFrame({"plain": ["content"]})
+            raise ValueError("malformed formatted table")
+
+        monkeypatch.setattr(r, "_df_for_format", format_sensitive_df)
+
+        assert r.markdown == ""
+        assert r.latex == ""
+
     def test_latex_returns_tabular_env(self, simple_tabular: TabularResult) -> None:
         latex = simple_tabular.latex
         assert "\\begin{tabular}" in latex

tests/unit/test_format_utils.py

@@ -0,0 +1,12 @@
+from uniparser_tools.common.constant import LayoutType
+from uniparser_tools.utils.format_utils import parse_inline_text
+
+
+def test_parse_inline_text_keeps_code_markup_as_text() -> None:
+    text = "Use <code>CCO</code> and `NCC` as code."
+
+    assert parse_inline_text(text) == ([text], [LayoutType.Text])
+
+
+def test_parse_inline_text_still_recognizes_explicit_molecule_markup() -> None:
+    assert parse_inline_text("***CCO***") == (["CCO"], [LayoutType.Molecule])

tests/unit/test_order_utils_sync.py

@@ -0,0 +1,63 @@
+from pathlib import Path
+
+from PIL import Image
+
+from uniparser_tools.common.constant import LayoutType
+from uniparser_tools.common.dataclass import BBox, GroupedResult, TextualResult
+from uniparser_tools.order.xy_cut_exp import sticky_items
+from uniparser_tools.utils.image import dump_image_base64_str, load_image_base64_str
+from uniparser_tools.utils.processor import tree_repr
+from uniparser_tools.utils.visualize_together import plotly_pdf_results
+
+
+def _text(block: int, text: str, bbox: BBox | None = None) -> TextualResult:
+    return TextualResult(
+        token="token",
+        page=0,
+        block=block,
+        bbox=bbox or BBox(0.1, 0.1, 0.4, 0.2),
+        conf=1.0,
+        page_size=(100, 100),
+        type=LayoutType.Paragraph,
+        contents=[text],
+        types=[LayoutType.Text],
+    )
+
+
+def test_sticky_items_keeps_valid_bbox_order() -> None:
+    items = [
+        _text(1, "first", BBox(0.10, 0.10, 0.21, 0.20)),
+        _text(2, "second", BBox(0.12, 0.11, 0.23, 0.21)),
+    ]
+
+    snapped = sticky_items(items, offset=5)
+
+    assert all(item.bbox.x1 <= item.bbox.x2 and item.bbox.y1 <= item.bbox.y2 for item in snapped)
+
+
+def test_tree_repr_keeps_legacy_verbose_option() -> None:
+    grouped = GroupedResult.clone(_text(1, "first"), items=[_text(2, "child")], method="synced")
+
+    representation = tree_repr(grouped, verbose=True)
+    assert "synced" in representation
+    assert "[2]" in representation
+    assert "BBox" in representation
+
+
+def test_image_helpers_support_alpha() -> None:
+    transparent = Image.new("RGBA", (4, 2), (255, 0, 0, 0))
+    decoded = load_image_base64_str(dump_image_base64_str(transparent, quality=100)).convert("RGB")
+
+    assert all(channel >= 245 for channel in decoded.getpixel((0, 0)))
+
+
+def test_visualization_adds_page_thumbnail_without_server_assets(tmp_path: Path) -> None:
+    image_path = tmp_path / "page.png"
+    Image.new("RGB", (40, 60), "white").save(image_path)
+
+    rendered = plotly_pdf_results([[]], str(image_path))
+
+    assert "page-thumb-img" in rendered
+    assert "data:image/jpeg;base64," in rendered
+    assert "cdn.jsdelivr.net/npm/mathjax" in rendered
+    assert "/static/images/parser-logo.png" not in rendered