Accept dangling pointers and cut parse memory

README.md

@@ -31,7 +31,7 @@ with open("out.ged", "wb") as f:
 
 Each record is a `GedcomStructure` with a `tag`, an optional `xref` and `pointer`, the raw `text` payload, and `children`. Its `value` property casts the payload to the data type the specification gives that structure type.
 
-`loads` and `dumps` are the string equivalents. Non-conforming input raises `GedcomParseError`, a `ValueError` carrying `line_number`.
+`loads` and `dumps` are the string equivalents. Non-conforming input raises `GedcomParseError`, a `ValueError` carrying `line_number`. A pointer to a record that does not exist is kept rather than rejected; `gedcom7.validate(records)` lists each one as a `dangling-pointer` error.
 
 ## Development
 

gedcom7/parser.py

@@ -3,13 +3,15 @@
 from __future__ import annotations
 
 import re
+import sys
 from typing import TYPE_CHECKING
 
 from . import const, grammar
 from .exceptions import GedcomParseError
 from .types import GedcomStructure
 
 if TYPE_CHECKING:
+    from collections.abc import Iterator
     from typing import BinaryIO
 
 # EOL = %x0D [%x0A] / %x0A -- CR-LF, CR, or LF
@@ -32,6 +34,21 @@ def _unescape(linestr: str) -> str:
     return linestr[1:] if linestr.startswith("@@") else linestr
 
 
+def _lines(string: str) -> Iterator[str]:
+    """Yield the lines of a data stream, without their EOLs, one at a time.
+
+    A terminating EOL does not start a further, empty line. A data stream whose
+    last line lacks its EOL is tolerated: the line is complete and unambiguous,
+    and dropping it would silently lose data.
+    """
+    start = 0
+    for eol in _EOL.finditer(string):
+        yield string[start : eol.start()]
+        start = eol.end()
+    if start < len(string):
+        yield string[start:]
+
+
 def load(fp: BinaryIO) -> list[GedcomStructure]:
     """Load a GEDCOM 7 dataset from a binary file object.
 
@@ -56,6 +73,10 @@ def loads(string: str) -> list[GedcomStructure]:
 
     Raises :class:`~gedcom7.exceptions.GedcomParseError` if the data stream does
     not conform to the specification. Non-conforming lines are never skipped.
+
+    A pointer that matches no cross-reference identifier is kept as written, so
+    one broken pointer does not cost the rest of the dataset;
+    :func:`~gedcom7.validate` reports each one as a ``dangling-pointer`` error.
     """
     string = string.removeprefix(_BOM)
 
@@ -66,32 +87,26 @@ def loads(string: str) -> list[GedcomStructure]:
             line_number=string.count("\n", 0, banned.start()) + 1,
         )
 
-    lines = _EOL.split(string)
-    # A terminating EOL leaves a final empty element that is not itself a line.
-    # A data stream whose last line lacks its EOL is tolerated: the line is
-    # complete and unambiguous, and dropping it would silently lose data.
-    if lines and lines[-1] == "":
-        lines.pop()
-
     records: list[GedcomStructure] = []
     # stack[i] is the structure encoded by the nearest preceding line of level i
     stack: list[GedcomStructure] = []
     # extension tag -> URIs declared for it by the header schema
     schema: dict[str, list[str]] = {}
     xrefs: set[str] = set()
-    pointers: list[tuple[str, int]] = []
     # the structure a CONT on the very next line would continue
     continuable: GedcomStructure | None = None
 
-    for number, text in enumerate(lines, start=1):
+    for number, text in enumerate(_lines(string), start=1):
         match = _LINE.fullmatch(text + "\n")
         if match is None:
             raise GedcomParseError(
                 f"malformed line: {text!r}", line_number=number, line=text
             )
 
         level = int(match.group("level"))
-        tag = match.group("tag")
+        # A dataset repeats a few dozen tags across every line, so each line's
+        # tag shares one string object rather than holding its own copy.
+        tag = sys.intern(match.group("tag"))
         xref = match.group("xref")
         pointer = match.group("pointer")
         linestr = match.group("linestr")
@@ -188,19 +203,6 @@ def loads(string: str) -> list[GedcomStructure]:
         stack.append(structure)
         continuable = structure
 
-        if pointer is not None and pointer != const.VOIDPTR:
-            pointers.append((pointer, number))
-
-    # Pointers may be forward references, so they are resolved once the whole
-    # data stream has been read.
-    for pointer, number in pointers:
-        if pointer not in xrefs:
-            raise GedcomParseError(
-                f"pointer {pointer} matches no cross-reference identifier in the "
-                "data stream",
-                line_number=number,
-            )
-
     if not records:
         raise GedcomParseError("a dataset must contain a header and a trailer")
     if records[0].tag != const.HEAD:

gedcom7/types.py

@@ -8,7 +8,7 @@
 from . import cast, const
 
 
-@dataclass
+@dataclass(slots=True)
 class GedcomStructure:
     """Gedcom structure class."""
 

test/test_conformance.py

@@ -1,7 +1,8 @@
 """Conformance tests against the FamilySearch GEDCOM 7 specification.
 
 Each test cites the requirement it covers. Cases that the specification
-prohibits must raise, not be silently skipped or reinterpreted.
+prohibits must raise, not be silently skipped or reinterpreted. The exception
+is a pointer to a missing record, which parses and is reported by ``validate``.
 """
 
 import pytest
@@ -185,10 +186,24 @@ def test_pointer_and_voidptr() -> None:
     assert indi.children[1].pointer == "@VOID@"
 
 
-def test_unresolved_pointer_rejected() -> None:
-    """A pointer must match an xref of a structure within the document."""
-    with pytest.raises(GedcomParseError, match="matches no cross-reference"):
-        parse("0 @I1@ INDI\n1 ALIA @I9@\n")
+def test_unresolved_pointers_parse_and_validate_reports_each() -> None:
+    """A pointer must match an xref of a structure within the document.
+
+    The parser keeps an unresolved pointer as written, and ``validate`` reports
+    every one, not only the first.
+    """
+    records = parse(
+        "0 @I1@ INDI\n1 ALIA @I9@\n1 FAMS @F9@\n1 FAMC @VOID@\n"
+        "0 @F1@ FAM\n1 HUSB @I1@\n1 WIFE @I8@\n"
+    )
+    assert records[1].children[0].pointer == "@I9@"
+    assert records[1].children[1].pointer == "@F9@"
+    errors = [e for e in gedcom7.validate(records) if e.category == "dangling-pointer"]
+    assert [e.path for e in errors] == [
+        "@I1@ INDI > ALIA",
+        "@I1@ INDI > FAMS",
+        "@F1@ FAM > WIFE",
+    ]
 
 
 def test_forward_pointer_allowed() -> None:

test/test_parser.py

@@ -1,6 +1,9 @@
+import itertools
 import pathlib
+import re
 
 import gedcom7
+from gedcom7.parser import _lines
 
 GEDCOM_MIN = """0 HEAD
 1 GEDC
@@ -103,3 +106,15 @@ def test_empty_line_value() -> None:
     assert file_struct.tag == "FILE"
     assert file_struct.text == ""
     assert file_struct.children[0].tag == "FORM"
+
+
+def test_lines_matches_splitting_on_every_eol() -> None:
+    """Every string over a character, CR and LF, up to seven long, splits into
+    the lines that splitting on EOL and dropping a final empty piece gives."""
+    for n in range(8):
+        for chars in itertools.product("a\r\n", repeat=n):
+            string = "".join(chars)
+            expected = re.split(r"\r\n|\r|\n", string)
+            if expected[-1] == "":
+                expected.pop()
+            assert list(_lines(string)) == expected, repr(string)