From 2fc522207c807dd0cf1b67809a14dbd29daf6372 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 16:14:43 +0000 Subject: [PATCH 1/3] Parity suite against another Python Jinja2: JINJA2CPP_TEST_ORACLE (0186 phase 1) CMake option JINJA2CPP_TEST_ORACLE=bundled|python|venv (cmake/parity_oracle.cmake) adds a CTest fixture that runs generate.py --out /parity-oracle --any-version under another Python Jinja2 at test time, and jinja2cpp_parity_oracle, which runs the narrow suite with JINJA2CPP_TEST_ORACLE_DIR naming that directory. Where the two oracles agree a case is judged by the allow-list as before; where they differ it is reported as tracks-bundled, tracks-oracle, neither or not-run in report.md, and fails only under JINJA2CPP_TEST_ORACLE_STRICT=1. Bundled builds are unchanged. Files 0372 for the corpus gap on code points changed after Unicode 15.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AWNL7oJQzYFjjoGQPuDCF2 --- CMakeLists.txt | 2 + cmake/parity_oracle.cmake | 73 ++++ cmake/parity_oracle_generate.cmake | 50 +++ docs/tasks/0186-python-oracle-target.md | 21 +- .../0372-unicode-version-corpus-cases.md | 31 ++ test/parity/README.md | 47 ++ test/parity/generate.py | 66 ++- test/parity/parity_test.cpp | 401 +++++++++++++++++- 8 files changed, 667 insertions(+), 24 deletions(-) create mode 100644 cmake/parity_oracle.cmake create mode 100644 cmake/parity_oracle_generate.cmake create mode 100644 docs/tasks/0372-unicode-version-corpus-cases.md diff --git a/CMakeLists.txt b/CMakeLists.txt index eee659885..2642a162e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -578,6 +578,8 @@ if (JINJA2CPP_BUILD_TESTS) # Differential corpus against Python Jinja2, see test/parity/README.md add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*) + # The same suite against another Python Jinja2, when JINJA2CPP_TEST_ORACLE asks (docs/tasks/0186) + include(cmake/parity_oracle.cmake) # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076) if (CMAKE_NM AND NOT MSVC) add_test(NAME jinja2cpp_abi_namespace diff --git a/cmake/parity_oracle.cmake b/cmake/parity_oracle.cmake new file mode 100644 index 000000000..dd49d9e68 --- /dev/null +++ b/cmake/parity_oracle.cmake @@ -0,0 +1,73 @@ +# Compare the parity suite with a Python Jinja2 other than the bundled one (docs/tasks/0186, +# test/parity/README.md "Other oracles"). Included from the test block of CMakeLists.txt. +# +# bundled (default): the checked-in test/parity/expected/; nothing here runs and no Python is needed +# python: test/parity/generate.py runs under JINJA2CPP_TEST_ORACLE_PYTHON (default: the +# Python3 interpreter CMake finds) with whatever jinja2 and markupsafe it has +# venv: a venv in the build directory gets JINJA2CPP_TEST_ORACLE_REQUIREMENTS from pip +# +# The other oracle's expectations are generated at test time, by the fixture +# jinja2cpp_parity_oracle_generate, so configure needs no network and a failing pip is a failing +# test. jinja2cpp_parity_oracle then runs the narrow suite against them and writes +# parity-oracle/report.md in the build directory. +set(JINJA2CPP_TEST_ORACLE bundled CACHE STRING "Python Jinja2 the parity suite compares against: bundled | python | venv") +set_property(CACHE JINJA2CPP_TEST_ORACLE PROPERTY STRINGS bundled python venv) +set(JINJA2CPP_TEST_ORACLE_PYTHON "" CACHE FILEPATH "Interpreter for JINJA2CPP_TEST_ORACLE=python, and the base of the venv (default: find_package(Python3))") +set(JINJA2CPP_TEST_ORACLE_REQUIREMENTS "" CACHE STRING + "pip requirements for JINJA2CPP_TEST_ORACLE=venv, ';'-separated, e.g. 'jinja2==3.1.2;markupsafe==2.1.5' or 'jinja2 @ git+https://github.com/pallets/jinja@main'") + +if (JINJA2CPP_TEST_ORACLE STREQUAL "bundled") + return() +endif () +if (NOT JINJA2CPP_TEST_ORACLE MATCHES "^(python|venv)$") + message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE must be bundled, python or venv, not '${JINJA2CPP_TEST_ORACLE}'") +endif () +if (JINJA2CPP_TEST_ORACLE STREQUAL "venv" AND JINJA2CPP_TEST_ORACLE_REQUIREMENTS STREQUAL "") + message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE=venv needs JINJA2CPP_TEST_ORACLE_REQUIREMENTS, e.g. 'jinja2==3.1.2'") +endif () + +if (JINJA2CPP_TEST_ORACLE_PYTHON) + set(_oracle_python "${JINJA2CPP_TEST_ORACLE_PYTHON}") +else () + find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter) + set(_oracle_python "${Python3_EXECUTABLE}") +endif () +execute_process(COMMAND "${_oracle_python}" -c "import sys; print('%d.%d' % sys.version_info[:2])" + OUTPUT_VARIABLE _oracle_python_version OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _oracle_python_result) +if (NOT _oracle_python_result EQUAL 0) + message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE: cannot run ${_oracle_python}") +endif () +# The Unicode tables in src/ come from the pinned Python, so Unicode cases may differ for a +# reason that is not a Jinja2C++ bug +if (NOT _oracle_python_version VERSION_EQUAL JINJA2CPP_ORACLE_PYTHON) + message(WARNING "JINJA2CPP_TEST_ORACLE: the oracle runs Python ${_oracle_python_version}, but Jinja2C++ follows Python " + "${JINJA2CPP_ORACLE_PYTHON} (.python-version); cases that depend on the Python version (Unicode data, textwrap, " + "\\N{} names) can show up in the report as deltas that are not Jinja2 changes") +endif () +message(STATUS "Parity oracle: ${JINJA2CPP_TEST_ORACLE}, Python ${_oracle_python_version} (${_oracle_python})") + +set(_oracle_dir ${CMAKE_CURRENT_BINARY_DIR}/parity-oracle) +set(_oracle_requirements "") +if (JINJA2CPP_TEST_ORACLE STREQUAL "venv") + # A file, so the list reaches pip without quoting trouble; the generate step reinstalls when it changes + set(_oracle_requirements ${_oracle_dir}/requirements.txt) + string(REPLACE ";" "\n" _oracle_requirement_lines "${JINJA2CPP_TEST_ORACLE_REQUIREMENTS}") + file(WRITE ${_oracle_requirements}.in "${_oracle_requirement_lines}\n") + configure_file(${_oracle_requirements}.in ${_oracle_requirements} COPYONLY) +endif () + +add_test(NAME jinja2cpp_parity_oracle_generate + COMMAND ${CMAKE_COMMAND} + -DPYTHON=${_oracle_python} + -DREQUIREMENTS=${_oracle_requirements} + -DGENERATE=${CMAKE_CURRENT_SOURCE_DIR}/test/parity/generate.py + -DOUT=${_oracle_dir} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/parity_oracle_generate.cmake) +set_tests_properties(jinja2cpp_parity_oracle_generate PROPERTIES FIXTURES_SETUP jinja2cpp_parity_oracle) + +# The narrow suite and the registry checks; the wide suite is held to the narrow result, not +# to an oracle, so it has nothing to add +add_test(NAME jinja2cpp_parity_oracle COMMAND jinja2cpp_tests --gtest_filter=Parity/*:ParityRegistry.*) +set_tests_properties(jinja2cpp_parity_oracle PROPERTIES + FIXTURES_REQUIRED jinja2cpp_parity_oracle + ENVIRONMENT "JINJA2CPP_TEST_ORACLE_DIR=${_oracle_dir}") diff --git a/cmake/parity_oracle_generate.cmake b/cmake/parity_oracle_generate.cmake new file mode 100644 index 000000000..71b38802c --- /dev/null +++ b/cmake/parity_oracle_generate.cmake @@ -0,0 +1,50 @@ +# The setup step of the jinja2cpp_parity_oracle fixture (cmake/parity_oracle.cmake): renders the +# parity corpus with the other oracle into OUT. +# cmake -DPYTHON= -DGENERATE= -DOUT= [-DREQUIREMENTS=] -P parity_oracle_generate.cmake +# With REQUIREMENTS, PYTHON is the base of a venv in OUT/venv that pip fills from that file. The venv +# is rebuilt when the file changes; delete OUT/venv to pick up a new commit of a branch requirement. +foreach (_var PYTHON GENERATE OUT) + if (NOT ${_var}) + message(FATAL_ERROR "parity_oracle_generate.cmake needs -D${_var}=...") + endif () +endforeach () + +set(_python "${PYTHON}") +if (REQUIREMENTS) + set(_venv ${OUT}/venv) + if (WIN32) + set(_python ${_venv}/Scripts/python.exe) + else () + set(_python ${_venv}/bin/python) + endif () + set(_stamp ${_venv}/jinja2cpp-requirements.txt) + set(_fresh FALSE) + if (EXISTS ${_stamp} AND EXISTS ${_python}) + file(READ ${_stamp} _installed) + file(READ ${REQUIREMENTS} _wanted) + if (_installed STREQUAL _wanted) + set(_fresh TRUE) + endif () + endif () + if (NOT _fresh) + message(STATUS "Creating ${_venv} with ${PYTHON}") + execute_process(COMMAND "${PYTHON}" -m venv --clear ${_venv} RESULT_VARIABLE _result) + if (NOT _result EQUAL 0) + message(FATAL_ERROR "python -m venv failed (${_result})") + endif () + execute_process(COMMAND "${_python}" -m pip install --disable-pip-version-check -r ${REQUIREMENTS} RESULT_VARIABLE _result) + if (NOT _result EQUAL 0) + message(FATAL_ERROR "pip install -r ${REQUIREMENTS} failed (${_result}); the venv oracle needs network access to the package index") + endif () + configure_file(${REQUIREMENTS} ${_stamp} COPYONLY) + endif () +endif () + +execute_process(COMMAND "${_python}" -c "import jinja2" RESULT_VARIABLE _result OUTPUT_QUIET ERROR_QUIET) +if (NOT _result EQUAL 0) + message(FATAL_ERROR "${_python} cannot import jinja2: install it there, or use JINJA2CPP_TEST_ORACLE=venv") +endif () +execute_process(COMMAND "${_python}" ${GENERATE} --out ${OUT} --any-version RESULT_VARIABLE _result) +if (NOT _result EQUAL 0) + message(FATAL_ERROR "generate.py --out ${OUT} failed (${_result})") +endif () diff --git a/docs/tasks/0186-python-oracle-target.md b/docs/tasks/0186-python-oracle-target.md index 2b7c955f7..901cf4204 100644 --- a/docs/tasks/0186-python-oracle-target.md +++ b/docs/tasks/0186-python-oracle-target.md @@ -2,7 +2,7 @@ status: in-progress priority: medium area: parity -touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in] +touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in, cmake/parity_oracle.cmake, cmake/parity_oracle_generate.cmake, test/parity/parity_test.cpp, test/parity/README.md] --- # Which Python and Jinja2 the parity oracle targets, and how users choose @@ -50,13 +50,18 @@ Decisions: O2 yes (version macros, before the G1 freeze), O3 no, O4 report-only `JINJA2CPP_ORACLE_PYTHON` and `JINJA2CPP_ORACLE_JINJA2`; `docs/parity.md` has the policy and the one-PR bump checklist ("Oracle pin"). This answers the policy question and the "where is it stated" question. -- **Phase 1, open.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`, - `_PYTHON`, `_REQUIREMENTS`), `generate.py --out DIR --any-version`, oracle-directory mode - in `parity_test.cpp` (env `JINJA2CPP_TEST_ORACLE_DIR`) that classifies each differing case - `tracks-bundled`, `tracks-oracle` or `neither`, writes `report.md`, and a task for the - corpus gap on code points changed after Unicode 15.0. Must land before 0355: the Python - module's parity proof (0355) waits on phase 1 and reuses its `generate.py --out` harness - and delta classification instead of a second one. +- **Phase 1, done in this PR.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`, + `_PYTHON`, `_REQUIREMENTS`; included from the test block) registers the CTest fixture + `jinja2cpp_parity_oracle_generate` (`cmake/parity_oracle_generate.cmake`: makes the venv, + runs `generate.py --out DIR --any-version`, which also writes `oracle.json`) and the test + `jinja2cpp_parity_oracle`. `parity_test.cpp` reads `JINJA2CPP_TEST_ORACLE_DIR`, judges a case + by the allow-list where the oracles agree, otherwise records it as `tracks-bundled`, + `tracks-oracle`, `neither` or `not-run` (failing only under `JINJA2CPP_TEST_ORACLE_STRICT=1`), + and writes `report.md`. Documented in test/parity/README.md, "Other oracles". The section 8 + check holds: Jinja2 3.1.2 (MarkupSafe 2.1.5, Python 3.12) reports the 8 cases of plan + section 2 plus `string_filters.m168`, all `tracks-bundled`; 3.0.3 on Python 3.13 reports 22. + The Unicode-corpus gap is 0372. 0355 reuses `generate.py --out/--any-version`, + `JINJA2CPP_TEST_ORACLE_DIR` and the verdicts rather than a second harness. - **Phase 2, open.** `.github/workflows/parity-oracles.yml` modelled on `bindings.yml` (paths-filtered PRs plus nightly, actions pinned by SHA): Jinja2 main, 3.0.x, Python 3.13/3.14 rows with the pinned Jinja2. diff --git a/docs/tasks/0372-unicode-version-corpus-cases.md b/docs/tasks/0372-unicode-version-corpus-cases.md new file mode 100644 index 000000000..b4d49a079 --- /dev/null +++ b/docs/tasks/0372-unicode-version-corpus-cases.md @@ -0,0 +1,31 @@ +--- +status: open +priority: low +area: parity +depends: [0186] +touches: [test/parity/cases/literals.py, test/parity/cases/string_filters.py, test/parity/expected/literals.json, test/parity/expected/string_filters.json] +--- +# Corpus cases on code points whose Unicode properties changed after 15.0 + +**Problem.** Jinja2C++'s generated tables (`src/unicode_printable.h`, `unicode_tables.h`, +`unicode_case.h`, `unicode_names.h`) follow the pinned Python 3.12, which ships Unicode 15.0. +Python 3.13 ships 15.1 and 3.14 ships 16.0. The 0186 oracle runs (`JINJA2CPP_TEST_ORACLE`, +test/parity/README.md "Other oracles") show the Python axis only where the corpus has a case, +and the corpus has none on a code point that 15.1 or 16.0 assigned or changed: under Python +3.13 the measured deltas were stdlib code (`textwrap`, `\N{}` name rules), never table data +(0186 plan, section 10). A table change between Python versions is therefore invisible to the +oracle rows of phase 2. + +For example, U+2EBF0 (first ideograph of CJK Extension I, new in 15.1): Python 3.12 gives +`repr` `'\U0002ebf0'`, `isalpha()` False, no name; Python 3.13 gives `'𮯰'`, True and +`CJK UNIFIED IDEOGRAPH-2EBF0`. + +**Proposal.** Add a few cases, in the areas that own each behaviour, on code points new or +changed in 15.1 and 16.0: `repr` of a string holding one (printable table), `upper`/`lower`/ +`title` on a cased letter added in 16.0 (case tables), a `\N{...}` escape naming one (names +table), and `int()`/`wordcount` style uses of `\d`/`\w` if 16.0 changed a digit or word +property. They match under the pinned 3.12 (no allow-list line), and the 3.13/3.14 oracle rows +then report them as `tracks-bundled`, which is the record that the Python axis is measured. + +**Done when.** The cases are in the corpus, the bundled suite matches them, and an oracle run +under Python 3.13 lists them in `report.md`. diff --git a/test/parity/README.md b/test/parity/README.md index 5c5dd9efd..8c42cf9f5 100644 --- a/test/parity/README.md +++ b/test/parity/README.md @@ -123,3 +123,50 @@ When two branches add cases to the same area, take either side of the conflict i Keep cases small and aimed at one feature. Print booleans as `'T' if ... else 'F'` and iterate lists instead of printing them when the case is about something else, so that the repr gaps (task 0012) do not mask the behaviour under test. + +## Other oracles + +The checked-in `expected/` comes from the pinned pair (`.python-version`, `requirements.txt`; +docs/parity.md, "Oracle pin"). To see how Jinja2C++ compares with another Python Jinja2, such +as an older 3.x, Jinja2's main branch or another Python, configure the build with +`JINJA2CPP_TEST_ORACLE` (docs/tasks/0186): + +| Value | What the oracle is | +|---|---| +| `bundled` (default) | `expected/`; nothing else runs and no Python is needed | +| `python` | the jinja2 and markupsafe installed for `JINJA2CPP_TEST_ORACLE_PYTHON` (default: the Python 3 CMake finds) | +| `venv` | a venv in `/parity-oracle/venv`, made from `JINJA2CPP_TEST_ORACLE_PYTHON` and filled by pip from `JINJA2CPP_TEST_ORACLE_REQUIREMENTS` (`;`-separated, e.g. `jinja2==3.1.2;markupsafe==2.1.5` or `jinja2 @ git+https://github.com/pallets/jinja@main`) | + +```bash +cmake -S . -B build -DJINJA2CPP_TEST_ORACLE=venv -DJINJA2CPP_TEST_ORACLE_REQUIREMENTS='jinja2==3.1.2' +ctest --test-dir build -R parity_oracle --output-on-failure +cat build/parity-oracle/report.md +``` + +`jinja2cpp_parity_oracle_generate` (a CTest fixture, so configure needs no Python packages +and no network) runs `generate.py --out /parity-oracle --any-version`, which writes the +other oracle's expectations there with `oracle.json` naming its Python, Jinja2 and MarkupSafe. +`venv` needs network at test time; the venv is rebuilt when the requirements change, and +deleting it picks up a new commit of a branch requirement. Supported oracles: Python >= 3.10, +Jinja2 >= 3.0. Configure warns when the oracle's Python is not the pinned one, because the +Unicode tables and Python's own behaviour (`textwrap`, `\N{}` names) follow the pinned one. + +`jinja2cpp_parity_oracle` runs the narrow suite with `JINJA2CPP_TEST_ORACLE_DIR` naming that +directory (set it by hand to compare against any `generate.py --out` directory; the test binary +needs no rebuild). Each case is judged by whether the two oracles agree (same output, or both +an error): + +- **they agree**: judged as without an oracle, allow-list included; a regression fails; +- **they differ** (a Jinja2 or Python version delta): reported, not failed. The verdict is + `tracks-bundled` (Jinja2C++ matches the pinned oracle), `tracks-oracle` (matches the other + one), `neither`, or `not-run` (a listed crash, an unsupported option, or a build without the + Unicode name table). With `JINJA2CPP_TEST_ORACLE_STRICT=1` every verdict but + `tracks-oracle` fails. + +The run writes `report.md` next to the expectations: the two oracles' versions, the counts, +and one row per differing case with the bundled result, the other oracle's and Jinja2C++'s. +Against Jinja2 3.1.2 with MarkupSafe 2.1.5 on Python 3.12 that is 9 rows, all +`tracks-bundled`; against 3.0.3 on Python 3.13, 22. `ParityRegistry.OracleCorpusMatches` +fails when the directory does not hold the same cases as the corpus (a stale generation). +The wide suite is held to the narrow result rather than to an oracle, so the oracle test does +not run it. diff --git a/test/parity/generate.py b/test/parity/generate.py index 2d66ea92e..4d732b10f 100644 --- a/test/parity/generate.py +++ b/test/parity/generate.py @@ -16,12 +16,21 @@ python3 test/parity/generate.py # rewrite expected/*.json python3 test/parity/generate.py --check # fail if expected/*.json is stale python3 test/parity/generate.py --report # parity table by area and task (Markdown) + python3 test/parity/generate.py --out DIR --any-version # another oracle (docs/tasks/0186) + +``--out DIR`` writes the expectations to DIR instead of ``expected/``, with ``oracle.json`` +naming the Python, Jinja2 and MarkupSafe that produced them. ``--any-version`` accepts a +Jinja2 other than the pinned one (only with ``--out``, so ``expected/`` keeps the pin); +``parity_test.cpp`` compares against DIR when ``JINJA2CPP_TEST_ORACLE_DIR`` names it. """ import argparse import difflib +import importlib.metadata import importlib.util import json import pathlib +import platform +import re import sys import jinja2 @@ -42,6 +51,33 @@ def pinned_jinja2_version(): JINJA2_VERSION = pinned_jinja2_version() +# The oldest oracle generate.py supports with --any-version (test/parity/README.md, "Other oracles") +MIN_PYTHON = (3, 10) +MIN_JINJA2 = (3, 0) + + +def package_version(name, module=None): + """A distribution's version, from its metadata: jinja2 and markupsafe deprecate __version__.""" + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return getattr(module, "__version__", "unknown") + + +def release_tuple(version): + """(3, 1) for "3.1.6", "3.2.0.dev0" or "3.0.0rc1".""" + match = re.match(r"(\d+)\.(\d+)", version) + return (int(match[1]), int(match[2])) if match else (0, 0) + + +def oracle_versions(): + import markupsafe # a dependency of jinja2, so always present + return {"python": platform.python_version(), "jinja2": package_version("jinja2", jinja2), + "markupsafe": package_version("markupsafe", markupsafe)} + + +JINJA2_FOUND = package_version("jinja2", jinja2) + # Options the C++ harness knows how to map (or knows it cannot map yet). ENV_OPTIONS = { "trim_blocks", "lstrip_blocks", "keep_trailing_newline", "autoescape", @@ -128,7 +164,7 @@ def load_area(path): raise ValueError(f"{case['id']}: output is not deterministic") case.update(result) cases.append(case) - return {"generator": f"jinja2 {jinja2.__version__}", "area": path.stem, "cases": cases} + return {"generator": f"jinja2 {JINJA2_FOUND}", "area": path.stem, "cases": cases} def report(): @@ -172,15 +208,28 @@ def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) parser.add_argument("--check", action="store_true", help="fail if expected/ is stale") parser.add_argument("--report", action="store_true", help="print the parity table and exit") + parser.add_argument("--out", type=pathlib.Path, help="write the expectations to this directory, not expected/") + parser.add_argument("--any-version", action="store_true", + help=f"accept a Jinja2 other than the pinned {JINJA2_VERSION} (needs --out)") args = parser.parse_args() if args.report: return report() - if jinja2.__version__ != JINJA2_VERSION: - sys.exit(f"expected jinja2 {JINJA2_VERSION}, found {jinja2.__version__}; " + if args.any_version and not args.out: + sys.exit("--any-version needs --out: expected/ holds the pinned oracle's results") + if args.check and args.out: + sys.exit("--check compares expected/; it does not take --out") + versions = oracle_versions() + if args.any_version: + if sys.version_info[:2] < MIN_PYTHON or release_tuple(JINJA2_FOUND) < MIN_JINJA2: + sys.exit(f"unsupported oracle Python {versions['python']} with Jinja2 {JINJA2_FOUND}; " + f"the parity oracle needs Python >= {'.'.join(map(str, MIN_PYTHON))} " + f"and Jinja2 >= {'.'.join(map(str, MIN_JINJA2))}") + elif JINJA2_FOUND != JINJA2_VERSION: + sys.exit(f"expected jinja2 {JINJA2_VERSION}, found {JINJA2_FOUND}; " f"pip install jinja2=={JINJA2_VERSION}") - out_dir = HERE / "expected" - out_dir.mkdir(exist_ok=True) + out_dir = args.out.resolve() if args.out else HERE / "expected" + out_dir.mkdir(parents=True, exist_ok=True) stale, total = [], 0 wanted = set() for path in sorted((HERE / "cases").glob("*.py")): @@ -200,6 +249,10 @@ def main(): target.write_text(text, encoding="utf-8") index = json.dumps({"areas": sorted(n[:-5] for n in wanted)}, indent=1) + "\n" wanted.add("index.json") + if args.out: + # Which oracle produced the directory, for the delta report of parity_test.cpp + (out_dir / "oracle.json").write_text(json.dumps(versions, indent=1) + "\n", encoding="utf-8") + wanted.add("oracle.json") index_path = out_dir / "index.json" if not index_path.exists() or index_path.read_text(encoding="utf-8") != index: if args.check: @@ -213,7 +266,8 @@ def main(): (out_dir / orphan).unlink() if stale: sys.exit(f"stale expectations: {', '.join(stale)}; run test/parity/generate.py") - print(f"{total} cases, jinja2 {jinja2.__version__}") + print(f"{total} cases, Python {versions['python']}, jinja2 {versions['jinja2']}, " + f"markupsafe {versions['markupsafe']}" + (f", in {out_dir}" if args.out else "")) if __name__ == "__main__": diff --git a/test/parity/parity_test.cpp b/test/parity/parity_test.cpp index 0b863af16..4ee760328 100644 --- a/test/parity/parity_test.cpp +++ b/test/parity/parity_test.cpp @@ -3,6 +3,8 @@ // Known divergences are listed in test/parity/divergences/.txt; see test/parity/README.md. // ParityWideTest renders the same cases through TemplateW and lists only the cases where // the wide result differs from the narrow one, under a "wide." id prefix. +// With JINJA2CPP_TEST_ORACLE_DIR set, ParityTest also compares against the expectations another +// Python Jinja2 wrote there (generate.py --out) and writes report.md beside them (docs/tasks/0186). #include "jinja2cpp/value.h" #include @@ -14,15 +16,18 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include #include @@ -171,6 +176,20 @@ std::string DivergenceProblem(const Divergence& d) return {}; } +// An environment variable, or an empty string +std::string GetEnv(const char* name) +{ +#ifdef _MSC_VER + // std::getenv is C4996 under /W4 /WX + char* value = nullptr; + _dupenv_s(&value, nullptr, name); + std::unique_ptr owner(value, &std::free); +#else + const char* value = std::getenv(name); +#endif + return value ? std::string(value) : std::string(); +} + struct ParityCase { std::string id; @@ -577,16 +596,11 @@ Result RenderCpp(const Json& c, const jinja2::Settings& settings) void RecordOutcome(const std::string& id, const char* outcome) { static std::unique_ptr out = []() -> std::unique_ptr { -#ifdef _MSC_VER - // std::getenv is C4996 under /W4 /WX - char* path = nullptr; - _dupenv_s(&path, nullptr, "JINJA2CPP_PARITY_RESULTS"); - std::unique_ptr owner(path, &std::free); -#else - const char* path = std::getenv("JINJA2CPP_PARITY_RESULTS"); -#endif - if (!path || !*path) + const auto path = GetEnv("JINJA2CPP_PARITY_RESULTS"); + if (path.empty()) + { return nullptr; + } auto file = std::make_unique(path, std::ios::binary | std::ios::app); *file << "# features:"; for (const auto& feature : BuildFeatures()) @@ -614,6 +628,299 @@ Outcome Classify(const Json& c, const Result& cpp) return pyOk ? Outcome::Rejects : Outcome::Match; } +// Another oracle (docs/tasks/0186, test/parity/README.md "Other oracles"). The allow-list is +// written against the bundled Jinja2, so a case is judged by it only where the two oracles agree; +// where they differ (a Jinja2 or Python version delta) the case is reported, and fails only +// under JINJA2CPP_TEST_ORACLE_STRICT=1 when Jinja2C++ does not follow the other oracle. +enum class OracleVerdict +{ + Agree, // the oracles agree: judged by the allow-list as without an oracle + TracksBundled, // they differ and Jinja2C++ matches the bundled one + TracksOracle, // they differ and Jinja2C++ matches the other one + Neither, // they differ and Jinja2C++ matches neither + NotRun, // they differ and the case does not run here (crash, unsupported, no name table) +}; + +const char* OracleVerdictName(OracleVerdict v) +{ + switch (v) + { + case OracleVerdict::Agree: + return "agree"; + case OracleVerdict::TracksBundled: + return "tracks-bundled"; + case OracleVerdict::TracksOracle: + return "tracks-oracle"; + case OracleVerdict::Neither: + return "neither"; + case OracleVerdict::NotRun: + return "not-run"; + } + return "?"; +} + +// Whether two oracles agree on a case: the same output, or both an error (messages are not +// compared, as Classify does not compare them) +bool OraclesAgree(const Json& bundled, const Json& oracle) +{ + if (bundled.contains("output") != oracle.contains("output")) + { + return false; + } + return !bundled.contains("output") || bundled["output"] == oracle["output"]; +} + +// cpp is null when the case did not run +OracleVerdict JudgeOracle(const Json& bundled, const Json& oracle, const Result* cpp) +{ + if (OraclesAgree(bundled, oracle)) + { + return OracleVerdict::Agree; + } + if (!cpp) + { + return OracleVerdict::NotRun; + } + if (Classify(bundled, *cpp) == Outcome::Match) + { + return OracleVerdict::TracksBundled; + } + return Classify(oracle, *cpp) == Outcome::Match ? OracleVerdict::TracksOracle : OracleVerdict::Neither; +} + +struct OracleRun +{ + std::string dir; // JINJA2CPP_TEST_ORACLE_DIR; empty = no other oracle + bool strict = false; // JINJA2CPP_TEST_ORACLE_STRICT=1 + Json versions; // oracle.json: python, jinja2, markupsafe + std::map cases; // by id + std::vector problems; // reported by ParityRegistry.OracleCorpusMatches +}; + +const OracleRun& Oracle() +{ + static const OracleRun run = [] { + OracleRun r; + r.dir = GetEnv("JINJA2CPP_TEST_ORACLE_DIR"); + r.strict = GetEnv("JINJA2CPP_TEST_ORACLE_STRICT") == "1"; + if (r.dir.empty()) + { + return r; + } + const auto versions = ReadFile(r.dir + "/oracle.json"); + if (versions.empty()) + { + r.problems.push_back("missing " + r.dir + "/oracle.json; run test/parity/generate.py --out " + r.dir + " --any-version"); + } + else + { + r.versions = Json::parse(versions); + } + const auto index = ReadFile(r.dir + "/index.json"); + if (index.empty()) + { + r.problems.push_back("missing " + r.dir + "/index.json"); + return r; + } + const auto areas = Json::parse(index)["areas"].get>(); + for (const auto& area : areas) + { + const auto text = ReadFile(r.dir + "/" + area + ".json"); + if (text.empty()) + { + r.problems.push_back("missing " + r.dir + "/" + area + ".json"); + continue; + } + auto doc = Json::parse(text); // named: a range-for over parse(text)["cases"] would outlive the document + for (auto& c : doc["cases"]) + { + r.cases.emplace(c["id"].get(), std::move(c)); + } + } + return r; + }(); + return run; +} + +struct OracleDelta +{ + std::string id; + std::string bundled; + std::string oracle; + std::string cpp; + OracleVerdict verdict; +}; + +std::vector& OracleDeltas() +{ + static std::vector deltas; + return deltas; +} + +std::size_t& OracleAgreeCount() +{ + static std::size_t count = 0; + return count; +} + +// One Markdown table cell: the text from byte skip on as a JSON string, so newlines and spaces +// show, cut short +std::string ReportCell(const std::string& text, std::size_t skip = 0) +{ + // Back to the start of a UTF-8 sequence, so the cell never begins inside a character + while (skip > 0 && skip < text.size() && (static_cast(text[skip]) & 0xC0) == 0x80) + { + --skip; + } + auto cell = Json(text.substr(std::min(skip, text.size()))).dump(); + if (skip > 0) + { + cell.insert(1, "..."); + } + if (cell.size() > 100) + { + cell = cell.substr(0, 97) + "..."; + } + std::string escaped; + for (char ch : cell) + { + if (ch == '|') + { + escaped += '\\'; + } + escaped += ch; + } + return "`" + escaped + "`"; +} + +// Where the report cells of a delta start: long outputs that differ late (urlize) show the +// difference rather than the shared head +std::size_t ReportSkip(const std::string& bundled, const std::string& oracle) +{ + const auto common = static_cast(std::mismatch(bundled.begin(), bundled.begin() + static_cast(std::min(bundled.size(), oracle.size())), oracle.begin()).first - bundled.begin()); + return common > 40 ? common - 20 : 0; +} + +std::string CppText(const Result* cpp) +{ + if (!cpp) + { + return "(not run)"; + } + return cpp->ok ? cpp->text : "error: " + cpp->text; +} + +// The pin of a package in test/parity/requirements.txt, or "?" +std::string BundledPin(const std::string& package) +{ + std::istringstream in(ReadFile(std::string(JINJA2CPP_PARITY_DIR) + "/requirements.txt")); + std::string line; + while (std::getline(in, line)) + { + if (line.compare(0, package.size() + 2, package + "==") == 0) + { + auto version = line.substr(package.size() + 2); + version.erase(std::find_if(version.begin(), version.end(), [](char ch) { return std::isspace(static_cast(ch)) || ch == '#'; }), + version.end()); + return version; + } + } + return "?"; +} + +std::string OracleVersion(const char* key) +{ + const auto& v = Oracle().versions; + return v.contains(key) ? v[key].get() : "?"; +} + +void WriteOracleReport() +{ + const auto& oracle = Oracle(); + auto deltas = OracleDeltas(); + if (oracle.dir.empty() || (deltas.empty() && OracleAgreeCount() == 0)) + { + return; + } + std::sort(deltas.begin(), deltas.end(), [](const OracleDelta& a, const OracleDelta& b) { return a.id < b.id; }); + std::map counts; + for (const auto& d : deltas) + { + ++counts[d.verdict]; + } + + std::ostringstream os; + os << "# Parity against another oracle\n\n" + << "| | Python | Jinja2 | MarkupSafe |\n|---|---|---|---|\n" + << "| bundled | " << JINJA2CPP_ORACLE_PYTHON << " | " << JINJA2CPP_ORACLE_JINJA2 << " | " << BundledPin("markupsafe") << " |\n" + << "| oracle | " << OracleVersion("python") << " | " << OracleVersion("jinja2") << " | " << OracleVersion("markupsafe") << " |\n\n" + << "Jinja2C++ " << JINJA2CPP_VERSION_STRING << ". Cases where the oracles agree: " << OracleAgreeCount() + << " (judged by the allow-list as usual). Cases where they differ: " << deltas.size(); + for (const auto verdict : { OracleVerdict::TracksBundled, OracleVerdict::TracksOracle, OracleVerdict::Neither, OracleVerdict::NotRun }) + { + os << ", " << OracleVerdictName(verdict) << " " << counts[verdict]; + } + os << ".\n\n"; + if (!deltas.empty()) + { + os << "| case | bundled | oracle | Jinja2C++ | verdict |\n|---|---|---|---|---|\n"; + for (const auto& d : deltas) + { + const auto skip = ReportSkip(d.bundled, d.oracle); + os << "| " << d.id << " | " << ReportCell(d.bundled, skip) << " | " << ReportCell(d.oracle, skip) << " | " << ReportCell(d.cpp, skip) << " | " + << OracleVerdictName(d.verdict) << " |\n"; + } + } + const auto path = oracle.dir + "/report.md"; + std::ofstream(path, std::ios::binary) << os.str(); + std::cout << "[parity oracle] Python " << OracleVersion("python") << ", Jinja2 " << OracleVersion("jinja2") << ": " << OracleAgreeCount() + << " cases agree with the bundled oracle, " << deltas.size() << " differ (tracks-bundled " << counts[OracleVerdict::TracksBundled] + << ", tracks-oracle " << counts[OracleVerdict::TracksOracle] << ", neither " << counts[OracleVerdict::Neither] << ", not-run " + << counts[OracleVerdict::NotRun] << "); report: " << path << std::endl; +} + +// Writes report.md once every case has run, whatever the filter or order +class OracleReportEnvironment : public ::testing::Environment +{ +public: + void TearDown() override { WriteOracleReport(); } +}; + +// NOLINTNEXTLINE(bugprone-throwing-static-initialization,cppcoreguidelines-owning-memory): only allocation can throw, at load time; gtest owns the environment +[[maybe_unused]] const auto* const g_oracleReport = ::testing::AddGlobalTestEnvironment(new OracleReportEnvironment); + +// With another oracle: whether the case is judged by the allow-list (the oracles agree, or no +// other oracle is set). Otherwise records the delta and, in strict mode, fails unless Jinja2C++ +// follows the other oracle. cpp is null when the case does not run. +bool JudgedByAllowList(const std::string& id, const Json& c, const Result* cpp) +{ + const auto& oracle = Oracle(); + if (oracle.dir.empty()) + { + return true; + } + const auto found = oracle.cases.find(id); + if (found == oracle.cases.end()) + { + ADD_FAILURE() << id << " is missing from " << oracle.dir << "; regenerate it with generate.py --out"; + return true; + } + const auto verdict = JudgeOracle(c, found->second, cpp); + if (verdict == OracleVerdict::Agree) + { + ++OracleAgreeCount(); + return true; + } + OracleDeltas().push_back({ id, PythonText(c), PythonText(found->second), CppText(cpp), verdict }); + if (oracle.strict && verdict != OracleVerdict::TracksOracle) + { + ADD_FAILURE() << "JINJA2CPP_TEST_ORACLE_STRICT: " << id << " is " << OracleVerdictName(verdict) << "\n template: " << c["template"].dump() + << "\n oracle: " << Json(PythonText(found->second)).dump() << "\n bundled: " << Json(PythonText(c)).dump() + << "\n c++: " << Json(CppText(cpp)).dump(); + } + return false; +} + std::string Details(const Json& c, const std::string& cppText) { std::ostringstream details; @@ -692,11 +999,13 @@ TEST_P(ParityTest, MatchesPython) const auto& c = GetParam().data; if (NeedsUnicodeNames(c)) { + JudgedByAllowList(id, c, nullptr); GTEST_SKIP() << "built without the Unicode name table"; } if (const auto* crash = ListedCrash(id)) { RecordOutcome(id, "crash"); + JudgedByAllowList(id, c, nullptr); GTEST_SKIP() << "crashes Jinja2C++, task " << crash->task << ": " << crash->reason; } @@ -715,7 +1024,10 @@ TEST_P(ParityTest, MatchesPython) outcome = Classify(c, cpp); } RecordOutcome(id, OutcomeName(outcome)); - CheckListed(id, outcome, Details(c, cpp.text)); + if (JudgedByAllowList(id, c, outcome == Outcome::Unsupported ? nullptr : &cpp)) + { + CheckListed(id, outcome, Details(c, cpp.text)); + } } // The wide suite holds TemplateW to the narrow result rather than to Python, so that a @@ -842,6 +1154,75 @@ TEST(ParityRegistry, BuildConditions) EXPECT_EQ(ListingProblem(id, nullptr, Outcome::Match, without), ""); } +// How a case is judged against another oracle (docs/tasks/0186) +TEST(ParityRegistry, OracleVerdicts) +{ + const auto out = [](const char* text) { return Json{ { "output", text } }; }; + const Json err = { { "error", { { "phase", "render" }, { "type", "TypeError" }, { "message", "m" } } } }; + const Json errOther = { { "error", { { "phase", "compile" }, { "type", "TemplateSyntaxError" }, { "message", "other" } } } }; + const Result cppA{ true, "a" }; + const Result cppB{ true, "b" }; + const Result cppC{ true, "c" }; + const Result cppErr{ false, "render: x" }; + struct Row + { + Json bundled; + Json oracle; + const Result* cpp; + OracleVerdict verdict; + }; + const Row rows[] = { + { out("a"), out("a"), &cppB, OracleVerdict::Agree }, // judged by the allow-list, whatever C++ does + { err, errOther, &cppA, OracleVerdict::Agree }, // errors agree by presence + { out("a"), out("b"), &cppA, OracleVerdict::TracksBundled }, + { out("a"), out("b"), &cppB, OracleVerdict::TracksOracle }, + { out("a"), out("b"), &cppC, OracleVerdict::Neither }, + { out("a"), out("b"), &cppErr, OracleVerdict::Neither }, + { out("a"), err, &cppErr, OracleVerdict::TracksOracle }, + { err, out("a"), &cppErr, OracleVerdict::TracksBundled }, + { err, out("a"), &cppA, OracleVerdict::TracksOracle }, + { out("a"), out("b"), nullptr, OracleVerdict::NotRun }, + { out("a"), out("a"), nullptr, OracleVerdict::Agree }, + }; + for (const auto& row : rows) + { + EXPECT_STREQ(OracleVerdictName(JudgeOracle(row.bundled, row.oracle, row.cpp)), OracleVerdictName(row.verdict)) + << row.bundled.dump() << " / " << row.oracle.dump() << " / " << CppText(row.cpp); + } + EXPECT_EQ(ReportCell("a|b\n"), "`\"a\\|b\\n\"`"); + EXPECT_EQ(ReportCell("ab\xC3\xA9z", 3), "`\"...\xC3\xA9z\"`"); // not inside the two bytes of U+00E9 + EXPECT_EQ(ReportSkip(std::string(50, 'x') + "a", std::string(50, 'x') + "b"), 30u); + EXPECT_EQ(ReportSkip("short a", "short b"), 0u); +} + +// The other oracle's directory holds the same corpus: a stale or partial one would misreport +TEST(ParityRegistry, OracleCorpusMatches) +{ + const auto& oracle = Oracle(); + if (oracle.dir.empty()) + { + GTEST_SKIP() << "JINJA2CPP_TEST_ORACLE_DIR is not set"; + } + for (const auto& problem : oracle.problems) + { + ADD_FAILURE() << problem; + } + std::set ids; + for (const auto& c : LoadCases()) + { + ids.insert(c.id); + EXPECT_TRUE(oracle.cases.count(c.id)) << c.id << " is missing from " << oracle.dir; + if (oracle.cases.count(c.id)) + { + EXPECT_EQ(oracle.cases.at(c.id)["template"], c.data["template"]) << c.id << " has another template in " << oracle.dir; + } + } + for (const auto& c : oracle.cases) + { + EXPECT_TRUE(ids.count(c.first)) << oracle.dir << " has case " << c.first << ", which the corpus does not"; + } +} + TEST(ParityRegistry, DivergencesAreWellFormed) { Divergences(); From 1f53fb87d9d483954bcb46187b350e7b3ac8e951 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 16:57:21 +0000 Subject: [PATCH 2/3] Oracle mode: verifier findings (0186 phase 1) generate.py --out no longer sweeps other *.json files from the directory and refuses expected/; report cells and strict-mode messages replace invalid UTF-8 and cut on a character boundary; a malformed oracle.json or index.json is a failure of OracleCorpusMatches rather than an exception at tear-down; clearer message for a venv without jinja2; README notes that ';' separates requirements. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AWNL7oJQzYFjjoGQPuDCF2 --- cmake/parity_oracle_generate.cmake | 3 ++ test/parity/README.md | 2 +- test/parity/generate.py | 5 ++- test/parity/parity_test.cpp | 61 +++++++++++++++++------------- 4 files changed, 42 insertions(+), 29 deletions(-) diff --git a/cmake/parity_oracle_generate.cmake b/cmake/parity_oracle_generate.cmake index 71b38802c..2255fc92e 100644 --- a/cmake/parity_oracle_generate.cmake +++ b/cmake/parity_oracle_generate.cmake @@ -42,6 +42,9 @@ endif () execute_process(COMMAND "${_python}" -c "import jinja2" RESULT_VARIABLE _result OUTPUT_QUIET ERROR_QUIET) if (NOT _result EQUAL 0) + if (REQUIREMENTS) + message(FATAL_ERROR "${_python} cannot import jinja2: JINJA2CPP_TEST_ORACLE_REQUIREMENTS must name jinja2") + endif () message(FATAL_ERROR "${_python} cannot import jinja2: install it there, or use JINJA2CPP_TEST_ORACLE=venv") endif () execute_process(COMMAND "${_python}" ${GENERATE} --out ${OUT} --any-version RESULT_VARIABLE _result) diff --git a/test/parity/README.md b/test/parity/README.md index 8c42cf9f5..f7f9da203 100644 --- a/test/parity/README.md +++ b/test/parity/README.md @@ -135,7 +135,7 @@ as an older 3.x, Jinja2's main branch or another Python, configure the build wit |---|---| | `bundled` (default) | `expected/`; nothing else runs and no Python is needed | | `python` | the jinja2 and markupsafe installed for `JINJA2CPP_TEST_ORACLE_PYTHON` (default: the Python 3 CMake finds) | -| `venv` | a venv in `/parity-oracle/venv`, made from `JINJA2CPP_TEST_ORACLE_PYTHON` and filled by pip from `JINJA2CPP_TEST_ORACLE_REQUIREMENTS` (`;`-separated, e.g. `jinja2==3.1.2;markupsafe==2.1.5` or `jinja2 @ git+https://github.com/pallets/jinja@main`) | +| `venv` | a venv in `/parity-oracle/venv`, made from `JINJA2CPP_TEST_ORACLE_PYTHON` and filled by pip from `JINJA2CPP_TEST_ORACLE_REQUIREMENTS` (`;`-separated, e.g. `jinja2==3.1.2;markupsafe==2.1.5` or `jinja2 @ git+https://github.com/pallets/jinja@main`; a `;` always separates requirements, so environment markers do not fit) | ```bash cmake -S . -B build -DJINJA2CPP_TEST_ORACLE=venv -DJINJA2CPP_TEST_ORACLE_REQUIREMENTS='jinja2==3.1.2' diff --git a/test/parity/generate.py b/test/parity/generate.py index 4d732b10f..d2db68978 100644 --- a/test/parity/generate.py +++ b/test/parity/generate.py @@ -229,6 +229,8 @@ def main(): f"pip install jinja2=={JINJA2_VERSION}") out_dir = args.out.resolve() if args.out else HERE / "expected" + if args.out and out_dir == (HERE / "expected").resolve(): + sys.exit("--out names expected/; run generate.py without --out to regenerate it") out_dir.mkdir(parents=True, exist_ok=True) stale, total = [], 0 wanted = set() @@ -259,7 +261,8 @@ def main(): stale.append("index.json") else: index_path.write_text(index, encoding="utf-8") - for orphan in sorted(p.name for p in out_dir.glob("*.json") if p.name not in wanted): + # Only expected/ is swept: a --out directory may hold other files, and index.json lists the areas + for orphan in [] if args.out else sorted(p.name for p in out_dir.glob("*.json") if p.name not in wanted): if args.check: stale.append(orphan) else: diff --git a/test/parity/parity_test.cpp b/test/parity/parity_test.cpp index 4ee760328..52b19db80 100644 --- a/test/parity/parity_test.cpp +++ b/test/parity/parity_test.cpp @@ -707,35 +707,42 @@ const OracleRun& Oracle() { return r; } - const auto versions = ReadFile(r.dir + "/oracle.json"); - if (versions.empty()) + try { - r.problems.push_back("missing " + r.dir + "/oracle.json; run test/parity/generate.py --out " + r.dir + " --any-version"); - } - else - { - r.versions = Json::parse(versions); - } - const auto index = ReadFile(r.dir + "/index.json"); - if (index.empty()) - { - r.problems.push_back("missing " + r.dir + "/index.json"); - return r; - } - const auto areas = Json::parse(index)["areas"].get>(); - for (const auto& area : areas) - { - const auto text = ReadFile(r.dir + "/" + area + ".json"); - if (text.empty()) + const auto versions = ReadFile(r.dir + "/oracle.json"); + if (versions.empty()) + { + r.problems.push_back("missing " + r.dir + "/oracle.json; run test/parity/generate.py --out " + r.dir + " --any-version"); + } + else { - r.problems.push_back("missing " + r.dir + "/" + area + ".json"); - continue; + r.versions = Json::parse(versions); } - auto doc = Json::parse(text); // named: a range-for over parse(text)["cases"] would outlive the document - for (auto& c : doc["cases"]) + const auto index = ReadFile(r.dir + "/index.json"); + if (index.empty()) { - r.cases.emplace(c["id"].get(), std::move(c)); + r.problems.push_back("missing " + r.dir + "/index.json"); + return r; } + const auto areas = Json::parse(index)["areas"].get>(); + for (const auto& area : areas) + { + const auto text = ReadFile(r.dir + "/" + area + ".json"); + if (text.empty()) + { + r.problems.push_back("missing " + r.dir + "/" + area + ".json"); + continue; + } + auto doc = Json::parse(text); // named: a range-for over parse(text)["cases"] would outlive the document + for (auto& c : doc["cases"]) + { + r.cases.emplace(c["id"].get(), std::move(c)); + } + } + } + catch (const std::exception& ex) // a malformed file: a clear failure in OracleCorpusMatches + { + r.problems.push_back("cannot read " + r.dir + ": " + ex.what()); } return r; }(); @@ -831,7 +838,7 @@ std::string BundledPin(const std::string& package) std::string OracleVersion(const char* key) { const auto& v = Oracle().versions; - return v.contains(key) ? v[key].get() : "?"; + return v.is_object() && v.contains(key) && v[key].is_string() ? v[key].get() : "?"; } void WriteOracleReport() @@ -915,8 +922,8 @@ bool JudgedByAllowList(const std::string& id, const Json& c, const Result* cpp) if (oracle.strict && verdict != OracleVerdict::TracksOracle) { ADD_FAILURE() << "JINJA2CPP_TEST_ORACLE_STRICT: " << id << " is " << OracleVerdictName(verdict) << "\n template: " << c["template"].dump() - << "\n oracle: " << Json(PythonText(found->second)).dump() << "\n bundled: " << Json(PythonText(c)).dump() - << "\n c++: " << Json(CppText(cpp)).dump(); + << "\n oracle: " << ReportCell(PythonText(found->second)) << "\n bundled: " << ReportCell(PythonText(c)) + << "\n c++: " << ReportCell(CppText(cpp)); } return false; } From a984a03326997a9d73fe2eb3cc0d7b435d25861b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 09:31:33 +0000 Subject: [PATCH 3/3] Oracle mode: Codex findings (0186 phase 1) The venv stamp records the base interpreter, so a new JINJA2CPP_TEST_ORACLE_PYTHON rebuilds the venv; OracleCorpusMatches compares every rendering input (context, env, templates), not only the template, so a directory generated before a case changed fails instead of reporting deltas. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AWNL7oJQzYFjjoGQPuDCF2 --- cmake/parity_oracle_generate.cmake | 8 +++++--- test/parity/parity_test.cpp | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmake/parity_oracle_generate.cmake b/cmake/parity_oracle_generate.cmake index 2255fc92e..e0e3f4ef2 100644 --- a/cmake/parity_oracle_generate.cmake +++ b/cmake/parity_oracle_generate.cmake @@ -2,7 +2,7 @@ # parity corpus with the other oracle into OUT. # cmake -DPYTHON= -DGENERATE= -DOUT= [-DREQUIREMENTS=] -P parity_oracle_generate.cmake # With REQUIREMENTS, PYTHON is the base of a venv in OUT/venv that pip fills from that file. The venv -# is rebuilt when the file changes; delete OUT/venv to pick up a new commit of a branch requirement. +# is rebuilt when the file or PYTHON changes; delete OUT/venv to pick up a new commit of a branch requirement. foreach (_var PYTHON GENERATE OUT) if (NOT ${_var}) message(FATAL_ERROR "parity_oracle_generate.cmake needs -D${_var}=...") @@ -17,11 +17,13 @@ if (REQUIREMENTS) else () set(_python ${_venv}/bin/python) endif () + # The base interpreter and the requirements the venv was made from: a change to either rebuilds it set(_stamp ${_venv}/jinja2cpp-requirements.txt) + file(READ ${REQUIREMENTS} _wanted) + string(PREPEND _wanted "# base: ${PYTHON}\n") set(_fresh FALSE) if (EXISTS ${_stamp} AND EXISTS ${_python}) file(READ ${_stamp} _installed) - file(READ ${REQUIREMENTS} _wanted) if (_installed STREQUAL _wanted) set(_fresh TRUE) endif () @@ -36,7 +38,7 @@ if (REQUIREMENTS) if (NOT _result EQUAL 0) message(FATAL_ERROR "pip install -r ${REQUIREMENTS} failed (${_result}); the venv oracle needs network access to the package index") endif () - configure_file(${REQUIREMENTS} ${_stamp} COPYONLY) + file(WRITE ${_stamp} "${_wanted}") endif () endif () diff --git a/test/parity/parity_test.cpp b/test/parity/parity_test.cpp index 6a71e141a..a37585356 100644 --- a/test/parity/parity_test.cpp +++ b/test/parity/parity_test.cpp @@ -1325,7 +1325,11 @@ TEST(ParityRegistry, OracleCorpusMatches) EXPECT_TRUE(oracle.cases.count(c.id)) << c.id << " is missing from " << oracle.dir; if (oracle.cases.count(c.id)) { - EXPECT_EQ(oracle.cases.at(c.id)["template"], c.data["template"]) << c.id << " has another template in " << oracle.dir; + // Every rendering input, so a directory generated before a case changed is caught + for (const char* input : { "template", "context", "env", "templates" }) + { + EXPECT_EQ(oracle.cases.at(c.id).value(input, Json()), c.data.value(input, Json())) << c.id << " has another " << input << " in " << oracle.dir; + } } } for (const auto& c : oracle.cases)