Parity suite against another Python Jinja2: JINJA2CPP_TEST_ORACLE (0186 phase 1)

CMakeLists.txt

@@ -580,6 +580,8 @@ if (JINJA2CPP_BUILD_TESTS)
 
     # Differential corpus against Python Jinja2, see test/parity/README.md
     add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*)
+    # The same suite against another Python Jinja2, when JINJA2CPP_TEST_ORACLE asks (docs/tasks/0186)
+    include(cmake/parity_oracle.cmake)
     # Hugging Face chat templates against transformers, see test/chat_templates/README.md
     add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*)
     # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076)

cmake/parity_oracle.cmake

@@ -0,0 +1,73 @@
+# Compare the parity suite with a Python Jinja2 other than the bundled one (docs/tasks/0186,
+# test/parity/README.md "Other oracles"). Included from the test block of CMakeLists.txt.
+#
+#   bundled (default): the checked-in test/parity/expected/; nothing here runs and no Python is needed
+#   python:  test/parity/generate.py runs under JINJA2CPP_TEST_ORACLE_PYTHON (default: the
+#            Python3 interpreter CMake finds) with whatever jinja2 and markupsafe it has
+#   venv:    a venv in the build directory gets JINJA2CPP_TEST_ORACLE_REQUIREMENTS from pip
+#
+# The other oracle's expectations are generated at test time, by the fixture
+# jinja2cpp_parity_oracle_generate, so configure needs no network and a failing pip is a failing
+# test. jinja2cpp_parity_oracle then runs the narrow suite against them and writes
+# parity-oracle/report.md in the build directory.
+set(JINJA2CPP_TEST_ORACLE bundled CACHE STRING "Python Jinja2 the parity suite compares against: bundled | python | venv")
+set_property(CACHE JINJA2CPP_TEST_ORACLE PROPERTY STRINGS bundled python venv)
+set(JINJA2CPP_TEST_ORACLE_PYTHON "" CACHE FILEPATH "Interpreter for JINJA2CPP_TEST_ORACLE=python, and the base of the venv (default: find_package(Python3))")
+set(JINJA2CPP_TEST_ORACLE_REQUIREMENTS "" CACHE STRING
+    "pip requirements for JINJA2CPP_TEST_ORACLE=venv, ';'-separated, e.g. 'jinja2==3.1.2;markupsafe==2.1.5' or 'jinja2 @ git+https://github.com/pallets/jinja@main'")
+
+if (JINJA2CPP_TEST_ORACLE STREQUAL "bundled")
+    return()
+endif ()
+if (NOT JINJA2CPP_TEST_ORACLE MATCHES "^(python|venv)$")
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE must be bundled, python or venv, not '${JINJA2CPP_TEST_ORACLE}'")
+endif ()
+if (JINJA2CPP_TEST_ORACLE STREQUAL "venv" AND JINJA2CPP_TEST_ORACLE_REQUIREMENTS STREQUAL "")
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE=venv needs JINJA2CPP_TEST_ORACLE_REQUIREMENTS, e.g. 'jinja2==3.1.2'")
+endif ()
+
+if (JINJA2CPP_TEST_ORACLE_PYTHON)
+    set(_oracle_python "${JINJA2CPP_TEST_ORACLE_PYTHON}")
+else ()
+    find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter)
+    set(_oracle_python "${Python3_EXECUTABLE}")
+endif ()
+execute_process(COMMAND "${_oracle_python}" -c "import sys; print('%d.%d' % sys.version_info[:2])"
+    OUTPUT_VARIABLE _oracle_python_version OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _oracle_python_result)
+if (NOT _oracle_python_result EQUAL 0)
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE: cannot run ${_oracle_python}")
+endif ()
+# The Unicode tables in src/ come from the pinned Python, so Unicode cases may differ for a
+# reason that is not a Jinja2C++ bug
+if (NOT _oracle_python_version VERSION_EQUAL JINJA2CPP_ORACLE_PYTHON)
+    message(WARNING "JINJA2CPP_TEST_ORACLE: the oracle runs Python ${_oracle_python_version}, but Jinja2C++ follows Python "
+        "${JINJA2CPP_ORACLE_PYTHON} (.python-version); cases that depend on the Python version (Unicode data, textwrap, "
+        "\\N{} names) can show up in the report as deltas that are not Jinja2 changes")
+endif ()
+message(STATUS "Parity oracle: ${JINJA2CPP_TEST_ORACLE}, Python ${_oracle_python_version} (${_oracle_python})")
+
+set(_oracle_dir ${CMAKE_CURRENT_BINARY_DIR}/parity-oracle)
+set(_oracle_requirements "")
+if (JINJA2CPP_TEST_ORACLE STREQUAL "venv")
+    # A file, so the list reaches pip without quoting trouble; the generate step reinstalls when it changes
+    set(_oracle_requirements ${_oracle_dir}/requirements.txt)
+    string(REPLACE ";" "\n" _oracle_requirement_lines "${JINJA2CPP_TEST_ORACLE_REQUIREMENTS}")
+    file(WRITE ${_oracle_requirements}.in "${_oracle_requirement_lines}\n")
+    configure_file(${_oracle_requirements}.in ${_oracle_requirements} COPYONLY)
+endif ()
+
+add_test(NAME jinja2cpp_parity_oracle_generate
+    COMMAND ${CMAKE_COMMAND}
+        -DPYTHON=${_oracle_python}
+        -DREQUIREMENTS=${_oracle_requirements}
+        -DGENERATE=${CMAKE_CURRENT_SOURCE_DIR}/test/parity/generate.py
+        -DOUT=${_oracle_dir}
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/parity_oracle_generate.cmake)
+set_tests_properties(jinja2cpp_parity_oracle_generate PROPERTIES FIXTURES_SETUP jinja2cpp_parity_oracle)
+
+# The narrow suite and the registry checks; the wide suite is held to the narrow result, not
+# to an oracle, so it has nothing to add
+add_test(NAME jinja2cpp_parity_oracle COMMAND jinja2cpp_tests --gtest_filter=Parity/*:ParityRegistry.*)
+set_tests_properties(jinja2cpp_parity_oracle PROPERTIES
+    FIXTURES_REQUIRED jinja2cpp_parity_oracle
+    ENVIRONMENT "JINJA2CPP_TEST_ORACLE_DIR=${_oracle_dir}")

cmake/parity_oracle_generate.cmake

@@ -0,0 +1,53 @@
+# The setup step of the jinja2cpp_parity_oracle fixture (cmake/parity_oracle.cmake): renders the
+# parity corpus with the other oracle into OUT.
+#   cmake -DPYTHON=<interpreter> -DGENERATE=<generate.py> -DOUT=<dir> [-DREQUIREMENTS=<file>] -P parity_oracle_generate.cmake
+# With REQUIREMENTS, PYTHON is the base of a venv in OUT/venv that pip fills from that file. The venv
+# is rebuilt when the file changes; delete OUT/venv to pick up a new commit of a branch requirement.
+foreach (_var PYTHON GENERATE OUT)
+    if (NOT ${_var})
+        message(FATAL_ERROR "parity_oracle_generate.cmake needs -D${_var}=...")
+    endif ()
+endforeach ()
+
+set(_python "${PYTHON}")
+if (REQUIREMENTS)
+    set(_venv ${OUT}/venv)
+    if (WIN32)
+        set(_python ${_venv}/Scripts/python.exe)
+    else ()
+        set(_python ${_venv}/bin/python)
+    endif ()
+    set(_stamp ${_venv}/jinja2cpp-requirements.txt)
+    set(_fresh FALSE)
+    if (EXISTS ${_stamp} AND EXISTS ${_python})
+        file(READ ${_stamp} _installed)
+        file(READ ${REQUIREMENTS} _wanted)
+        if (_installed STREQUAL _wanted)
+            set(_fresh TRUE)
+        endif ()
+    endif ()
+    if (NOT _fresh)
+        message(STATUS "Creating ${_venv} with ${PYTHON}")
+        execute_process(COMMAND "${PYTHON}" -m venv --clear ${_venv} RESULT_VARIABLE _result)
+        if (NOT _result EQUAL 0)
+            message(FATAL_ERROR "python -m venv failed (${_result})")
+        endif ()
+        execute_process(COMMAND "${_python}" -m pip install --disable-pip-version-check -r ${REQUIREMENTS} RESULT_VARIABLE _result)
+        if (NOT _result EQUAL 0)
+            message(FATAL_ERROR "pip install -r ${REQUIREMENTS} failed (${_result}); the venv oracle needs network access to the package index")
+        endif ()
+        configure_file(${REQUIREMENTS} ${_stamp} COPYONLY)
+    endif ()
+endif ()
+
+execute_process(COMMAND "${_python}" -c "import jinja2" RESULT_VARIABLE _result OUTPUT_QUIET ERROR_QUIET)
+if (NOT _result EQUAL 0)
+    if (REQUIREMENTS)
+        message(FATAL_ERROR "${_python} cannot import jinja2: JINJA2CPP_TEST_ORACLE_REQUIREMENTS must name jinja2")
+    endif ()
+    message(FATAL_ERROR "${_python} cannot import jinja2: install it there, or use JINJA2CPP_TEST_ORACLE=venv")
+endif ()
+execute_process(COMMAND "${_python}" ${GENERATE} --out ${OUT} --any-version RESULT_VARIABLE _result)
+if (NOT _result EQUAL 0)
+    message(FATAL_ERROR "generate.py --out ${OUT} failed (${_result})")
+endif ()

docs/tasks/0186-python-oracle-target.md

@@ -2,7 +2,7 @@
 status: in-progress
 priority: medium
 area: parity
-touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in]
+touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in, cmake/parity_oracle.cmake, cmake/parity_oracle_generate.cmake, test/parity/parity_test.cpp, test/parity/README.md]
 ---
 # Which Python and Jinja2 the parity oracle targets, and how users choose
 
@@ -50,13 +50,18 @@ Decisions: O2 yes (version macros, before the G1 freeze), O3 no, O4 report-only
   `JINJA2CPP_ORACLE_PYTHON` and `JINJA2CPP_ORACLE_JINJA2`; `docs/parity.md` has the policy
   and the one-PR bump checklist ("Oracle pin"). This answers the policy question and the
   "where is it stated" question.
-- **Phase 1, open.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`,
-  `_PYTHON`, `_REQUIREMENTS`), `generate.py --out DIR --any-version`, oracle-directory mode
-  in `parity_test.cpp` (env `JINJA2CPP_TEST_ORACLE_DIR`) that classifies each differing case
-  `tracks-bundled`, `tracks-oracle` or `neither`, writes `report.md`, and a task for the
-  corpus gap on code points changed after Unicode 15.0. Must land before 0355: the Python
-  module's parity proof (0355) waits on phase 1 and reuses its `generate.py --out` harness
-  and delta classification instead of a second one.
+- **Phase 1, done in this PR.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`,
+  `_PYTHON`, `_REQUIREMENTS`; included from the test block) registers the CTest fixture
+  `jinja2cpp_parity_oracle_generate` (`cmake/parity_oracle_generate.cmake`: makes the venv,
+  runs `generate.py --out DIR --any-version`, which also writes `oracle.json`) and the test
+  `jinja2cpp_parity_oracle`. `parity_test.cpp` reads `JINJA2CPP_TEST_ORACLE_DIR`, judges a case
+  by the allow-list where the oracles agree, otherwise records it as `tracks-bundled`,
+  `tracks-oracle`, `neither` or `not-run` (failing only under `JINJA2CPP_TEST_ORACLE_STRICT=1`),
+  and writes `report.md`. Documented in test/parity/README.md, "Other oracles". The section 8
+  check holds: Jinja2 3.1.2 (MarkupSafe 2.1.5, Python 3.12) reports the 8 cases of plan
+  section 2 plus `string_filters.m168`, all `tracks-bundled`; 3.0.3 on Python 3.13 reports 22.
+  The Unicode-corpus gap is 0372. 0355 reuses `generate.py --out/--any-version`,
+  `JINJA2CPP_TEST_ORACLE_DIR` and the verdicts rather than a second harness.
 - **Phase 2, open.** `.github/workflows/parity-oracles.yml` modelled on `bindings.yml`
   (paths-filtered PRs plus nightly, actions pinned by SHA): Jinja2 main, 3.0.x, Python
   3.13/3.14 rows with the pinned Jinja2.

docs/tasks/0372-unicode-version-corpus-cases.md

@@ -0,0 +1,31 @@
+---
+status: open
+priority: low
+area: parity
+depends: [0186]
+touches: [test/parity/cases/literals.py, test/parity/cases/string_filters.py, test/parity/expected/literals.json, test/parity/expected/string_filters.json]
+---
+# Corpus cases on code points whose Unicode properties changed after 15.0
+
+**Problem.** Jinja2C++'s generated tables (`src/unicode_printable.h`, `unicode_tables.h`,
+`unicode_case.h`, `unicode_names.h`) follow the pinned Python 3.12, which ships Unicode 15.0.
+Python 3.13 ships 15.1 and 3.14 ships 16.0. The 0186 oracle runs (`JINJA2CPP_TEST_ORACLE`,
+test/parity/README.md "Other oracles") show the Python axis only where the corpus has a case,
+and the corpus has none on a code point that 15.1 or 16.0 assigned or changed: under Python
+3.13 the measured deltas were stdlib code (`textwrap`, `\N{}` name rules), never table data
+(0186 plan, section 10). A table change between Python versions is therefore invisible to the
+oracle rows of phase 2.
+
+For example, U+2EBF0 (first ideograph of CJK Extension I, new in 15.1): Python 3.12 gives
+`repr` `'\U0002ebf0'`, `isalpha()` False, no name; Python 3.13 gives `'𮯰'`, True and
+`CJK UNIFIED IDEOGRAPH-2EBF0`.
+
+**Proposal.** Add a few cases, in the areas that own each behaviour, on code points new or
+changed in 15.1 and 16.0: `repr` of a string holding one (printable table), `upper`/`lower`/
+`title` on a cased letter added in 16.0 (case tables), a `\N{...}` escape naming one (names
+table), and `int()`/`wordcount` style uses of `\d`/`\w` if 16.0 changed a digit or word
+property. They match under the pinned 3.12 (no allow-list line), and the 3.13/3.14 oracle rows
+then report them as `tracks-bundled`, which is the record that the Python axis is measured.
+
+**Done when.** The cases are in the corpus, the bundled suite matches them, and an oracle run
+under Python 3.13 lists them in `report.md`.

test/parity/README.md

@@ -138,3 +138,50 @@ When two branches add cases to the same area, take either side of the conflict i
 Keep cases small and aimed at one feature. Print booleans as `'T' if ... else 'F'` and
 iterate lists instead of printing them when the case is about something else, so that the
 repr gaps (task 0012) do not mask the behaviour under test.
+
+## Other oracles
+
+The checked-in `expected/` comes from the pinned pair (`.python-version`, `requirements.txt`;
+docs/parity.md, "Oracle pin"). To see how Jinja2C++ compares with another Python Jinja2, such
+as an older 3.x, Jinja2's main branch or another Python, configure the build with
+`JINJA2CPP_TEST_ORACLE` (docs/tasks/0186):
+
+| Value | What the oracle is |
+|---|---|
+| `bundled` (default) | `expected/`; nothing else runs and no Python is needed |
+| `python` | the jinja2 and markupsafe installed for `JINJA2CPP_TEST_ORACLE_PYTHON` (default: the Python 3 CMake finds) |
+| `venv` | a venv in `<build>/parity-oracle/venv`, made from `JINJA2CPP_TEST_ORACLE_PYTHON` and filled by pip from `JINJA2CPP_TEST_ORACLE_REQUIREMENTS` (`;`-separated, e.g. `jinja2==3.1.2;markupsafe==2.1.5` or `jinja2 @ git+https://github.com/pallets/jinja@main`; a `;` always separates requirements, so environment markers do not fit) |
+
+```bash
+cmake -S . -B build -DJINJA2CPP_TEST_ORACLE=venv -DJINJA2CPP_TEST_ORACLE_REQUIREMENTS='jinja2==3.1.2'
+ctest --test-dir build -R parity_oracle --output-on-failure
+cat build/parity-oracle/report.md
+```
+
+`jinja2cpp_parity_oracle_generate` (a CTest fixture, so configure needs no Python packages
+and no network) runs `generate.py --out <build>/parity-oracle --any-version`, which writes the
+other oracle's expectations there with `oracle.json` naming its Python, Jinja2 and MarkupSafe.
+`venv` needs network at test time; the venv is rebuilt when the requirements change, and
+deleting it picks up a new commit of a branch requirement. Supported oracles: Python >= 3.10,
+Jinja2 >= 3.0. Configure warns when the oracle's Python is not the pinned one, because the
+Unicode tables and Python's own behaviour (`textwrap`, `\N{}` names) follow the pinned one.
+
+`jinja2cpp_parity_oracle` runs the narrow suite with `JINJA2CPP_TEST_ORACLE_DIR` naming that
+directory (set it by hand to compare against any `generate.py --out` directory; the test binary
+needs no rebuild). Each case is judged by whether the two oracles agree (same output, or both
+an error):
+
+- **they agree**: judged as without an oracle, allow-list included; a regression fails;
+- **they differ** (a Jinja2 or Python version delta): reported, not failed. The verdict is
+  `tracks-bundled` (Jinja2C++ matches the pinned oracle), `tracks-oracle` (matches the other
+  one), `neither`, or `not-run` (a listed crash, an unsupported option, or a build without the
+  Unicode name table). With `JINJA2CPP_TEST_ORACLE_STRICT=1` every verdict but
+  `tracks-oracle` fails.
+
+The run writes `report.md` next to the expectations: the two oracles' versions, the counts,
+and one row per differing case with the bundled result, the other oracle's and Jinja2C++'s.
+Against Jinja2 3.1.2 with MarkupSafe 2.1.5 on Python 3.12 that is 9 rows, all
+`tracks-bundled`; against 3.0.3 on Python 3.13, 22. `ParityRegistry.OracleCorpusMatches`
+fails when the directory does not hold the same cases as the corpus (a stale generation).
+The wide suite is held to the narrow result rather than to an oracle, so the oracle test does
+not run it.