Train: #725 (0186 ph1), #735 (0330 item 0, 0376), #736 (0362) on master 2fc3fac

CMakeLists.txt

@@ -580,6 +580,8 @@ if (JINJA2CPP_BUILD_TESTS)
 
     # Differential corpus against Python Jinja2, see test/parity/README.md
     add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*)
+    # The same suite against another Python Jinja2, when JINJA2CPP_TEST_ORACLE asks (docs/tasks/0186)
+    include(cmake/parity_oracle.cmake)
     # Hugging Face chat templates against transformers, see test/chat_templates/README.md
     add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*)
     # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076)

cmake/parity_oracle.cmake

@@ -0,0 +1,73 @@
+# Compare the parity suite with a Python Jinja2 other than the bundled one (docs/tasks/0186,
+# test/parity/README.md "Other oracles"). Included from the test block of CMakeLists.txt.
+#
+#   bundled (default): the checked-in test/parity/expected/; nothing here runs and no Python is needed
+#   python:  test/parity/generate.py runs under JINJA2CPP_TEST_ORACLE_PYTHON (default: the
+#            Python3 interpreter CMake finds) with whatever jinja2 and markupsafe it has
+#   venv:    a venv in the build directory gets JINJA2CPP_TEST_ORACLE_REQUIREMENTS from pip
+#
+# The other oracle's expectations are generated at test time, by the fixture
+# jinja2cpp_parity_oracle_generate, so configure needs no network and a failing pip is a failing
+# test. jinja2cpp_parity_oracle then runs the narrow suite against them and writes
+# parity-oracle/report.md in the build directory.
+set(JINJA2CPP_TEST_ORACLE bundled CACHE STRING "Python Jinja2 the parity suite compares against: bundled | python | venv")
+set_property(CACHE JINJA2CPP_TEST_ORACLE PROPERTY STRINGS bundled python venv)
+set(JINJA2CPP_TEST_ORACLE_PYTHON "" CACHE FILEPATH "Interpreter for JINJA2CPP_TEST_ORACLE=python, and the base of the venv (default: find_package(Python3))")
+set(JINJA2CPP_TEST_ORACLE_REQUIREMENTS "" CACHE STRING
+    "pip requirements for JINJA2CPP_TEST_ORACLE=venv, ';'-separated, e.g. 'jinja2==3.1.2;markupsafe==2.1.5' or 'jinja2 @ git+https://github.com/pallets/jinja@main'")
+
+if (JINJA2CPP_TEST_ORACLE STREQUAL "bundled")
+    return()
+endif ()
+if (NOT JINJA2CPP_TEST_ORACLE MATCHES "^(python|venv)$")
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE must be bundled, python or venv, not '${JINJA2CPP_TEST_ORACLE}'")
+endif ()
+if (JINJA2CPP_TEST_ORACLE STREQUAL "venv" AND JINJA2CPP_TEST_ORACLE_REQUIREMENTS STREQUAL "")
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE=venv needs JINJA2CPP_TEST_ORACLE_REQUIREMENTS, e.g. 'jinja2==3.1.2'")
+endif ()
+
+if (JINJA2CPP_TEST_ORACLE_PYTHON)
+    set(_oracle_python "${JINJA2CPP_TEST_ORACLE_PYTHON}")
+else ()
+    find_package(Python3 3.10 REQUIRED COMPONENTS Interpreter)
+    set(_oracle_python "${Python3_EXECUTABLE}")
+endif ()
+execute_process(COMMAND "${_oracle_python}" -c "import sys; print('%d.%d' % sys.version_info[:2])"
+    OUTPUT_VARIABLE _oracle_python_version OUTPUT_STRIP_TRAILING_WHITESPACE RESULT_VARIABLE _oracle_python_result)
+if (NOT _oracle_python_result EQUAL 0)
+    message(FATAL_ERROR "JINJA2CPP_TEST_ORACLE: cannot run ${_oracle_python}")
+endif ()
+# The Unicode tables in src/ come from the pinned Python, so Unicode cases may differ for a
+# reason that is not a Jinja2C++ bug
+if (NOT _oracle_python_version VERSION_EQUAL JINJA2CPP_ORACLE_PYTHON)
+    message(WARNING "JINJA2CPP_TEST_ORACLE: the oracle runs Python ${_oracle_python_version}, but Jinja2C++ follows Python "
+        "${JINJA2CPP_ORACLE_PYTHON} (.python-version); cases that depend on the Python version (Unicode data, textwrap, "
+        "\\N{} names) can show up in the report as deltas that are not Jinja2 changes")
+endif ()
+message(STATUS "Parity oracle: ${JINJA2CPP_TEST_ORACLE}, Python ${_oracle_python_version} (${_oracle_python})")
+
+set(_oracle_dir ${CMAKE_CURRENT_BINARY_DIR}/parity-oracle)
+set(_oracle_requirements "")
+if (JINJA2CPP_TEST_ORACLE STREQUAL "venv")
+    # A file, so the list reaches pip without quoting trouble; the generate step reinstalls when it changes
+    set(_oracle_requirements ${_oracle_dir}/requirements.txt)
+    string(REPLACE ";" "\n" _oracle_requirement_lines "${JINJA2CPP_TEST_ORACLE_REQUIREMENTS}")
+    file(WRITE ${_oracle_requirements}.in "${_oracle_requirement_lines}\n")
+    configure_file(${_oracle_requirements}.in ${_oracle_requirements} COPYONLY)
+endif ()
+
+add_test(NAME jinja2cpp_parity_oracle_generate
+    COMMAND ${CMAKE_COMMAND}
+        -DPYTHON=${_oracle_python}
+        -DREQUIREMENTS=${_oracle_requirements}
+        -DGENERATE=${CMAKE_CURRENT_SOURCE_DIR}/test/parity/generate.py
+        -DOUT=${_oracle_dir}
+        -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/parity_oracle_generate.cmake)
+set_tests_properties(jinja2cpp_parity_oracle_generate PROPERTIES FIXTURES_SETUP jinja2cpp_parity_oracle)
+
+# The narrow suite and the registry checks; the wide suite is held to the narrow result, not
+# to an oracle, so it has nothing to add
+add_test(NAME jinja2cpp_parity_oracle COMMAND jinja2cpp_tests --gtest_filter=Parity/*:ParityRegistry.*)
+set_tests_properties(jinja2cpp_parity_oracle PROPERTIES
+    FIXTURES_REQUIRED jinja2cpp_parity_oracle
+    ENVIRONMENT "JINJA2CPP_TEST_ORACLE_DIR=${_oracle_dir}")

cmake/parity_oracle_generate.cmake

@@ -0,0 +1,55 @@
+# The setup step of the jinja2cpp_parity_oracle fixture (cmake/parity_oracle.cmake): renders the
+# parity corpus with the other oracle into OUT.
+#   cmake -DPYTHON=<interpreter> -DGENERATE=<generate.py> -DOUT=<dir> [-DREQUIREMENTS=<file>] -P parity_oracle_generate.cmake
+# With REQUIREMENTS, PYTHON is the base of a venv in OUT/venv that pip fills from that file. The venv
+# is rebuilt when the file or PYTHON changes; delete OUT/venv to pick up a new commit of a branch requirement.
+foreach (_var PYTHON GENERATE OUT)
+    if (NOT ${_var})
+        message(FATAL_ERROR "parity_oracle_generate.cmake needs -D${_var}=...")
+    endif ()
+endforeach ()
+
+set(_python "${PYTHON}")
+if (REQUIREMENTS)
+    set(_venv ${OUT}/venv)
+    if (WIN32)
+        set(_python ${_venv}/Scripts/python.exe)
+    else ()
+        set(_python ${_venv}/bin/python)
+    endif ()
+    # The base interpreter and the requirements the venv was made from: a change to either rebuilds it
+    set(_stamp ${_venv}/jinja2cpp-requirements.txt)
+    file(READ ${REQUIREMENTS} _wanted)
+    string(PREPEND _wanted "# base: ${PYTHON}\n")
+    set(_fresh FALSE)
+    if (EXISTS ${_stamp} AND EXISTS ${_python})
+        file(READ ${_stamp} _installed)
+        if (_installed STREQUAL _wanted)
+            set(_fresh TRUE)
+        endif ()
+    endif ()
+    if (NOT _fresh)
+        message(STATUS "Creating ${_venv} with ${PYTHON}")
+        execute_process(COMMAND "${PYTHON}" -m venv --clear ${_venv} RESULT_VARIABLE _result)
+        if (NOT _result EQUAL 0)
+            message(FATAL_ERROR "python -m venv failed (${_result})")
+        endif ()
+        execute_process(COMMAND "${_python}" -m pip install --disable-pip-version-check -r ${REQUIREMENTS} RESULT_VARIABLE _result)
+        if (NOT _result EQUAL 0)
+            message(FATAL_ERROR "pip install -r ${REQUIREMENTS} failed (${_result}); the venv oracle needs network access to the package index")
+        endif ()
+        file(WRITE ${_stamp} "${_wanted}")
+    endif ()
+endif ()
+
+execute_process(COMMAND "${_python}" -c "import jinja2" RESULT_VARIABLE _result OUTPUT_QUIET ERROR_QUIET)
+if (NOT _result EQUAL 0)
+    if (REQUIREMENTS)
+        message(FATAL_ERROR "${_python} cannot import jinja2: JINJA2CPP_TEST_ORACLE_REQUIREMENTS must name jinja2")
+    endif ()
+    message(FATAL_ERROR "${_python} cannot import jinja2: install it there, or use JINJA2CPP_TEST_ORACLE=venv")
+endif ()
+execute_process(COMMAND "${_python}" ${GENERATE} --out ${OUT} --any-version RESULT_VARIABLE _result)
+if (NOT _result EQUAL 0)
+    message(FATAL_ERROR "generate.py --out ${OUT} failed (${_result})")
+endif ()

docs/release-2.0.md

@@ -147,11 +147,12 @@ cannot happen before a tag exists, or it has no end state a release could wait f
 | Publishing wheels to PyPI | publication part of 0248 | Like the other registries, it follows the tag. The drop-in module itself, its wheels built in CI and its test-suite pass rate are in scope (0248) |
 | Parity cases found after the freeze | — | The corpus keeps growing; new divergences found during the RC window are filed for 2.x unless they are crashes, leaks or silent wrong values, which block (G8) |
 
-**Deliberate divergences** stay documented, not bugs (14 corpus lines): integer overflow
+**Deliberate divergences** stay documented, not bugs (13 corpus lines): integer overflow
 raises in the default build (0015; big integers are opt-in through 0178's backend),
 names captured by a closure that outlives its frame read as undefined (0165, 6 lines),
 `select`/`map`/`reject` results can be iterated twice (0101, 2 lines), `%c` of a lone
-surrogate in wide templates (0106), a call block returning a dict prints it (0156).
+surrogate in wide templates (0106). Outside the corpus, a call block escapes a callable's
+plain string result under autoescape, which Jinja2 writes raw (0362, Ruslan 2026-11-02).
 Corpus on master `b0fb0ef`: 1929 cases, 1886 match.
 
 ## 6. Confirmed facts for this revision

docs/tasks/0186-python-oracle-target.md

@@ -2,7 +2,7 @@
 status: in-progress
 priority: medium
 area: parity
-touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in]
+touches: [test/parity/requirements.txt, test/parity/generate.py, .github/workflows/parity-expectations.yml, docs/parity.md, .python-version, cmake/version.h.in, cmake/parity_oracle.cmake, cmake/parity_oracle_generate.cmake, test/parity/parity_test.cpp, test/parity/README.md]
 ---
 # Which Python and Jinja2 the parity oracle targets, and how users choose
 
@@ -50,13 +50,18 @@ Decisions: O2 yes (version macros, before the G1 freeze), O3 no, O4 report-only
   `JINJA2CPP_ORACLE_PYTHON` and `JINJA2CPP_ORACLE_JINJA2`; `docs/parity.md` has the policy
   and the one-PR bump checklist ("Oracle pin"). This answers the policy question and the
   "where is it stated" question.
-- **Phase 1, open.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`,
-  `_PYTHON`, `_REQUIREMENTS`), `generate.py --out DIR --any-version`, oracle-directory mode
-  in `parity_test.cpp` (env `JINJA2CPP_TEST_ORACLE_DIR`) that classifies each differing case
-  `tracks-bundled`, `tracks-oracle` or `neither`, writes `report.md`, and a task for the
-  corpus gap on code points changed after Unicode 15.0. Must land before 0355: the Python
-  module's parity proof (0355) waits on phase 1 and reuses its `generate.py --out` harness
-  and delta classification instead of a second one.
+- **Phase 1, done in this PR.** `cmake/parity_oracle.cmake` (`JINJA2CPP_TEST_ORACLE=bundled|python|venv`,
+  `_PYTHON`, `_REQUIREMENTS`; included from the test block) registers the CTest fixture
+  `jinja2cpp_parity_oracle_generate` (`cmake/parity_oracle_generate.cmake`: makes the venv,
+  runs `generate.py --out DIR --any-version`, which also writes `oracle.json`) and the test
+  `jinja2cpp_parity_oracle`. `parity_test.cpp` reads `JINJA2CPP_TEST_ORACLE_DIR`, judges a case
+  by the allow-list where the oracles agree, otherwise records it as `tracks-bundled`,
+  `tracks-oracle`, `neither` or `not-run` (failing only under `JINJA2CPP_TEST_ORACLE_STRICT=1`),
+  and writes `report.md`. Documented in test/parity/README.md, "Other oracles". The section 8
+  check holds: Jinja2 3.1.2 (MarkupSafe 2.1.5, Python 3.12) reports the 8 cases of plan
+  section 2 plus `string_filters.m168`, all `tracks-bundled`; 3.0.3 on Python 3.13 reports 22.
+  The Unicode-corpus gap is 0372. 0355 reuses `generate.py --out/--any-version`,
+  `JINJA2CPP_TEST_ORACLE_DIR` and the verdicts rather than a second harness.
 - **Phase 2, open.** `.github/workflows/parity-oracles.yml` modelled on `bindings.yml`
   (paths-filtered PRs plus nightly, actions pinned by SHA): Jinja2 main, 3.0.x, Python
   3.13/3.14 rows with the pinned Jinja2.

docs/tasks/0330-gate-reports-newly-outlined-frames.md

@@ -47,6 +47,10 @@ the gate already has.
 0. Force-inline `InternalValue`'s copy assignment where `RenderLoopInSlots` unpacks
    loop variables, or the whole operator if no case grows. Check that `Render/dict_ops`
    stays at master's count when `statements.cpp` changes.
+   *Done* (with 0376): `InternalValue::CopyFrom` is the copy assignment forced inline,
+   and `Slot::Bind` calls it for an lvalue. Forcing `operator=` itself inline grew
+   `BindArg` and `FillDefaultArgs` out of `ParseCallParams` (`Render/for_range` +0.10%),
+   so `operator=` stays a plain inline call to it.
 1. `count.py --keep-profiles DIR` keeps each case's callgrind output instead of
    deleting it.
 2. A new `bench/frames.py base_dir head_dir` compares the two sets per case. It lists

docs/tasks/0362-host-object-python-errors.md

@@ -1,5 +1,5 @@
 ---
-status: open
+status: in-progress
 priority: low
 area: parity
 depends: [0352]
@@ -32,3 +32,9 @@ engine object shows the same gap.
 
 **Done when.** Each bullet renders or fails as Python does on the toy object, narrow and
 wide.
+
+**Decisions (Ruslan, 2026-11-02, #736).** A call block whose call returns something other
+than a string fails as in Jinja2, so `statements.call_block_dict` (0156) is no longer a
+deliberate divergence. A plain string result stays escaped under autoescape, where Jinja2
+writes it raw. This is a deliberate divergence that keeps a callable's data escaped; Markup
+results (macros) are written as they are.

docs/tasks/0372-unicode-version-corpus-cases.md

@@ -0,0 +1,31 @@
+---
+status: open
+priority: low
+area: parity
+depends: [0186]
+touches: [test/parity/cases/literals.py, test/parity/cases/string_filters.py, test/parity/expected/literals.json, test/parity/expected/string_filters.json]
+---
+# Corpus cases on code points whose Unicode properties changed after 15.0
+
+**Problem.** Jinja2C++'s generated tables (`src/unicode_printable.h`, `unicode_tables.h`,
+`unicode_case.h`, `unicode_names.h`) follow the pinned Python 3.12, which ships Unicode 15.0.
+Python 3.13 ships 15.1 and 3.14 ships 16.0. The 0186 oracle runs (`JINJA2CPP_TEST_ORACLE`,
+test/parity/README.md "Other oracles") show the Python axis only where the corpus has a case,
+and the corpus has none on a code point that 15.1 or 16.0 assigned or changed: under Python
+3.13 the measured deltas were stdlib code (`textwrap`, `\N{}` name rules), never table data
+(0186 plan, section 10). A table change between Python versions is therefore invisible to the
+oracle rows of phase 2.
+
+For example, U+2EBF0 (first ideograph of CJK Extension I, new in 15.1): Python 3.12 gives
+`repr` `'\U0002ebf0'`, `isalpha()` False, no name; Python 3.13 gives `'𮯰'`, True and
+`CJK UNIFIED IDEOGRAPH-2EBF0`.
+
+**Proposal.** Add a few cases, in the areas that own each behaviour, on code points new or
+changed in 15.1 and 16.0: `repr` of a string holding one (printable table), `upper`/`lower`/
+`title` on a cased letter added in 16.0 (case tables), a `\N{...}` escape naming one (names
+table), and `int()`/`wordcount` style uses of `\d`/`\w` if 16.0 changed a digit or word
+property. They match under the pinned 3.12 (no allow-list line), and the 3.13/3.14 oracle rows
+then report them as `tracks-bundled`, which is the record that the Python axis is measured.
+
+**Done when.** The cases are in the corpus, the bundled suite matches them, and an oracle run
+under Python 3.13 lists them in `report.md`.

docs/tasks/0376-same-string-getter-round-trip.md

@@ -1,5 +1,6 @@
 ---
-status: open
+status: in-progress
+pr: https://github.com/jinja2cpp/Jinja2Cpp/pull/735
 priority: low
 area: perf
 touches: [src/value_visitors.h#SameStringGetter]

docs/tasks/0380-markup-escape-copy-assignment.md

@@ -0,0 +1,31 @@
+---
+status: open
+priority: low
+area: perf
+touches: [src/markup.h#MarkupEscape]
+shares: [src/internal_value.h]
+---
+# MarkupEscape calls InternalValue's copy assignment out of line
+
+**Problem.** `MarkupEscape`'s escaping lambda default-constructs its result and then assigns
+`val` to it when there is nothing to escape. GCC emits that assignment as an out-of-line
+`InternalValue::operator=(const InternalValue&) [clone .isra.0]`. On `Render/html_autoescape`
+this costs about 3,000 Ir a render (0.25%) on master after #735.
+
+#735 (0330 item 0) tried building the result in place in each branch
+(`InternalValue result(val)`, then `SetMarkup`). That recovered the 0.25% on
+`html_autoescape`, but it shifted GCC's inlining in other translation units that include
+markup.h. `Render/strings` went +0.39%, `Module/keep` +0.38% and `Render/filters` +0.25%
+against the change without it, so #735 left it out.
+
+**Proposal.**
+- Build the result in place without changing what the including translation units inline,
+  for example by moving `MarkupEscape` out of line into one .cpp.
+- Or call `CopyFrom` only on the no-escape branch.
+- Measure every row with `count.py`. The whole-TU inlining shifts are the risk here, not
+  the escaping itself.
+
+**Done when.**
+- `Render/html_autoescape` recovers at least 0.2% against the master base.
+- Callgrind shows no out-of-line copy assignment from a `MarkupEscape` frame.
+- No row gets slower by more than 0.1%.

docs/tasks/0381-host-object-operators.md

@@ -0,0 +1,30 @@
+---
+status: open
+priority: low
+area: parity
+depends: [0352]
+touches: [src/host_object.h, src/value_visitors.h#BinaryMathOperation]
+shares: [test/host_object_test.cpp]
+---
+# Host objects: arithmetic and ordering operators
+
+**Problem.** `IHostObject` (0352) has `Equals`, but no entry for the other operators.
+The engine therefore answers every `+ - * / // % **`, every ordering (`< <= > >=`) and
+`odd`/`even` (`x % 2`, since 0362) on a host object with Python's TypeError:
+"unsupported operand type(s)" or "'<' not supported between instances". That is correct
+for a plain object. The Python module (0248) also passes objects that define these
+methods, though: `decimal.Decimal`, `datetime` values with `timedelta`, `fractions`,
+numpy scalars, and user classes with `__lt__` for `sort` and `max`. Python Jinja2 calls
+those methods.
+
+**Proposal.** Give the seam one binary-operator entry. It would take the operator and the
+other operand, and say which side the object is on, so that `__radd__` and friends work.
+It returns nullopt for NotImplemented. The engine then tries the other operand's
+reflected method, then raises the TypeError it raises today. Ordering inside `sort`,
+`min`, `max` and `dictsort` should use the same entry (`__lt__`). Results come back as
+values the template can read, which may be other host objects.
+
+**Done when.** On the toy (`test/host_object_test.cpp`), an object with `__add__`,
+`__radd__`, `__mod__` and `__lt__` renders as an equivalent Python class does under the
+pinned Jinja2, narrow and wide. This covers `x + 1`, `1 + x`, `x is odd`, `[x, y]|sort`
+and `x < y`. An object without them keeps today's errors.

docs/tasks/0382-urlencode-of-loop.md

@@ -0,0 +1,23 @@
+---
+status: open
+priority: low
+area: parity
+depends: []
+touches: [src/string_converter_filter.cpp#ApplyUrlEncode]
+---
+# `loop|urlencode` iterates the rest of the loop
+
+**Problem.** Jinja2's `do_urlencode` treats any `abc.Iterable` as a sequence of pairs,
+and `LoopContext` is iterable: it yields the items the loop has not reached yet, and
+advances the loop. So `{% for i in [1] %}[{{ loop|urlencode }}]{% endfor %}` renders
+`[]` in Python. On the last item nothing is left, and on an earlier item the rest is
+unpacked as pairs, which usually fails. Jinja2C++ quotes `str(loop)` and renders
+`[%3CLoopContext%201/1%3E]`. The corpus pins this as `filters.urlencode_loop`, kind
+`output`. Found by the 0362 verifier.
+
+**Proposal.** Iterate the loop's remaining items in `ApplyUrlEncode`, as a `for` over
+`loop` does (see the 0304 lines in statements.txt for how such a loop advances the outer
+loop). Then unpack them as for any iterable.
+
+**Done when.** `filters.urlencode_loop` matches, narrow and wide, and a case with items
+left behaves as Python does.