Render Hugging Face chat templates as transformers does: ConfigureForChatTemplates (0360)

.github/workflows/benchmark.yml

@@ -129,6 +129,9 @@ jobs:
         # run those cases with other settings; it counts its own cases then, and the new ones show as new
         base_cases=bench/cases
         grep -q 'settings.json' ../base/bench/jinja2cpp_bench.cpp || base_cases=../base/bench/cases
+        # Likewise a base driver that does not know "transformers" (docs/tasks/0360) would render the
+        # hf_* chat-template cases in plain Jinja2, or fail on them
+        grep -q '"transformers"' ../base/bench/jinja2cpp_bench.cpp || base_cases=../base/bench/cases
         python bench/count.py --bench build-base/bench/jinja2cpp_bench --cases-dir "$base_cases" --cache-sim --out base.json
         python bench/count.py --bench build/bench/jinja2cpp_bench --baseline base.json --cache-sim \
           --threshold 0.03 | tee -a "$GITHUB_STEP_SUMMARY"

.github/workflows/linux-build.yml

@@ -62,11 +62,13 @@ jobs:
   # One Debug row builds the borrowed-cell trace hooks (`celltrace`, JINJA2CPP_CELL_TRACE) and
   # checks the traces of the cell and module tests against tla/CellTrace.tla with TLC
   # (docs/tasks/0338 step B). It is a check after the tests, not an axis.
+  # One row builds without the chat-template preset (`chat: 'OFF'`, JINJA2CPP_WITH_CHAT_TEMPLATES,
+  # docs/tasks/0360): no chat_template.h, no test/chat_templates suite, the core the same.
   # JSON bindings are package components (docs/tasks/0311): these jobs test the nlohmann_json
   # binding and the JsonTraits contract on toy traits (JINJA2CPP_TEST_BINDINGS default);
   # bindings.yml tests every binding on PRs that touch them, and nightly.
   linux-build:
-    name: C++${{ matrix.std }} ${{ matrix.compiler }} ${{ matrix.config }} shared=${{ matrix.shared }}${{ matrix.noderef == 'OFF' && ' noderef=OFF' || '' }}${{ matrix.bigint && format(' bigint={0}', matrix.bigint) || '' }}
+    name: C++${{ matrix.std }} ${{ matrix.compiler }} ${{ matrix.config }} shared=${{ matrix.shared }}${{ matrix.noderef == 'OFF' && ' noderef=OFF' || '' }}${{ matrix.bigint && format(' bigint={0}', matrix.bigint) || '' }}${{ matrix.chat == 'OFF' && ' chat=OFF' || '' }}
     runs-on: ubuntu-24.04
 
     strategy:
@@ -79,7 +81,7 @@ jobs:
           - { std: 17, compiler: clang-20, config: Release, shared: 'OFF', package: 23, bigint: boost }
           - { std: 20, compiler: gcc-12,   config: RelWithDebInfo, shared: 'ON', abi: true }
           - { std: 20, compiler: gcc-14,   config: Release, shared: 'ON',  bigint: boost }
-          - { std: 20, compiler: clang-18, config: Debug,   shared: 'OFF' }
+          - { std: 20, compiler: clang-18, config: Debug,   shared: 'OFF', chat: 'OFF' }
           - { std: 20, compiler: clang-20, config: Debug,   shared: 'OFF', noderef: 'OFF', bigint: boost }
           - { std: 23, compiler: gcc-12,   config: Debug,   shared: 'OFF' }
           - { std: 23, compiler: gcc-14,   config: RelWithDebInfo, shared: 'ON', noderef: 'OFF', package: 17, abi: true }
@@ -98,7 +100,7 @@ jobs:
 
     - uses: ./.github/actions/setup-ccache
       with:
-        key: linux-${{ matrix.compiler }}-${{ matrix.config }}-${{ matrix.std }}-${{ matrix.shared }}-${{ matrix.noderef || 'default' }}-${{ matrix.bigint || 'none' }}
+        key: linux-${{ matrix.compiler }}-${{ matrix.config }}-${{ matrix.std }}-${{ matrix.shared }}-${{ matrix.noderef || 'default' }}-${{ matrix.bigint || 'none' }}-${{ matrix.chat || 'ON' }}
 
     - name: Configure
       run: |
@@ -111,6 +113,7 @@ jobs:
           -DJINJA2CPP_NODEREF_CHECKS=${{ matrix.noderef == 'OFF' && 'OFF' || '' }} \
           -DJINJA2CPP_WITH_BIGINT=${{ matrix.bigint || 'none' }} \
           -DJINJA2CPP_CELL_TRACE=${{ matrix.celltrace && 'ON' || 'OFF' }} \
+          -DJINJA2CPP_WITH_CHAT_TEMPLATES=${{ matrix.chat || 'ON' }} \
           "-DCMAKE_CXX_FLAGS=${{ startsWith(matrix.compiler, 'gcc') && '-Wno-error=maybe-uninitialized' || '' }}"
 
     - name: Build

CMakeLists.txt

@@ -59,6 +59,7 @@ option(JINJA2CPP_BUILD_FUZZERS "Build the libFuzzer targets in fuzz/ (clang only
 option(JINJA2CPP_STRICT_WARNINGS "Enable additional warnings and treat them as errors" ON)
 set(JINJA2CPP_NODEREF_CHECKS "" CACHE STRING "Checks of the handles that link a parse tree: OFF, ON (Seal and cross-template links), FULL (every access); empty picks FULL for Debug, sanitizer and fuzz builds, else ON")
 set_property(CACHE JINJA2CPP_NODEREF_CHECKS PROPERTY STRINGS "" ON FULL OFF)
+option(JINJA2CPP_WITH_CHAT_TEMPLATES "Build and install jinja2cpp/chat_template.h: ConfigureForChatTemplates renders Hugging Face chat templates as transformers does (docs/tasks/0360)" ON)
 option(JINJA2CPP_WITH_UNICODE_NAMES "Decode \\N{name} escapes in string literals with the Unicode name table (about 450 KB of data)" ON)
 option(JINJA2CPP_CELL_TRACE "Log the borrowed-cell protocol's events to the file the JINJA2CPP_CELL_TRACE environment variable names, for tla/check.py --trace (docs/tasks/0338); a test build only" OFF)
 option(JINJA2CPP_CHECK_SLOTS "Check every name read from a slot against its lookup by name (always on without NDEBUG)" OFF)
@@ -263,6 +264,15 @@ set(JINJA2CPP_SOURCES
 )
 
 CollectSources(PublicSources PublicHeaders ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include)
+# The chat-template preset is its header and one source file; what it switches on is in the core
+# either way, so the switch changes no other part of the library or its ABI (docs/tasks/0360)
+if (JINJA2CPP_WITH_CHAT_TEMPLATES)
+    list(APPEND JINJA2CPP_SOURCES src/chat_template.cpp)
+    set(JINJA2CPP_HAS_CHAT_TEMPLATES 1)
+else ()
+    list(REMOVE_ITEM PublicHeaders include/jinja2cpp/chat_template.h)
+    set(JINJA2CPP_HAS_CHAT_TEMPLATES 0)
+endif ()
 
 add_library(${LIB_TARGET_NAME} ${LIB_LINK_TYPE}
     ${JINJA2CPP_SOURCES}
@@ -284,10 +294,19 @@ endif()
 string(REGEX REPLACE "^([0-9]+\\.[0-9]+).*" "\\1" JINJA2CPP_ORACLE_PYTHON "${JINJA2CPP_ORACLE_PYTHON_LINE}")
 string(REGEX REPLACE "^[Jj]inja2==([^ #;]+).*" "\\1" JINJA2CPP_ORACLE_JINJA2 "${JINJA2CPP_ORACLE_JINJA2_LINE}")
 configure_file(cmake/version.h.in ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h @ONLY)
+# jinja2cpp/features.h: the build options a user's code can test, and the transformers version the
+# chat-template preset follows, read from the pin of its test oracle
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/test/chat_templates/requirements.txt)
+file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/test/chat_templates/requirements.txt JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE REGEX "^transformers==[0-9]")
+if(NOT JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE)
+    message(FATAL_ERROR "Cannot read the transformers pin from test/chat_templates/requirements.txt")
+endif()
+string(REGEX REPLACE "^transformers==([^ #;]+).*" "\\1" JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS "${JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE}")
+configure_file(cmake/features.h.in ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/features.h @ONLY)
 
 target_sources(${LIB_TARGET_NAME}
     PUBLIC FILE_SET HEADERS
-    FILES ${PublicHeaders} ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h
+    FILES ${PublicHeaders} ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/features.h
     BASE_DIRS include ${JINJA2CPP_GENERATED_INCLUDE_DIR}
 )
 
@@ -522,8 +541,10 @@ if (JINJA2CPP_BUILD_TESTS)
         test/v1_names_test.cpp
         test/value_ptr_test.cpp
         test/parity/parity_test.cpp
-        test/chat_templates/chat_template_test.cpp
     )
+    if (JINJA2CPP_WITH_CHAT_TEMPLATES)
+        list(APPEND JINJA2CPP_TEST_SOURCES test/chat_templates/chat_template_test.cpp test/chat_template_preset_test.cpp)
+    endif ()
     add_executable(jinja2cpp_tests ${JINJA2CPP_TEST_SOURCES})
     target_compile_options(jinja2cpp_tests PRIVATE ${JINJA2CPP_TARGET_CXX_FLAGS})
     # The parity corpus reads its cases with nlohmann::json and copies each context through the
@@ -581,7 +602,9 @@ if (JINJA2CPP_BUILD_TESTS)
     # Differential corpus against Python Jinja2, see test/parity/README.md
     add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*)
     # Hugging Face chat templates against transformers, see test/chat_templates/README.md
-    add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*)
+    if (JINJA2CPP_WITH_CHAT_TEMPLATES)
+        add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*)
+    endif ()
     # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076)
     if (CMAKE_NM AND NOT MSVC)
         add_test(NAME jinja2cpp_abi_namespace

bench/README.md

@@ -23,12 +23,14 @@ render (docs/tasks/0119):
 | Case | What it is |
 |---|---|
 | `chat_llama`, `chat_qwen`, `chat_mistral` | LLM chat templates in the style of Llama 3.1, Qwen 2.5 (ChatML) and Mistral v3, with tool definitions and tool calls (`tojson`, `trim`, slicing, `namespace`, `loop.index0` lookbehind), over a 23-message conversation; `trim_blocks` and `lstrip_blocks` as Hugging Face sets them |
+| `hf_qwen3`, `hf_gpt_oss`, `hf_deepseek_v31`, `hf_mistral_small` | real Hugging Face chat templates (copies from `test/chat_templates/templates/`, Apache-2.0 and MIT model licenses) with the suite's `tools` conversation, in transformers' environment: `"transformers": true` in `settings.json` (`ConfigureForChatTemplates` in C++, the same `ImmutableSandboxedEnvironment` setup in `python_bench.py`; other engines skip them) |
 | `html_autoescape` | a shop page with `autoescape` on: escaped user text, `safe`, `e`, imported macros for form fields |
 | `config_file` | an nginx-style config built from many small `include`s and imported macros, `dictsort`, `trim_blocks` |
 | `mitsuhiko_table_wide` | `mitsuhiko_table` with `TemplateW`, so the two compare narrow and wide rendering |
 
 Real Hugging Face templates run in an environment whose `tojson` neither sorts keys nor
-escapes HTML characters; these cases use Jinja2's own `tojson`, which both engines share.
+escapes HTML characters; the `chat_*` cases use Jinja2's own `tojson`, which every engine shares,
+and the `hf_*` cases the real templates in transformers' environment (docs/tasks/0360).
 
 Each case gives two benchmarks: `Load/<case>` parses `main.j2`, `Render/<case>` renders
 the parsed template to a string. To add a workload, add a directory; both drivers pick it

bench/cases/hf_deepseek_v31/data.json

@@ -0,0 +1,75 @@
+{
+ "messages": [
+  {
+   "role": "system",
+   "content": "You can call tools."
+  },
+  {
+   "role": "user",
+   "content": "Weather in Paris?"
+  },
+  {
+   "role": "assistant",
+   "content": "",
+   "tool_calls": [
+    {
+     "type": "function",
+     "id": "call_0001",
+     "function": {
+      "name": "get_weather",
+      "arguments": {
+       "city": "Paris",
+       "unit": "celsius"
+      }
+     }
+    }
+   ]
+  },
+  {
+   "role": "tool",
+   "name": "get_weather",
+   "tool_call_id": "call_0001",
+   "content": "{\"temp\": 18, \"sky\": \"cloudy\"}"
+  },
+  {
+   "role": "assistant",
+   "content": "It is 18°C and cloudy in Paris."
+  },
+  {
+   "role": "user",
+   "content": "Thanks"
+  }
+ ],
+ "tools": [
+  {
+   "type": "function",
+   "function": {
+    "name": "get_weather",
+    "description": "Get the current weather in a city.",
+    "parameters": {
+     "type": "object",
+     "properties": {
+      "city": {
+       "type": "string",
+       "description": "City name, e.g. Paris"
+      },
+      "unit": {
+       "type": "string",
+       "enum": [
+        "celsius",
+        "fahrenheit"
+       ]
+      }
+     },
+     "required": [
+      "city"
+     ]
+    }
+   }
+  }
+ ],
+ "add_generation_prompt": true,
+ "documents": null,
+ "bos_token": "<|bos|>",
+ "eos_token": "<|eos|>"
+}

bench/cases/hf_deepseek_v31/main.j2

@@ -0,0 +1,71 @@
+{% if not add_generation_prompt is defined -%}
+  {%- set add_generation_prompt = false -%}
+{%- endif -%}
+{%- if not thinking is defined -%}
+  {%- if enable_thinking is defined -%}
+    {%- set thinking = enable_thinking -%}
+    {%- else -%}
+    {%- set thinking = false -%}
+  {%- endif -%}
+{%- endif -%}
+{%- set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) -%}
+{%- for message in messages -%}
+  {%- if message['role'] == 'system' -%}
+    {%- if ns.is_first_sp -%}
+      {%- set ns.system_prompt = ns.system_prompt + message['content'] -%}
+      {%- set ns.is_first_sp = false -%}
+      {%- else -%}
+      {%- set ns.system_prompt = ns.system_prompt + '
+
+' + message['content'] -%}
+    {%- endif -%}
+  {%- endif -%}
+{%- endfor -%}{{ bos_token }}{{ ns.system_prompt }}
+{%- for message in messages -%}
+  {%- if message['role'] == 'user' -%}
+    {%- set ns.is_tool = false -%}
+    {%- set ns.is_first = false -%}
+    {%- set ns.is_last_user = true -%}{{'<｜User｜>' + message['content']}}
+  {%- endif -%}
+  {%- if message['role'] == 'assistant' and message['tool_calls'] -%}
+    {%- if ns.is_last_user -%}{{'<｜Assistant｜><think></think>'}}
+    {%- endif -%}
+    {%- set ns.is_last_user = false -%}
+    {%- set ns.is_first = false -%}
+    {%- set ns.is_tool = false -%}
+    {%- for tool in message['tool_calls'] -%}
+      {%- if not ns.is_first -%}
+        {%- if not message['content'] -%}{{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] | tojson + '<｜tool▁call▁end｜>'}}
+          {%- else -%}{{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] | tojson + '<｜tool▁call▁end｜>'}}
+        {%- endif -%}
+        {%- set ns.is_first = true -%}
+        {%- else -%}{{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments'] | tojson + '<｜tool▁call▁end｜>'}}
+      {%- endif -%}
+    {%- endfor -%}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}
+  {%- endif -%}
+  {%- if message['role'] == 'assistant' and not message['tool_calls'] -%}
+    {%- if ns.is_last_user -%}{{'<｜Assistant｜>'}}
+      {%- if message['prefix'] is defined and message['prefix'] and thinking -%}{{'<think>'}}
+        {%- else -%}{{'<think></think>'}}
+      {%- endif -%}
+    {%- endif -%}
+    {%- set ns.is_last_user = false -%}
+    {%- if ns.is_tool -%}{{message['content'] + '<｜end▁of▁sentence｜>'}}
+      {%- set ns.is_tool = false -%}
+      {%- else -%}
+      {%- set content = message['content'] -%}
+      {%- if '</think>' in content -%}
+        {%- set content = content.split('</think>', 1)[1] -%}
+      {%- endif -%}{{content + '<｜end▁of▁sentence｜>'}}
+    {%- endif -%}
+  {%- endif -%}
+  {%- if message['role'] == 'tool' -%}
+    {%- set ns.is_last_user = false -%}
+    {%- set ns.is_tool = true -%}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}
+  {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool -%}{{'<｜Assistant｜>'}}
+  {%- if not thinking -%}{{'<think></think>'}}
+  {%- else -%}{{'<think>'}}
+  {%- endif -%}
+{%- endif %}

bench/cases/hf_deepseek_v31/settings.json

@@ -0,0 +1 @@
+{"transformers": true}

bench/cases/hf_gpt_oss/data.json

@@ -0,0 +1,75 @@
+{
+ "messages": [
+  {
+   "role": "system",
+   "content": "You can call tools."
+  },
+  {
+   "role": "user",
+   "content": "Weather in Paris?"
+  },
+  {
+   "role": "assistant",
+   "content": "",
+   "tool_calls": [
+    {
+     "type": "function",
+     "id": "call_0001",
+     "function": {
+      "name": "get_weather",
+      "arguments": {
+       "city": "Paris",
+       "unit": "celsius"
+      }
+     }
+    }
+   ]
+  },
+  {
+   "role": "tool",
+   "name": "get_weather",
+   "tool_call_id": "call_0001",
+   "content": "{\"temp\": 18, \"sky\": \"cloudy\"}"
+  },
+  {
+   "role": "assistant",
+   "content": "It is 18°C and cloudy in Paris."
+  },
+  {
+   "role": "user",
+   "content": "Thanks"
+  }
+ ],
+ "tools": [
+  {
+   "type": "function",
+   "function": {
+    "name": "get_weather",
+    "description": "Get the current weather in a city.",
+    "parameters": {
+     "type": "object",
+     "properties": {
+      "city": {
+       "type": "string",
+       "description": "City name, e.g. Paris"
+      },
+      "unit": {
+       "type": "string",
+       "enum": [
+        "celsius",
+        "fahrenheit"
+       ]
+      }
+     },
+     "required": [
+      "city"
+     ]
+    }
+   }
+  }
+ ],
+ "add_generation_prompt": true,
+ "documents": null,
+ "bos_token": "<|bos|>",
+ "eos_token": "<|eos|>"
+}