From 51befffe1e09cfee5051d8226304a705b3536ba1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 08:50:16 +0000 Subject: [PATCH 1/8] Chat-template preset: ConfigureForChatTemplates and the transformers dialect (0360) --- CMakeLists.txt | 29 ++- cmake/features.h.in | 13 ++ docs/api-2.0.md | 26 +++ include/jinja2cpp/chat_template.h | 60 ++++++ include/jinja2cpp/config.h | 2 + src/chat_template.cpp | 117 +++++++++++ src/dialect.h | 29 +++ src/expression_evaluator.cpp | 6 + src/filter_factories.cpp | 23 +- src/filters.cpp | 2 +- src/filters.h | 7 +- src/load_settings.h | 6 +- src/recursion_guard.h | 10 +- src/render_context.h | 4 + src/serialize_filters.cpp | 233 ++++++++++++++------- src/template_env.cpp | 28 ++- src/template_env_impl.h | 9 + src/template_impl.h | 5 +- src/template_parser.cpp | 43 ++++ src/template_parser.h | 8 +- src/value_methods.cpp | 37 +++- src/value_methods.h | 6 + test/chat_template_preset_test.cpp | 176 ++++++++++++++++ test/chat_templates/chat_template_test.cpp | 60 +----- test/chat_templates/divergences.txt | 138 ------------ 25 files changed, 789 insertions(+), 288 deletions(-) create mode 100644 cmake/features.h.in create mode 100644 include/jinja2cpp/chat_template.h create mode 100644 src/chat_template.cpp create mode 100644 src/dialect.h create mode 100644 test/chat_template_preset_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 72c35b959..b97c1341d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ option(JINJA2CPP_BUILD_FUZZERS "Build the libFuzzer targets in fuzz/ (clang only option(JINJA2CPP_STRICT_WARNINGS "Enable additional warnings and treat them as errors" ON) set(JINJA2CPP_NODEREF_CHECKS "" CACHE STRING "Checks of the handles that link a parse tree: OFF, ON (Seal and cross-template links), FULL (every access); empty picks FULL for Debug, sanitizer and fuzz builds, else ON") set_property(CACHE JINJA2CPP_NODEREF_CHECKS PROPERTY STRINGS "" ON FULL OFF) +option(JINJA2CPP_WITH_CHAT_TEMPLATES "Build and install jinja2cpp/chat_template.h: ConfigureForChatTemplates renders Hugging Face chat templates as transformers does (docs/tasks/0360)" ON) option(JINJA2CPP_WITH_UNICODE_NAMES "Decode \\N{name} escapes in string literals with the Unicode name table (about 450 KB of data)" ON) option(JINJA2CPP_CELL_TRACE "Log the borrowed-cell protocol's events to the file the JINJA2CPP_CELL_TRACE environment variable names, for tla/check.py --trace (docs/tasks/0338); a test build only" OFF) option(JINJA2CPP_CHECK_SLOTS "Check every name read from a slot against its lookup by name (always on without NDEBUG)" OFF) @@ -263,6 +264,15 @@ set(JINJA2CPP_SOURCES ) CollectSources(PublicSources PublicHeaders ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include) +# The chat-template preset is its header and one source file; what it switches on is in the core +# either way, so the switch changes no other part of the library or its ABI (docs/tasks/0360) +if (JINJA2CPP_WITH_CHAT_TEMPLATES) + list(APPEND JINJA2CPP_SOURCES src/chat_template.cpp) + set(JINJA2CPP_HAS_CHAT_TEMPLATES 1) +else () + list(REMOVE_ITEM PublicHeaders include/jinja2cpp/chat_template.h) + set(JINJA2CPP_HAS_CHAT_TEMPLATES 0) +endif () add_library(${LIB_TARGET_NAME} ${LIB_LINK_TYPE} ${JINJA2CPP_SOURCES} @@ -284,10 +294,19 @@ endif() string(REGEX REPLACE "^([0-9]+\\.[0-9]+).*" "\\1" JINJA2CPP_ORACLE_PYTHON "${JINJA2CPP_ORACLE_PYTHON_LINE}") string(REGEX REPLACE "^[Jj]inja2==([^ #;]+).*" "\\1" JINJA2CPP_ORACLE_JINJA2 "${JINJA2CPP_ORACLE_JINJA2_LINE}") configure_file(cmake/version.h.in ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h @ONLY) +# jinja2cpp/features.h: the build options a user's code can test, and the transformers version the +# chat-template preset follows, read from the pin of its test oracle +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/test/chat_templates/requirements.txt) +file(STRINGS ${CMAKE_CURRENT_SOURCE_DIR}/test/chat_templates/requirements.txt JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE REGEX "^transformers==[0-9]") +if(NOT JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE) + message(FATAL_ERROR "Cannot read the transformers pin from test/chat_templates/requirements.txt") +endif() +string(REGEX REPLACE "^transformers==([^ #;]+).*" "\\1" JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS "${JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS_LINE}") +configure_file(cmake/features.h.in ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/features.h @ONLY) target_sources(${LIB_TARGET_NAME} PUBLIC FILE_SET HEADERS - FILES ${PublicHeaders} ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h + FILES ${PublicHeaders} ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/version.h ${JINJA2CPP_GENERATED_INCLUDE_DIR}/jinja2cpp/features.h BASE_DIRS include ${JINJA2CPP_GENERATED_INCLUDE_DIR} ) @@ -522,8 +541,10 @@ if (JINJA2CPP_BUILD_TESTS) test/v1_names_test.cpp test/value_ptr_test.cpp test/parity/parity_test.cpp - test/chat_templates/chat_template_test.cpp ) + if (JINJA2CPP_WITH_CHAT_TEMPLATES) + list(APPEND JINJA2CPP_TEST_SOURCES test/chat_templates/chat_template_test.cpp test/chat_template_preset_test.cpp) + endif () add_executable(jinja2cpp_tests ${JINJA2CPP_TEST_SOURCES}) target_compile_options(jinja2cpp_tests PRIVATE ${JINJA2CPP_TARGET_CXX_FLAGS}) # The parity corpus reads its cases with nlohmann::json and copies each context through the @@ -581,7 +602,9 @@ if (JINJA2CPP_BUILD_TESTS) # Differential corpus against Python Jinja2, see test/parity/README.md add_test(NAME jinja2cpp_parity COMMAND jinja2cpp_tests --gtest_filter=Parity*) # Hugging Face chat templates against transformers, see test/chat_templates/README.md - add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*) + if (JINJA2CPP_WITH_CHAT_TEMPLATES) + add_test(NAME jinja2cpp_chat_templates COMMAND jinja2cpp_tests --gtest_filter=ChatTemplate*) + endif () # Every jinja2 symbol the library defines is in the inline ABI namespace v2 (docs/tasks/0076) if (CMAKE_NM AND NOT MSVC) add_test(NAME jinja2cpp_abi_namespace diff --git a/cmake/features.h.in b/cmake/features.h.in new file mode 100644 index 000000000..f89c643b7 --- /dev/null +++ b/cmake/features.h.in @@ -0,0 +1,13 @@ +// Generated by CMake from the build options in CMakeLists.txt; edit cmake/features.h.in instead. +#ifndef JINJA2CPP_FEATURES_H +#define JINJA2CPP_FEATURES_H + +// Macros, not an enum: users test them in #if. +// NOLINTBEGIN(modernize-macro-to-enum) +// 1 when the library has jinja2cpp/chat_template.h (CMake option JINJA2CPP_WITH_CHAT_TEMPLATES), else 0 +#define JINJA2CPP_HAS_CHAT_TEMPLATES @JINJA2CPP_HAS_CHAT_TEMPLATES@ +// The transformers version whose apply_chat_template the chat-template preset follows (test/chat_templates) +#define JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS "@JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS@" +// NOLINTEND(modernize-macro-to-enum) + +#endif // JINJA2CPP_FEATURES_H diff --git a/docs/api-2.0.md b/docs/api-2.0.md index a1eea536e..194347650 100644 --- a/docs/api-2.0.md +++ b/docs/api-2.0.md @@ -279,6 +279,28 @@ Every environment starts with the built-in one: none by default, `cpp_int` with sample for writing a backend over another library. Only int64 overflow calls the backend, so int64 arithmetic costs the same with or without one. +**Chat-template preset (task 0360, decided by Ruslan 2026-11-01).** `jinja2cpp/chat_template.h`: +`ConfigureForChatTemplates(TemplateEnv&, ChatTemplateOptions = {})` configures an environment as +transformers' `apply_chat_template` does: trim/lstrip blocks, loop controls, autoescape off, +`limits.maxRange` at most 100000, transformers' `tojson` (`json.dumps` with `ensure_ascii=False`, +keys in insertion order, no HTML escaping, `indent`/`separators`/`sort_keys`), the +`{% generation %}` tag (body rendered; no assistant mask), read-only lists and dicts +(`ImmutableSandboxedEnvironment`: `append`, `update`, ... are unsafe attributes) and the globals +`raise_exception` (`ErrorCode::TemplateRuntimeError`) and `strftime_now` (clock in +`ChatTemplateOptions::now`). Later knobs are new fields of `ChatTemplateOptions`, not new entry points. +The header and its one source file are built and installed only with the CMake option +`JINJA2CPP_WITH_CHAT_TEMPLATES` (default ON); `JINJA2CPP_HAS_CHAT_TEMPLATES` (generated +`jinja2cpp/features.h`, included by `config.h`) says whether they are, and +`JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS` names the transformers version the preset follows (the pin +of `test/chat_templates/requirements.txt`, which catches its drift). What the preset switches on is +an internal dialect of the core, always compiled and dormant unless the preset sets it, so the +option changes no other symbol or layout. Considered: a documented `AddGlobal`/`AddFilter` recipe +(a correct transformers `tojson` is too easy to get wrong, and the tag and immutable containers +need the core anyway) and an `ApplyChatTemplate(tpl, messages, ...)` mirror of transformers +(can be added after 2.0 without breaking this one). Known limits: it is not the rest of +`apply_chat_template` (the host builds `messages`, `tools`, `bos_token`, ...) and no security +sandbox beyond what it lists. + ### 3.4 Errors (`error_info.h`, `error_handler.h`) | Now | Problem | 2.0 | Migration | @@ -757,6 +779,10 @@ Recommend A now, C only if someone asks. R1 no allocator parameters in 2.0; R2 `bad_alloc` only for system OOM, budgets are `ResourceLimitExceeded`; R3 `Render(ostream&)` stays all-or-nothing; R4 output targets as `RenderTo` overloads (string, sink) plus the header-only `ContainerSink`. +8. **Chat-template preset** (0360, 3.3). *Decided by Ruslan 2026-11-01:* public, one entry point + `ConfigureForChatTemplates(TemplateEnv&, ChatTemplateOptions)` in its own header behind the CMake + option `JINJA2CPP_WITH_CHAT_TEMPLATES` (default ON) with a feature macro; the internals it enables + stay in the core, dormant otherwise. ## 8. Work breakdown diff --git a/include/jinja2cpp/chat_template.h b/include/jinja2cpp/chat_template.h new file mode 100644 index 000000000..daddbdbac --- /dev/null +++ b/include/jinja2cpp/chat_template.h @@ -0,0 +1,60 @@ +#ifndef JINJA2CPP_CHAT_TEMPLATE_H +#define JINJA2CPP_CHAT_TEMPLATE_H + +// Rendering Hugging Face chat templates as transformers' apply_chat_template does (docs/tasks/0360). +// Installed and built only with the CMake option JINJA2CPP_WITH_CHAT_TEMPLATES (default ON); +// JINJA2CPP_HAS_CHAT_TEMPLATES in config.h tells whether it is. + +#include "config.h" +#include "template_env.h" + +#include +#include + +#if !JINJA2CPP_HAS_CHAT_TEMPLATES +#error "Jinja2C++ was built without chat templates (JINJA2CPP_WITH_CHAT_TEMPLATES=OFF)" +#endif + +namespace jinja2 +{ +inline namespace v2 +{ + +//! Options of \ref ConfigureForChatTemplates. Later knobs (special tokens, say) are new fields +struct ChatTemplateOptions +{ + //! The clock `strftime_now` reads. Empty: `std::chrono::system_clock::now`. The time is shown in local time, as + //! Python's `datetime.now()` + std::function now; +}; + +/*! + * \brief Configures `env` as transformers' `apply_chat_template` configures its Jinja2 environment + * + * Tracks transformers \ref JINJA2CPP_CHAT_TEMPLATES_TRANSFORMERS. Templates loaded in `env` afterwards (call it before + * loading any) get: + * - the settings `trimBlocks`, `lstripBlocks` and `extensions.loopControls` on, `autoescape` off, and + * `limits.maxRange` at most 100000 (the sandbox's `MAX_RANGE`); the other settings stay as they are; + * - transformers' `tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False)` in place of Jinja2's: + * keys in insertion order, no HTML escaping, a plain string; + * - the `{% generation %}...{% endgeneration %}` tag, which renders its body (Jinja2C++ does not compute the + * assistant token mask); + * - read-only lists and dicts, as in `ImmutableSandboxedEnvironment`: `append`, `pop`, `update` and the other methods + * that change their receiver are unsafe attributes, undefined whose use fails with the sandbox's message; + * - the globals `raise_exception(message)`, which fails the render with ErrorCode::TemplateRuntimeError and the + * message, and `strftime_now(format)`, the time of `options.now` formatted as Python's `strftime` does in the "C" + * locale. + * + * Templates loaded before keep what they were loaded with. It is not the rest of `apply_chat_template`: the host builds + * the context (`messages`, `tools`, `add_generation_prompt`, `bos_token`, ...), and it is no security sandbox beyond what + * is listed here: untrusted templates also need \ref UntrustedLimits. + * + * @param env The environment to configure + * @param options The clock of `strftime_now` + */ +JINJA2CPP_EXPORT void ConfigureForChatTemplates(TemplateEnv& env, ChatTemplateOptions options = {}); + +} // namespace v2 +} // namespace jinja2 + +#endif // JINJA2CPP_CHAT_TEMPLATE_H diff --git a/include/jinja2cpp/config.h b/include/jinja2cpp/config.h index 61d8035aa..423e50d2f 100644 --- a/include/jinja2cpp/config.h +++ b/include/jinja2cpp/config.h @@ -4,6 +4,8 @@ // JINJA2CPP_VERSION (major * 10000 + minor * 100 + patch), its components and // JINJA2CPP_VERSION_STRING; generated from project(VERSION) by CMake #include "jinja2cpp/version.h" // IWYU pragma: export +// JINJA2CPP_HAS_CHAT_TEMPLATES and the other build options; generated by CMake +#include "jinja2cpp/features.h" // IWYU pragma: export // Jinja2C++ reports errors through exceptions internally and in a few documented calls // (docs/api-2.0.md, 2.1): building without them is not supported diff --git a/src/chat_template.cpp b/src/chat_template.cpp new file mode 100644 index 000000000..177ca2da8 --- /dev/null +++ b/src/chat_template.cpp @@ -0,0 +1,117 @@ +// The chat-template preset (docs/tasks/0360): settings, the transformers dialect and the two +// globals of transformers' apply_chat_template. Compiled only with JINJA2CPP_WITH_CHAT_TEMPLATES +#include + +#include "dialect.h" +#include "template_env_impl.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace jinja2 +{ +inline namespace v2 +{ +namespace +{ + +// The sandbox's MAX_RANGE +constexpr std::uint64_t SandboxMaxRange = 100000; + +std::tm LocalTime(std::time_t time) +{ + std::tm result{}; +#ifdef _WIN32 + localtime_s(&result, &time); +#else + localtime_r(&time, &result); +#endif + return result; +} + +// datetime.strftime: %f is the microseconds and %z, %:z and %Z are empty for a naive datetime; +// the rest is the C library's strftime, here in the "C" locale +std::string StrftimeNow(const std::string& format, std::chrono::system_clock::time_point now) +{ + const auto seconds = std::chrono::time_point_cast(now); + auto micros = std::chrono::duration_cast(now - seconds).count(); + auto time = std::chrono::system_clock::to_time_t(seconds); + if (micros < 0) + { + micros += 1000000; + --time; + } + const auto local = LocalTime(time); + + std::string pyFormat; + pyFormat.reserve(format.size()); + for (std::size_t pos = 0; pos < format.size(); ++pos) + { + if (format[pos] != '%' || pos + 1 == format.size()) + { + pyFormat.push_back(format[pos]); + continue; + } + const char directive = format[++pos]; + if (directive == 'f') + { + std::string digits = std::to_string(micros); + pyFormat.append(6 - std::min(6, digits.size()), '0'); + pyFormat += digits; + } + else if (directive == 'z' || directive == 'Z') + { + } + else if (directive == ':' && pos + 1 < format.size() && format[pos + 1] == 'z') + { + ++pos; + } + else + { + pyFormat.push_back('%'); + pyFormat.push_back(directive); + } + } + + std::ostringstream os; + os.imbue(std::locale::classic()); + os << std::put_time(&local, pyFormat.c_str()); + return os.str(); +} + +} // namespace + +void ConfigureForChatTemplates(TemplateEnv& env, ChatTemplateOptions options) +{ + auto& settings = env.GetSettings(); + settings.trimBlocks = true; + settings.lstripBlocks = true; + settings.extensions.loopControls = true; + settings.autoescape = false; + settings.autoescapeSelector = AutoescapeSelector(); + if (settings.limits.maxRange == 0 || settings.limits.maxRange > SandboxMaxRange) + { + settings.limits.maxRange = SandboxMaxRange; + } + detail::TemplateEnvAccess::GetImpl(env)->SetDialect(detail::Dialect::Transformers); + + // transformers: raise jinja2.exceptions.TemplateError(message) + env.AddGlobal("raise_exception", MakeCallable([](const std::string& message) -> Value { throw Exception(ErrorCode::TemplateRuntimeError, message); }, ArgInfo{ "message", true })); + auto now = options.now ? std::move(options.now) : [] { return std::chrono::system_clock::now(); }; + env.AddGlobal("strftime_now", MakeCallable([now = std::move(now)](const std::string& format) { return StrftimeNow(format, now()); }, ArgInfo{ "format", true })); +} + +} // namespace v2 +} // namespace jinja2 diff --git a/src/dialect.h b/src/dialect.h new file mode 100644 index 000000000..ab8e622b6 --- /dev/null +++ b/src/dialect.h @@ -0,0 +1,29 @@ +#ifndef JINJA2CPP_SRC_DIALECT_H +#define JINJA2CPP_SRC_DIALECT_H + +namespace jinja2 +{ +inline namespace v2 +{ +namespace detail +{ + +// The behaviour a template is loaded and rendered with besides Settings. Not public: a preset +// switches it (ConfigureForChatTemplates, docs/tasks/0360), and it stays Jinja2 otherwise, so +// the core compiles the same with or without the preset +enum class Dialect : unsigned char +{ + // Python Jinja2's Environment + Jinja2, + // transformers' apply_chat_template environment: an ImmutableSandboxedEnvironment whose + // `tojson` is json.dumps (keys in insertion order, no HTML escaping, ensure_ascii, indent, + // separators, sort_keys) and which knows the `{% generation %}` tag. The list and dict + // methods that change their receiver are unsafe attributes there + Transformers +}; + +} // namespace detail +} // namespace v2 +} // namespace jinja2 + +#endif // JINJA2CPP_SRC_DIALECT_H diff --git a/src/expression_evaluator.cpp b/src/expression_evaluator.cpp index 1e792e005..c185b3a6b 100644 --- a/src/expression_evaluator.cpp +++ b/src/expression_evaluator.cpp @@ -1613,6 +1613,12 @@ bool CallExpression::TryCallMethod(RenderContext& values, InternalValue& result, if (method->isMutating) { + if (methods::IsReadOnlyContainers(values)) + { + // Calling the sandbox's unsafe attribute fails with its message + callee = methods::UnsafeMethod(receiver, *name, values); + return false; + } receiver = methods::MakeMutable(receiver); } auto callParams = helpers::EvaluateCallParams(m_params, values); diff --git a/src/filter_factories.cpp b/src/filter_factories.cpp index b8c2d1796..3c2e74246 100644 --- a/src/filter_factories.cpp +++ b/src/filter_factories.cpp @@ -5,9 +5,11 @@ #include "expression_evaluator.h" #include "engine_errors.h" +#include "dialect.h" #include "function_base.h" #include "internal_value.h" #include "node_arena.h" +#include "recursion_guard.h" #include "render_context.h" #include "value_visitors.h" @@ -127,17 +129,23 @@ const std::vector& BuiltinFilters() return filters; } -const FilterEntry* FindBuiltinFilter(std::string_view filterName) +const FilterEntry* FindBuiltinFilter(std::string_view filterName, detail::Dialect dialect) { + // transformers' tojson replaces Jinja2's in its dialect, and no name reaches it otherwise + if (dialect == detail::Dialect::Transformers && filterName == "tojson") + { + static const auto transformersToJson = Entry("tojson"); + return &transformersToJson; + } const auto& filters = BuiltinFilters(); auto p = std::lower_bound(filters.begin(), filters.end(), filterName, [](const FilterEntry& entry, std::string_view name) { return entry.name < name; }); return p != filters.end() && p->name == filterName ? &*p : nullptr; } } // namespace -FilterPtr CreateFilter(std::string_view filterName, FunctionArgs params) +FilterPtr CreateFilter(std::string_view filterName, FunctionArgs params, detail::Dialect dialect) { - if (const auto* entry = FindBuiltinFilter(filterName)) + if (const auto* entry = FindBuiltinFilter(filterName, dialect)) { return entry->create(params); } @@ -146,7 +154,8 @@ FilterPtr CreateFilter(std::string_view filterName, FunctionArgs params) NodeRef CreateFilter(NodeArena& nodes, std::string_view filterName, FunctionArgs params, bool* unknown) { - if (const auto* entry = FindBuiltinFilter(filterName)) + // Made while the template loads, in the dialect of its settings + if (const auto* entry = FindBuiltinFilter(filterName, detail::t_parseLimits.dialect)) { return entry->make(nodes, params); } @@ -163,7 +172,7 @@ FilterPtr CreateFilter(const std::string& filterName, FunctionArgs params, Rende auto registered = env ? env->FindFilter(filterName) : std::optional(); if (!registered) { - return CreateFilter(filterName, params); + return CreateFilter(filterName, params, context.GetDialect()); } auto callable = visitors::InputValueConvertor::ConvertUserCallable(*registered); return std::make_shared(filterName, params, std::move(callable)); @@ -171,7 +180,7 @@ FilterPtr CreateFilter(const std::string& filterName, FunctionArgs params, Rende NodeRef CreateUnpackingFilter(NodeArena& nodes, const std::string& filterName, const CallArgsInfo& params, InternalValue registered, bool* unknown) { - if (unknown && !GetIf(®istered) && !FindBuiltinFilter(filterName)) + if (unknown && !GetIf(®istered) && !FindBuiltinFilter(filterName, detail::Dialect::Jinja2)) { *unknown = true; } @@ -258,7 +267,7 @@ InternalValue UnpackingFilter::Filter(const InternalValue& baseVal, RenderContex { const auto args = helpers::EvaluateCallParams(m_params, context); // Filters added to the environment take precedence over the builtins (ExpressionFilter::Make) - const FilterPtr filter = GetIf(&m_registered) ? std::make_shared(m_filterName, args, m_registered) : CreateFilter(m_filterName, args); + const FilterPtr filter = GetIf(&m_registered) ? std::make_shared(m_filterName, args, m_registered) : CreateFilter(m_filterName, args, context.GetDialect()); if (auto error = filter->GetArgumentsError(); !error.empty()) { throw TemplateRuntimeError(m_filterName + "() " + error); diff --git a/src/filters.cpp b/src/filters.cpp index 61269a989..c4a5ddacd 100644 --- a/src/filters.cpp +++ b/src/filters.cpp @@ -437,7 +437,7 @@ InternalValue Attribute::Filter(const InternalValue& baseVal, RenderContext& con { if (isPlainDict || !map || map->GetAttrPolicy() == MapAttrPolicy::MethodsFirst || !map->HasValue(attrName)) { - return methods::MakeBoundMethod(baseVal, *method); + return method->isMutating && methods::IsReadOnlyContainers(context) ? methods::UnsafeMethod(baseVal, attrName, context) : methods::MakeBoundMethod(baseVal, *method); } } if (isPlainDict) diff --git a/src/filters.h b/src/filters.h index 6870f982c..0116b8aa0 100644 --- a/src/filters.h +++ b/src/filters.h @@ -1,6 +1,7 @@ #ifndef JINJA2CPP_SRC_FILTERS_H #define JINJA2CPP_SRC_FILTERS_H +#include "dialect.h" #include "expression_evaluator.h" #include "function_base.h" #include "internal_value.h" @@ -21,7 +22,7 @@ inline namespace v2 using FilterPtr = std::shared_ptr; using FilterParams = FunctionArgs; -extern FilterPtr CreateFilter(std::string_view filterName, FunctionArgs params); +extern FilterPtr CreateFilter(std::string_view filterName, FunctionArgs params, detail::Dialect dialect = detail::Dialect::Jinja2); // The same in the arena of the template that names the filter // A name that is no built-in filter makes a UserDefinedFilter and sets `*unknown` extern NodeRef CreateFilter(NodeArena& nodes, std::string_view filterName, FunctionArgs params, bool* unknown = nullptr); @@ -168,7 +169,9 @@ class Serialize : public FilterBase enum Mode { JsonMode, - YamlMode + YamlMode, + // transformers' tojson (Dialect::Transformers) + TransformersJsonMode }; Serialize(const FilterParams& params, Mode mode); diff --git a/src/load_settings.h b/src/load_settings.h index b8829aa87..b1014e672 100644 --- a/src/load_settings.h +++ b/src/load_settings.h @@ -1,6 +1,7 @@ #ifndef JINJA2CPP_SRC_LOAD_SETTINGS_H #define JINJA2CPP_SRC_LOAD_SETTINGS_H +#include "dialect.h" #include "recursion_guard.h" #include @@ -99,7 +100,7 @@ struct Delimiters // (TemplateEnvImpl::GetLoadSettings), instead of each copying the settings and rebuilding the delimiters struct LoadSettings { - explicit LoadSettings(const Settings& setts, const BigIntBackend* backend = nullptr) + explicit LoadSettings(const Settings& setts, const BigIntBackend* backend = nullptr, Dialect templateDialect = Dialect::Jinja2) : settings(setts) , narrow(setts) , wide(setts) @@ -109,6 +110,7 @@ struct LoadSettings , hasRenderLimits(jinja2::HasRenderLimits(setts.limits) || backend != nullptr) #endif , bigIntBackend(backend) + , dialect(templateDialect) { } @@ -134,6 +136,8 @@ struct LoadSettings bool hasRenderLimits; // The big-integer arithmetic the template's renders use (TemplateEnv::SetBigIntBackend); null for none const BigIntBackend* bigIntBackend; + // What a preset changed besides the settings (ConfigureForChatTemplates) + Dialect dialect; }; using LoadSettingsPtr = std::shared_ptr; diff --git a/src/recursion_guard.h b/src/recursion_guard.h index fb5700d89..9f88a3f68 100644 --- a/src/recursion_guard.h +++ b/src/recursion_guard.h @@ -1,6 +1,7 @@ #ifndef JINJA2CPP_SRC_RECURSION_GUARD_H #define JINJA2CPP_SRC_RECURSION_GUARD_H +#include "dialect.h" #include "engine_errors.h" #include @@ -64,13 +65,14 @@ inline thread_local ThreadStack t_threadStack; bool StackNearlyExhaustedSlow(std::uintptr_t frame); -// The parser's limits for the template being loaded (ParseLimitsScope), in a thread-local -// rather than in each parser, which lives for one tag +// The parser's limits and dialect for the template being loaded (ParseLimitsScope), in a +// thread-local rather than in each parser, which lives for one tag struct ParseLimits { unsigned maxExpressionDepth = 70; unsigned maxExpressionOperators = 491; unsigned maxBlockNesting = 248; + Dialect dialect = Dialect::Jinja2; }; static_assert(ParseLimits{}.maxExpressionDepth == Settings::Limits{}.maxExpressionDepth && ParseLimits{}.maxExpressionOperators == Settings::Limits{}.maxExpressionOperators @@ -227,10 +229,10 @@ inline void AfterHostCall() class ParseLimitsScope { public: - explicit ParseLimitsScope(const Settings::Limits& limits) + explicit ParseLimitsScope(const Settings::Limits& limits, detail::Dialect dialect = detail::Dialect::Jinja2) : m_saved(detail::t_parseLimits) { - detail::t_parseLimits = { LimitOrNone(limits.maxExpressionDepth), LimitOrNone(limits.maxExpressionOperators), LimitOrNone(limits.maxBlockNesting) }; + detail::t_parseLimits = { LimitOrNone(limits.maxExpressionDepth), LimitOrNone(limits.maxExpressionOperators), LimitOrNone(limits.maxBlockNesting), dialect }; } ~ParseLimitsScope() { detail::t_parseLimits = m_saved; } ParseLimitsScope(const ParseLimitsScope&) = delete; diff --git a/src/render_context.h b/src/render_context.h index feae89c83..1c9726812 100644 --- a/src/render_context.h +++ b/src/render_context.h @@ -1,6 +1,7 @@ #ifndef JINJA2CPP_SRC_RENDER_CONTEXT_H #define JINJA2CPP_SRC_RENDER_CONTEXT_H +#include "dialect.h" #include "internal_value.h" #include "lookup_result.h" #include "node_arena.h" @@ -121,6 +122,8 @@ struct IRendererCallback : IComparable // Always throws: callers rely on it not returning (docs/tasks/0055) [[noreturn]] virtual void ThrowRuntimeError(ErrorCode code, ValuesList extraParams) = 0; [[nodiscard]] virtual const Settings& GetSettings() const = 0; + // What a preset changed besides the settings (load_settings.h) + [[nodiscard]] virtual detail::Dialect GetDialect() const { return detail::Dialect::Jinja2; } // The environment the template was loaded in, if any [[nodiscard]] virtual TemplateEnv* GetEnv() const { return nullptr; } // The random generator of lipsum, one per render @@ -927,6 +930,7 @@ class RenderContext } // Settings::userCallablesAsFilters of the template: whether a name that is no filter or test // of the template may be a callable among the values + [[nodiscard]] detail::Dialect GetDialect() const { return m_rendererCallback ? m_rendererCallback->GetDialect() : detail::Dialect::Jinja2; } [[nodiscard]] bool UserCallablesAsFilters() const { return m_rendererCallback ? m_rendererCallback->GetSettings().userCallablesAsFilters : false; } // A context for a nested render: with `includeCurrentContext` it sees all of this // context's names (see the constructor above for how long it may live), otherwise only diff --git a/src/serialize_filters.cpp b/src/serialize_filters.cpp index c7532c413..1f96ad592 100644 --- a/src/serialize_filters.cpp +++ b/src/serialize_filters.cpp @@ -680,6 +680,12 @@ Serialize::Serialize(const FilterParams& params, const Serialize::Mode mode) ParseParams(args, params); break; } + case TransformersJsonMode: + { + static const auto args = MakeArgumentsTable({ { "ensure_ascii", false }, { "indent", false }, { "separators", false }, { "sort_keys", false } }); + ParseParams(args, params); + break; + } default: ParseParams(NoArguments(), params); break; @@ -708,15 +714,34 @@ uint32_t Utf8CharValue(std::string_view ch) return cp; } -// Python's json.dumps(value, sort_keys=True, indent=indent) with the default ensure_ascii, as -// Jinja2's htmlsafe_json_dumps escapes it: <, >, & and ' become \u escapes. Outside strings -// those characters can only come from the indent, which is escaped once up front. +// The arguments of Python's json.dumps that the two tojson filters pass +struct JsonOptions +{ + std::optional indent; + // Jinja2's htmlsafe_json_dumps: <, >, & and ' become \u escapes, also in the indent + bool htmlSafe = true; + bool ensureAscii = true; + bool sortKeys = true; + // separators: by default ", " and ": ", or "," and ": " with an indent + std::optional> separators; +}; + +// Python's json.dumps(value, ...) with the options above. Jinja2's tojson is sort_keys=True, +// the default ensure_ascii and htmlsafe_json_dumps' escaping; transformers' is +// ensure_ascii=False and insertion order, unescaped, with its arguments passed through. +// Outside strings the escaped characters can only come from the indent, which is escaped once +// up front. class PythonJsonWriter { public: - PythonJsonWriter(RenderContext& context, std::optional indent) + PythonJsonWriter(RenderContext& context, JsonOptions options) : m_context(context) - , m_indent(HtmlSafeIndent(std::move(indent))) + , m_htmlSafe(options.htmlSafe) + , m_ensureAscii(options.ensureAscii) + , m_sortKeys(options.sortKeys) + , m_indent(options.htmlSafe ? HtmlSafeIndent(std::move(options.indent)) : std::move(options.indent)) + , m_itemSeparator(options.separators ? options.separators->first : m_indent ? "," : ", ") + , m_keySeparator(options.separators ? options.separators->second : ": ") { } @@ -777,11 +802,20 @@ class PythonJsonWriter } } - // Bytes json.dumps keeps as they are and htmlsafe_json_dumps does not escape: printable - // ASCII but ", \, <, >, & and ' - static bool IsPlainJsonByte(unsigned char ch) + // Bytes json.dumps keeps as they are (and htmlsafe_json_dumps does not escape): printable + // ASCII but " and \ (and <, >, & and '); without ensure_ascii, also 0x7F and every byte of + // a character beyond ASCII + bool IsPlainJsonByte(unsigned char ch) const { - return ch >= 0x20 && ch < 0x7F && ch != '"' && ch != '\\' && ch != '<' && ch != '>' && ch != '&' && ch != '\''; + if (ch < 0x20 || ch == '"' || ch == '\\') + { + return false; + } + if (ch >= 0x7F) + { + return !m_ensureAscii; + } + return !m_htmlSafe || (ch != '<' && ch != '>' && ch != '&' && ch != '\''); } void AppendUnit(uint32_t unit) @@ -855,7 +889,7 @@ class PythonJsonWriter m_out += "\\f"; break; default: - if (cp < 0x20 || cp >= 0x7F || cp == '<' || cp == '>' || cp == '&' || cp == '\'') + if (cp < 0x20 || (m_ensureAscii && cp >= 0x7F) || (m_htmlSafe && (cp == '<' || cp == '>' || cp == '&' || cp == '\''))) { if (cp >= 0x10000) { @@ -886,7 +920,7 @@ class PythonJsonWriter CheckSequenceSize(m_out.View().size(), "a JSON document"); if (!isFirst) { - m_out += m_indent ? "," : ", "; + m_out += m_itemSeparator; } isFirst = false; if (m_indent) @@ -951,11 +985,19 @@ class PythonJsonWriter // namespace(), cycler() and joiner() are objects, which json.dumps rejects throw TemplateRuntimeError("Object of type "s + objectClass + " is not JSON serializable"); } - // sort_keys: Python orders str keys by code point, which is UTF-8 byte order auto entries = map->GetEntries(); std::vector> order; order.reserve(entries.size()); - if (std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) + if (!m_sortKeys) + { + // Insertion order; json.dumps writes each key as it comes to it + for (size_t idx = 0; idx != entries.size(); ++idx) + { + order.emplace_back(KeyText(entries[idx].key), idx); + } + } + // sort_keys: Python orders str keys by code point, which is UTF-8 byte order + else if (std::all_of(entries.begin(), entries.end(), [](const KeyValuePair& entry) { return IsStringValue(entry.key); })) { for (size_t idx = 0; idx != entries.size(); ++idx) { @@ -969,7 +1011,7 @@ class PythonJsonWriter } WriteContainer('{', '}', order, level, [this, &entries, level](const std::pair& item) { WriteString(item.first); - m_out += ": "; + m_out += m_keySeparator; WriteValue(entries[item.second].value, level + 1); }); } @@ -1013,47 +1055,51 @@ class PythonJsonWriter // The keys of a mapping with a key that is not a string, in sort_keys order, with the JSON // spelling of each: 1 is "1", 2.5 is "2.5", True is "true" and None is "null". A tuple or // an object is no JSON key; keys of types Python cannot compare (a str and an int) are a TypeError + static std::string KeyText(const InternalValue& key) + { + if (IsStringValue(key)) + { + return AsString(key); + } + if (const auto* i = GetIf(&key)) + { + return std::to_string(*i); + } + if (const auto big = AsBigInt(key)) + { + return std::string(CheckedDecimal(*big)); + } + if (const auto* b = GetIf(&key)) + { + return *b ? "true" : "false"; + } + if (const auto* d = GetIf(&key)) + { + return JsonFloatText(*d); + } + if (key.IsNone()) + { + return "null"; + } + throw TemplateRuntimeError("keys must be str, int, float, bool or None, not "s + Apply(key)); + } + static void OrderNonStringKeys(const std::vector& entries, std::vector>& order) { for (size_t idx = 0; idx != entries.size(); ++idx) { - const auto& key = entries[idx].key; - std::string text; - if (const auto* i = GetIf(&key)) - { - text = std::to_string(*i); - } - else if (const auto big = AsBigInt(key)) - { - text = CheckedDecimal(*big); - } - else if (const auto* b = GetIf(&key)) - { - text = *b ? "true" : "false"; - } - else if (const auto* d = GetIf(&key)) - { - text = JsonFloatText(*d); - } - else if (key.IsNone()) - { - text = "null"; - } - else if (IsStringValue(key)) - { - text = AsString(key); - } - else - { - throw TemplateRuntimeError("keys must be str, int, float, bool or None, not "s + Apply(key)); - } - order.emplace_back(std::move(text), idx); + order.emplace_back(KeyText(entries[idx].key), idx); } StableSortValues(order.begin(), order.end(), [&entries](const auto& left, const auto& right) { return KeyLess(entries[left.second].key, entries[right.second].key); }); } RenderContext& m_context; + bool m_htmlSafe; + bool m_ensureAscii; + bool m_sortKeys; std::optional m_indent; + std::string m_itemSeparator; + std::string m_keySeparator; StringBuilder m_out; }; @@ -1414,49 +1460,94 @@ class YamlWriter StringBuilder m_out; }; -} // namespace - -InternalValue Serialize::Filter(const InternalValue& value, RenderContext& context) +// json.dumps' indent: a string as it is, an int (a bool among them) as that many spaces, none +// for None or no argument +std::optional JsonIndent(const InternalValue& indentVal) { - if (m_mode == YamlMode) - { - // YAML is no HTML: under autoescape the result is escaped like any string - return YamlWriter(context).Write(value); - } - - // Jinja2's do_tojson: json.dumps with sort_keys=True, then htmlsafe_json_dumps escapes - // <, >, & and ' so the result is safe in HTML and