From e18baf612d5886a5d4c38b382737b7b789a74e3d Mon Sep 17 00:00:00 2001 From: Geng Tian Date: Thu, 11 Jun 2026 18:25:24 +0000 Subject: [PATCH] MDEV-39995 JSON_CONTAINS and JSON_EQUALS do not compare strings based on semantic JSON_CONTAINS, JSON_EQUALS, and JSON_OVERLAPS used raw byte-level comparison (memcmp) for JSON string values, which meant semantically equivalent strings like "A" and "\u0041" were incorrectly treated as different. Fix: add json_string_compare() that decodes Unicode escape sequences before comparing, and fix json_normalize to produce a canonical form for strings with escapes so JSON_EQUALS works correctly. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- include/json_lib.h | 9 ++ .../main/func_json_unicode_escape.result | 129 +++++++++++++++++ mysql-test/main/func_json_unicode_escape.test | 83 +++++++++++ sql/item_jsonfunc.cc | 16 +-- strings/json_lib.c | 37 +++++ strings/json_normalize.c | 136 +++++++++++++++++- 6 files changed, 398 insertions(+), 12 deletions(-) create mode 100644 mysql-test/main/func_json_unicode_escape.result create mode 100644 mysql-test/main/func_json_unicode_escape.test diff --git a/include/json_lib.h b/include/json_lib.h index debfa99eaca5c..fab12f9eeb03b 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -478,6 +478,15 @@ int json_normalize(DYNAMIC_STRING *result, int json_skip_array_and_count(json_engine_t *j, int* n_item); +/* + Compare two JSON string values semantically, taking Unicode escape + sequences into account. For example, "A" and "\u0041" are considered equal. + Returns 0 if the strings are equal, non-zero otherwise. +*/ +int json_string_compare(CHARSET_INFO *cs, + const uchar *str1, int len1, int escaped1, + const uchar *str2, int len2, int escaped2); + inline static int json_scan_ended(json_engine_t *j) { return (j->state == JST_ARRAY_END && j->stack_p == 0); diff --git a/mysql-test/main/func_json_unicode_escape.result b/mysql-test/main/func_json_unicode_escape.result new file mode 100644 index 0000000000000..7afebb0c801d5 --- /dev/null +++ b/mysql-test/main/func_json_unicode_escape.result @@ -0,0 +1,129 @@ +# +# MDEV-39995: JSON_CONTAINS and JSON_EQUALS do not compare strings +# based on semantic +# +# +# JSON string values with Unicode escape sequences should be treated +# as semantically equal to their literal equivalents. +# \u0041 is the Unicode escape for 'A'. +# +# JSON_CONTAINS: should return 1 for semantically equal strings +SELECT JSON_CONTAINS('"A"', '"\\u0041"'); +JSON_CONTAINS('"A"', '"\\u0041"') +1 +SELECT JSON_CONTAINS('"\\u0041"', '"A"'); +JSON_CONTAINS('"\\u0041"', '"A"') +1 +# JSON_OVERLAPS: should return 1 for semantically equal strings +SELECT JSON_OVERLAPS('"A"', '"\\u0041"'); +JSON_OVERLAPS('"A"', '"\\u0041"') +1 +# JSON_EQUALS: should return 1 for semantically equal strings +SELECT JSON_EQUALS('"A"', '"\\u0041"'); +JSON_EQUALS('"A"', '"\\u0041"') +1 +# JSON_UNQUOTE correctly resolves the escape (proving they are the same) +SELECT JSON_UNQUOTE('"A"') = JSON_UNQUOTE('"\\u0041"'); +JSON_UNQUOTE('"A"') = JSON_UNQUOTE('"\\u0041"') +1 +# +# Additional test from MDEV-39995 comment: +# Using hex literal that represents the bytes of '"\u0041"' +# +SELECT JSON_UNQUOTE('"A"'); +JSON_UNQUOTE('"A"') +A +SELECT JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)); +JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)) +A +SELECT JSON_CONTAINS('"A"', CAST(0x225C753030343122 AS CHAR)); +JSON_CONTAINS('"A"', CAST(0x225C753030343122 AS CHAR)) +1 +SELECT JSON_CONTAINS(JSON_QUOTE(JSON_UNQUOTE('"A"')), +JSON_QUOTE(JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)))); +JSON_CONTAINS(JSON_QUOTE(JSON_UNQUOTE('"A"')), +JSON_QUOTE(JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)))) +1 +# +# More Unicode escape equivalences +# +# \u0048\u0065\u006C\u006C\u006F = "Hello" +SELECT JSON_CONTAINS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); +JSON_CONTAINS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"') +1 +SELECT JSON_EQUALS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); +JSON_EQUALS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"') +1 +SELECT JSON_OVERLAPS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); +JSON_OVERLAPS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"') +1 +# Mixed literal and escape in the same string: "H\u0065llo" = "Hello" +SELECT JSON_EQUALS('"Hello"', '"H\\u0065llo"'); +JSON_EQUALS('"Hello"', '"H\\u0065llo"') +1 +# +# Test within arrays and objects +# +SELECT JSON_CONTAINS('["A", "B"]', '["\\u0041"]'); +JSON_CONTAINS('["A", "B"]', '["\\u0041"]') +1 +SELECT JSON_CONTAINS('{"key": "A"}', '{"key": "\\u0041"}'); +JSON_CONTAINS('{"key": "A"}', '{"key": "\\u0041"}') +1 +SELECT JSON_EQUALS('["A", "B"]', '["\\u0041", "\\u0042"]'); +JSON_EQUALS('["A", "B"]', '["\\u0041", "\\u0042"]') +1 +SELECT JSON_EQUALS('{"key": "A"}', '{"key": "\\u0041"}'); +JSON_EQUALS('{"key": "A"}', '{"key": "\\u0041"}') +1 +# +# Surrogate pairs: characters above U+FFFF encoded as two \uXXXX escapes. +# U+1F600 (πŸ˜€) = \uD83D\uDE00 +# U+1F60A (😊) = \uD83D\uDE0A +# +SET NAMES utf8mb4; +SELECT JSON_EQUALS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +JSON_EQUALS('"?"', '"\\uD83D\\uDE00"') +1 +SELECT JSON_CONTAINS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +JSON_CONTAINS('"?"', '"\\uD83D\\uDE00"') +1 +SELECT JSON_OVERLAPS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +JSON_OVERLAPS('"?"', '"\\uD83D\\uDE00"') +1 +SELECT JSON_EQUALS('"😊"', '"\\uD83D\\uDE0A"'); +JSON_EQUALS('"?"', '"\\uD83D\\uDE0A"') +1 +SELECT JSON_CONTAINS('["πŸ˜€", "hello"]', '["\\uD83D\\uDE00"]'); +JSON_CONTAINS('["?", "hello"]', '["\\uD83D\\uDE00"]') +1 +SELECT JSON_EQUALS('{"emoji": "πŸ˜€"}', '{"emoji": "\\uD83D\\uDE00"}'); +JSON_EQUALS('{"emoji": "?"}', '{"emoji": "\\uD83D\\uDE00"}') +1 +# +# Escaped object keys: \u006B\u0065\u0079 = "key" +# +SELECT JSON_EQUALS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}'); +JSON_EQUALS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}') +1 +SELECT JSON_CONTAINS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}'); +JSON_CONTAINS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}') +1 +# +# BMP non-ASCII: Γ© = U+00E9, literal UTF-8 vs escape +# +SELECT JSON_EQUALS('"Γ©"', '"\\u00E9"'); +JSON_EQUALS('"Γ©"', '"\\u00E9"') +1 +SELECT JSON_CONTAINS('"Γ©"', '"\\u00E9"'); +JSON_CONTAINS('"Γ©"', '"\\u00E9"') +1 +SELECT JSON_OVERLAPS('["Γ©"]', '["\\u00E9"]'); +JSON_OVERLAPS('["Γ©"]', '["\\u00E9"]') +1 +# +# CJK: δΈ­ = U+4E2D +# +SELECT JSON_EQUALS('"δΈ­"', '"\\u4E2D"'); +JSON_EQUALS('"δΈ­"', '"\\u4E2D"') +1 diff --git a/mysql-test/main/func_json_unicode_escape.test b/mysql-test/main/func_json_unicode_escape.test new file mode 100644 index 0000000000000..b7847312aae6c --- /dev/null +++ b/mysql-test/main/func_json_unicode_escape.test @@ -0,0 +1,83 @@ +--echo # +--echo # MDEV-39995: JSON_CONTAINS and JSON_EQUALS do not compare strings +--echo # based on semantic +--echo # + +--echo # +--echo # JSON string values with Unicode escape sequences should be treated +--echo # as semantically equal to their literal equivalents. +--echo # \u0041 is the Unicode escape for 'A'. +--echo # + +--echo # JSON_CONTAINS: should return 1 for semantically equal strings +SELECT JSON_CONTAINS('"A"', '"\\u0041"'); +SELECT JSON_CONTAINS('"\\u0041"', '"A"'); + +--echo # JSON_OVERLAPS: should return 1 for semantically equal strings +SELECT JSON_OVERLAPS('"A"', '"\\u0041"'); + +--echo # JSON_EQUALS: should return 1 for semantically equal strings +SELECT JSON_EQUALS('"A"', '"\\u0041"'); + +--echo # JSON_UNQUOTE correctly resolves the escape (proving they are the same) +SELECT JSON_UNQUOTE('"A"') = JSON_UNQUOTE('"\\u0041"'); + +--echo # +--echo # Additional test from MDEV-39995 comment: +--echo # Using hex literal that represents the bytes of '"\u0041"' +--echo # +SELECT JSON_UNQUOTE('"A"'); +SELECT JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)); +SELECT JSON_CONTAINS('"A"', CAST(0x225C753030343122 AS CHAR)); +SELECT JSON_CONTAINS(JSON_QUOTE(JSON_UNQUOTE('"A"')), + JSON_QUOTE(JSON_UNQUOTE(CAST(0x225C753030343122 AS CHAR)))); + +--echo # +--echo # More Unicode escape equivalences +--echo # +--echo # \u0048\u0065\u006C\u006C\u006F = "Hello" +SELECT JSON_CONTAINS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); +SELECT JSON_EQUALS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); +SELECT JSON_OVERLAPS('"Hello"', '"\\u0048\\u0065\\u006C\\u006C\\u006F"'); + +--echo # Mixed literal and escape in the same string: "H\u0065llo" = "Hello" +SELECT JSON_EQUALS('"Hello"', '"H\\u0065llo"'); + +--echo # +--echo # Test within arrays and objects +--echo # +SELECT JSON_CONTAINS('["A", "B"]', '["\\u0041"]'); +SELECT JSON_CONTAINS('{"key": "A"}', '{"key": "\\u0041"}'); +SELECT JSON_EQUALS('["A", "B"]', '["\\u0041", "\\u0042"]'); +SELECT JSON_EQUALS('{"key": "A"}', '{"key": "\\u0041"}'); + +--echo # +--echo # Surrogate pairs: characters above U+FFFF encoded as two \uXXXX escapes. +--echo # U+1F600 (πŸ˜€) = \uD83D\uDE00 +--echo # U+1F60A (😊) = \uD83D\uDE0A +--echo # +SET NAMES utf8mb4; +SELECT JSON_EQUALS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +SELECT JSON_CONTAINS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +SELECT JSON_OVERLAPS('"πŸ˜€"', '"\\uD83D\\uDE00"'); +SELECT JSON_EQUALS('"😊"', '"\\uD83D\\uDE0A"'); +SELECT JSON_CONTAINS('["πŸ˜€", "hello"]', '["\\uD83D\\uDE00"]'); +SELECT JSON_EQUALS('{"emoji": "πŸ˜€"}', '{"emoji": "\\uD83D\\uDE00"}'); + +--echo # +--echo # Escaped object keys: \u006B\u0065\u0079 = "key" +--echo # +SELECT JSON_EQUALS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}'); +SELECT JSON_CONTAINS('{"key":"A"}', '{"\\u006B\\u0065\\u0079":"A"}'); + +--echo # +--echo # BMP non-ASCII: Γ© = U+00E9, literal UTF-8 vs escape +--echo # +SELECT JSON_EQUALS('"Γ©"', '"\\u00E9"'); +SELECT JSON_CONTAINS('"Γ©"', '"\\u00E9"'); +SELECT JSON_OVERLAPS('["Γ©"]', '["\\u00E9"]'); + +--echo # +--echo # CJK: δΈ­ = U+4E2D +--echo # +SELECT JSON_EQUALS('"δΈ­"', '"\\u4E2D"'); diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 24b91c2e149e0..e86ef1209b361 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1704,12 +1704,10 @@ int Item_func_json_contains::check_contains(json_engine_t *js, { return FALSE; } - /* - TODO: make proper json-json comparison here that takes excipient - into account. - */ - return value->value_len == js->value_len && - memcmp(value->value, js->value, value->value_len) == 0; + return json_string_compare(js->s.cs, + js->value, js->value_len, js->value_escaped, + value->value, value->value_len, + value->value_escaped) == 0; case JSON_VALUE_NUMBER: if (value->value_type == JSON_VALUE_NUMBER) { @@ -5044,8 +5042,10 @@ static bool json_find_overlap_with_scalar(json_engine_t *js, json_engine_t *valu } else if (js->value_type == JSON_VALUE_STRING) { - return value->value_len == js->value_len && - memcmp(value->value, js->value, value->value_len) == 0; + return json_string_compare(js->s.cs, + js->value, js->value_len, js->value_escaped, + value->value, value->value_len, + value->value_escaped) == 0; } } return value->value_type == js->value_type; diff --git a/strings/json_lib.c b/strings/json_lib.c index df9e05d5b5c11..62ee61302debc 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1818,6 +1818,43 @@ int json_unescape(CHARSET_INFO *json_cs, } +/* + Compare two JSON string values semantically, resolving escape sequences. + If neither string has escapes, falls back to memcmp for speed. + Returns 0 if strings are equal, non-zero otherwise. +*/ +int json_string_compare(CHARSET_INFO *cs, + const uchar *str1, int len1, int escaped1, + const uchar *str2, int len2, int escaped2) +{ + json_string_t s1, s2; + int r1; + + if (!escaped1 && !escaped2) + { + if (len1 != len2) + return 1; + return memcmp(str1, str2, len1); + } + + json_string_setup(&s1, cs, str1, str1 + len1); + json_string_setup(&s2, cs, str2, str2 + len2); + + for (r1= json_read_string_const_chr(&s1); r1 == 0; + r1= json_read_string_const_chr(&s1)) + { + int r2= json_read_string_const_chr(&s2); + if (r2) + return 1; + if (s1.c_next != s2.c_next) + return 1; + } + + return (json_read_string_const_chr(&s2) != 0 && + s1.error == JE_EOS && s2.error == JE_EOS) ? 0 : 1; +} + + /* When we need to replace a character with the escaping. */ enum json_esc_char_classes { ESC_= 0, /* No need to escape. */ diff --git a/strings/json_normalize.c b/strings/json_normalize.c index 81575f07069bc..06d1349bb0943 100644 --- a/strings/json_normalize.c +++ b/strings/json_normalize.c @@ -750,6 +750,19 @@ json_norm_value_true_init(struct json_norm_value *val) } +/* + Store the raw (unprocessed) string value from the JSON engine + into a normalized value. Used as a fallback when escape decoding fails. +*/ +static int +json_norm_value_raw_string_init(struct json_norm_value *val, json_engine_t *je) +{ + const char *je_value_begin= (const char *)je->value_begin; + size_t je_value_len= (size_t)(je->value_end - je->value_begin); + return json_norm_value_string_init(val, je_value_begin, je_value_len); +} + + static int json_norm_value_init(struct json_norm_value *val, json_engine_t *je) { @@ -757,9 +770,88 @@ json_norm_value_init(struct json_norm_value *val, json_engine_t *je) switch (je->value_type) { case JSON_VALUE_STRING: { - const char *je_value_begin= (const char *)je->value_begin; - size_t je_value_len= (je->value_end - je->value_begin); - err= json_norm_value_string_init(val, je_value_begin, je_value_len); + if (je->value_escaped) + { + /* + The string contains escape sequences (e.g. \u0041). + Decode and re-encode to get a canonical form so that + semantically equal strings like "A" and "\u0041" normalize + to the same representation. + */ + uchar stack_buf[256]; + uchar *unescaped= stack_buf; + int unesc_len; + + if ((size_t)je->value_len > sizeof(stack_buf)) + { + unescaped= (uchar *) my_malloc(PSI_JSON, je->value_len, MYF(0)); + if (!unescaped) + { + err= json_norm_value_raw_string_init(val, je); + break; + } + } + + unesc_len= json_unescape(je->s.cs, + je->value, je->value + je->value_len, + je->s.cs, unescaped, + unescaped + je->value_len); + if (unesc_len < 0) + { + if (unescaped != stack_buf) + my_free(unescaped); + err= json_norm_value_raw_string_init(val, je); + } + else + { + /* + Re-escape the decoded string. The escaped form can be at most + 6x the unescaped length (\uXXXX) plus quotes. + */ + size_t esc_buf_len= (size_t)unesc_len * 6 + 3; + uchar esc_stack_buf[1536]; + uchar *esc_buf= esc_stack_buf; + int esc_len; + + if (esc_buf_len > sizeof(esc_stack_buf)) + { + esc_buf= (uchar *) my_malloc(PSI_JSON, esc_buf_len, MYF(0)); + if (!esc_buf) + { + if (unescaped != stack_buf) + my_free(unescaped); + err= json_norm_value_raw_string_init(val, je); + break; + } + } + + esc_buf[0]= '"'; + esc_len= json_escape(je->s.cs, unescaped, unescaped + unesc_len, + je->s.cs, esc_buf + 1, + esc_buf + esc_buf_len - 1); + if (unescaped != stack_buf) + my_free(unescaped); + + if (esc_len < 0) + { + if (esc_buf != esc_stack_buf) + my_free(esc_buf); + err= json_norm_value_raw_string_init(val, je); + } + else + { + esc_buf[1 + esc_len]= '"'; + err= json_norm_value_string_init(val, (const char *)esc_buf, + (size_t)(esc_len + 2)); + if (esc_buf != esc_stack_buf) + my_free(esc_buf); + } + } + } + else + { + err= json_norm_value_raw_string_init(val, je); + } break; } case JSON_VALUE_NULL: @@ -888,7 +980,43 @@ json_norm_parse(struct json_norm_value *root, json_engine_t *je, MEM_ROOT *curre /* we have the key name */ /* reset the dynstr: */ dynstr_trunc(&key, key.length); - dynstr_append_mem(&key, (char*)key_start, (key_end - key_start)); + + { + /* + Decode the key to resolve any Unicode escape sequences + (e.g. \u006B\u0065\u0079 -> "key"), so that semantically + equal keys normalize to the same representation. + */ + size_t raw_len= (size_t)(key_end - key_start); + uchar key_stack_buf[256]; + uchar *decoded= key_stack_buf; + int use_heap= (raw_len > sizeof(key_stack_buf)); + + if (use_heap) + { + decoded= (uchar *) my_malloc(PSI_JSON, raw_len, MYF(0)); + if (!decoded) + { + dynstr_append_mem(&key, (char*)key_start, raw_len); + use_heap= 0; + } + } + + if (decoded) + { + int dec_len= json_unescape(je->s.cs, key_start, + key_start + raw_len, + je->s.cs, decoded, + decoded + raw_len); + if (dec_len >= 0) + dynstr_append_mem(&key, (char*)decoded, (size_t)dec_len); + else + dynstr_append_mem(&key, (char*)key_start, raw_len); + } + + if (use_heap) + my_free(decoded); + } /* After reading the key, we have a follow-up value. */ err = json_read_value(je);