From fcb384fddcf0a6f627e622722afb1c6e95c8b5a9 Mon Sep 17 00:00:00 2001 From: "Yuval (iMac-Studio)" Date: Sat, 18 Jul 2026 03:28:33 +0300 Subject: [PATCH 1/2] Fix compact_script fusing adjacent operators (e.g. `< -` -> `<-`) `Engine::compact_script`'s whitespace removal only inserted a separating space between two identifier characters (so `let x` doesn't become `letx`). It had no equivalent guard for operators, so two adjacent tokens could fuse into a different token: `x < -1` compacted to `x<-1`, and the lexer then reads `<-` as an invalid operator ("This is not Go!"), so the compacted script no longer compiles -- breaking compact_script's documented "semantically identical to the input" contract. Also insert a separating space when the two boundary characters would merge into a recognized operator or a reserved symbol, decided by the tokenizer's own tables (`Token::lookup_symbol_from_syntax` / `is_reserved_keyword_or_symbol`) rather than a hardcoded character set. Ordinary compaction (`let x=1;`) is unchanged. Adds a regression test. --- src/tokenizer.rs | 35 ++++++++++++++++++++++++++++------- tests/tokens.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 86b59feb6..c92a58490 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2844,13 +2844,34 @@ impl Iterator for TokenIterator<'_> { if !buf.is_empty() && !compressed.is_empty() { let cur = buf.chars().next().unwrap(); - - if cur == '_' || is_id_first_alphabetic(cur) || is_id_continue(cur) { - let prev = compressed.chars().last().unwrap(); - - if prev == '_' || is_id_first_alphabetic(prev) || is_id_continue(prev) { - *compressed += " "; - } + let prev = compressed.chars().last().unwrap(); + + // A separating space is needed when concatenating this + // token onto the output would change how the boundary + // re-tokenizes: + // * two identifier characters merge into one + // identifier/keyword (e.g. `let` + `x` -> `letx`); + // * the two boundary characters merge into a + // recognized operator or a reserved symbol (e.g. + // `<` + `-` -> `<-`, which the lexer rejects, or + // `<` + `=` -> `<=`). + // The second case is decided by the tokenizer's own + // symbol tables, so nothing has to be hardcoded here. + let is_id = + |c: char| c == '_' || is_id_first_alphabetic(c) || is_id_continue(c); + + let need_space = if is_id(prev) && is_id(cur) { + true + } else { + let mut pair = SmartString::new_const(); + pair.push(prev); + pair.push(cur); + Token::lookup_symbol_from_syntax(&pair).is_some() + || is_reserved_keyword_or_symbol(&pair).0 + }; + + if need_space { + *compressed += " "; } } diff --git a/tests/tokens.rs b/tests/tokens.rs index 52eaf16de..c0b33188d 100644 --- a/tests/tokens.rs +++ b/tests/tokens.rs @@ -94,3 +94,34 @@ fn test_tokens_unicode_xid_ident() { ); let _ = result.unwrap_err(); } + +/// Regression: `Engine::compact_script` must not fuse an operator with a +/// following character into a different token. In particular `x < -1` must not +/// compact to `x<-1`, which the lexer rejects as the invalid operator `<-` +/// ("This is not Go!"). The compacted output must always compile back. +/// +/// Uses only core integer expressions so it holds under every feature set +/// (e.g. `no_function`, `no_float`, `only_i32`). +#[test] +fn test_compact_script_operator_adjacency() { + let engine = Engine::new(); + + for source in [ + "let a = 1 < -1;", // `< -` would fuse into the invalid `<-` + "let b = 5 - -2;", // `- -` would fuse into the reserved `--` + "let c = 2 * -1;", // must stay valid + "let d = 10 % -4;", // must stay valid + "let e = 3 <= -2;", // `<=` must not be broken by the fix + ] { + // Sanity: the source itself compiles. + engine.compile(source).unwrap(); + + let compacted = engine.compact_script(source).unwrap(); + + // No operator must have fused with the following `-`. + assert!(!compacted.contains("<-") && !compacted.contains("--"), "compacted form fused an operator with `-`: {:?}", compacted); + + // The compacted form must compile back to the same program. + engine.compile(&compacted).unwrap_or_else(|e| panic!("compacted {:?} failed to recompile: {}", compacted, e)); + } +} From 61f877384116610d38491083ff0c2db2a01cc672 Mon Sep 17 00:00:00 2001 From: "Yuval (iMac-Studio)" Date: Sat, 18 Jul 2026 03:29:05 +0300 Subject: [PATCH 2/2] Fix no_std_test panic handler for current nightly `#[lang = "panic_impl"]` on a function is now a hard error on nightly ("attribute cannot be used on functions / can only be applied to foreign functions"). `#[panic_handler]` already installs the `panic_impl` lang item, so the explicit `#[lang = "panic_impl"]` is redundant. Remove it (and the now-unused `lang_items` feature) so `NoStdBuild` builds on current nightly. --- no_std/no_std_test/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/no_std/no_std_test/src/main.rs b/no_std/no_std_test/src/main.rs index 5325dc510..577b25498 100644 --- a/no_std/no_std_test/src/main.rs +++ b/no_std/no_std_test/src/main.rs @@ -3,7 +3,7 @@ #![no_main] #![no_std] -#![feature(alloc_error_handler, core_intrinsics, lang_items, link_cfg)] +#![feature(alloc_error_handler, core_intrinsics, link_cfg)] extern crate alloc; extern crate wee_alloc; @@ -35,7 +35,6 @@ fn foo(_: core::alloc::Layout) -> ! { } #[panic_handler] -#[lang = "panic_impl"] fn rust_begin_panic(_: &core::panic::PanicInfo) -> ! { core::intrinsics::abort(); }