Fix compact_script fusing adjacent operators into an invalid token (< - -> <-)

no_std/no_std_test/src/main.rs

@@ -3,7 +3,7 @@
 
 #![no_main]
 #![no_std]
-#![feature(alloc_error_handler, core_intrinsics, lang_items, link_cfg)]
+#![feature(alloc_error_handler, core_intrinsics, link_cfg)]
 
 extern crate alloc;
 extern crate wee_alloc;
@@ -35,7 +35,6 @@ fn foo(_: core::alloc::Layout) -> ! {
 }
 
 #[panic_handler]
-#[lang = "panic_impl"]
 fn rust_begin_panic(_: &core::panic::PanicInfo) -> ! {
     core::intrinsics::abort();
 }

src/tokenizer.rs

@@ -2844,13 +2844,34 @@ impl Iterator for TokenIterator<'_> {
 
                     if !buf.is_empty() && !compressed.is_empty() {
                         let cur = buf.chars().next().unwrap();
-
-                        if cur == '_' || is_id_first_alphabetic(cur) || is_id_continue(cur) {
-                            let prev = compressed.chars().last().unwrap();
-
-                            if prev == '_' || is_id_first_alphabetic(prev) || is_id_continue(prev) {
-                                *compressed += " ";
-                            }
+                        let prev = compressed.chars().last().unwrap();
+
+                        // A separating space is needed when concatenating this
+                        // token onto the output would change how the boundary
+                        // re-tokenizes:
+                        //   * two identifier characters merge into one
+                        //     identifier/keyword (e.g. `let` + `x` -> `letx`);
+                        //   * the two boundary characters merge into a
+                        //     recognized operator or a reserved symbol (e.g.
+                        //     `<` + `-` -> `<-`, which the lexer rejects, or
+                        //     `<` + `=` -> `<=`).
+                        // The second case is decided by the tokenizer's own
+                        // symbol tables, so nothing has to be hardcoded here.
+                        let is_id =
+                            |c: char| c == '_' || is_id_first_alphabetic(c) || is_id_continue(c);
+
+                        let need_space = if is_id(prev) && is_id(cur) {
+                            true
+                        } else {
+                            let mut pair = SmartString::new_const();
+                            pair.push(prev);
+                            pair.push(cur);
+                            Token::lookup_symbol_from_syntax(&pair).is_some()
+                                || is_reserved_keyword_or_symbol(&pair).0
+                        };
+
+                        if need_space {
+                            *compressed += " ";
                         }
                     }
 

tests/tokens.rs

@@ -94,3 +94,34 @@ fn test_tokens_unicode_xid_ident() {
     );
     let _ = result.unwrap_err();
 }
+
+/// Regression: `Engine::compact_script` must not fuse an operator with a
+/// following character into a different token. In particular `x < -1` must not
+/// compact to `x<-1`, which the lexer rejects as the invalid operator `<-`
+/// ("This is not Go!"). The compacted output must always compile back.
+///
+/// Uses only core integer expressions so it holds under every feature set
+/// (e.g. `no_function`, `no_float`, `only_i32`).
+#[test]
+fn test_compact_script_operator_adjacency() {
+    let engine = Engine::new();
+
+    for source in [
+        "let a = 1 < -1;",  // `< -` would fuse into the invalid `<-`
+        "let b = 5 - -2;",  // `- -` would fuse into the reserved `--`
+        "let c = 2 * -1;",  // must stay valid
+        "let d = 10 % -4;", // must stay valid
+        "let e = 3 <= -2;", // `<=` must not be broken by the fix
+    ] {
+        // Sanity: the source itself compiles.
+        engine.compile(source).unwrap();
+
+        let compacted = engine.compact_script(source).unwrap();
+
+        // No operator must have fused with the following `-`.
+        assert!(!compacted.contains("<-") && !compacted.contains("--"), "compacted form fused an operator with `-`: {:?}", compacted);
+
+        // The compacted form must compile back to the same program.
+        engine.compile(&compacted).unwrap_or_else(|e| panic!("compacted {:?} failed to recompile: {}", compacted, e));
+    }
+}