From e4c0cd6ad14f068225d1ffc8e24f4c642f2e00ab Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Sun, 31 Aug 2025 18:58:10 +1000 Subject: [PATCH 1/5] WIP: make `control_char: impl AsChar` (#1679) --- src/bytes/mod.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/bytes/mod.rs b/src/bytes/mod.rs index 96732cd57..7bede24d0 100644 --- a/src/bytes/mod.rs +++ b/src/bytes/mod.rs @@ -725,7 +725,7 @@ where /// pub fn escaped( normal: F, - control_char: char, + control_char: impl AsChar, escapable: G, ) -> impl Parser where @@ -744,17 +744,18 @@ where } /// Parser implementation for [escaped] -pub struct Escaped { +pub struct Escaped { normal: F, escapable: G, - control_char: char, + control_char: C, e: PhantomData, } -impl, F, G> Parser for Escaped +impl, F, G, C> Parser for Escaped where I: Input + Clone + crate::traits::Offset, ::Item: crate::traits::AsChar, + C: crate::traits::AsChar, F: Parser, G: Parser, Error: ParseError, @@ -798,8 +799,8 @@ where } Err(Err::Error(_)) => { // unwrap() should be safe here since index < $i.input_len() - if i.iter_elements().next().unwrap().as_char() == self.control_char { - let next = self.control_char.len_utf8(); + if i.iter_elements().next().unwrap().as_char() == self.control_char.as_char() { + let next = self.control_char.len(); if next >= i.input_len() { if OM::Incomplete::is_streaming() { return Err(Err::Incomplete(Needed::new(1))); @@ -900,7 +901,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl AsChar, transform: G, ) -> impl Parser where @@ -924,23 +925,24 @@ where } /// Parser implementation for [escaped_transform] -pub struct EscapedTransform { +pub struct EscapedTransform { normal: F, transform: G, - control_char: char, + control_char: C, e: PhantomData, extend: PhantomData, o: PhantomData, } -impl, F, G, ExtendItem, Output> Parser - for EscapedTransform +impl, F, G, C, ExtendItem, Output> Parser + for EscapedTransform where I: Clone + crate::traits::Offset + Input, I: crate::traits::ExtendInto, >::Output: crate::traits::ExtendInto, >::Output: crate::traits::ExtendInto, ::Item: crate::traits::AsChar, + C: crate::traits::AsChar, F: Parser, G: Parser, Error: ParseError, @@ -979,8 +981,8 @@ where } Err(Err::Error(_)) => { // unwrap() should be safe here since index < $i.input_len() - if remainder.iter_elements().next().unwrap().as_char() == self.control_char { - let next = index + self.control_char.len_utf8(); + if remainder.iter_elements().next().unwrap().as_char() == self.control_char.as_char() { + let next = index + self.control_char.len(); let input_len = input.input_len(); if next >= input_len { From 767f2a7c9511ef38cc05363348bfda8fa943a906 Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Sun, 31 Aug 2025 19:17:10 +1000 Subject: [PATCH 2/5] WIP: Update more types --- src/bytes/complete.rs | 4 ++-- src/bytes/streaming.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bytes/complete.rs b/src/bytes/complete.rs index 28cbb717d..0a5ee7448 100644 --- a/src/bytes/complete.rs +++ b/src/bytes/complete.rs @@ -417,7 +417,7 @@ where /// pub fn escaped<'a, I, Error, F, G>( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, escapable: G, ) -> impl FnMut(I) -> IResult where @@ -467,7 +467,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, transform: G, ) -> impl FnMut(I) -> IResult where diff --git a/src/bytes/streaming.rs b/src/bytes/streaming.rs index e240b5e52..905e9fe85 100644 --- a/src/bytes/streaming.rs +++ b/src/bytes/streaming.rs @@ -431,7 +431,7 @@ where /// pub fn escaped( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, escapable: G, ) -> impl FnMut(I) -> IResult where @@ -480,7 +480,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, transform: G, ) -> impl FnMut(I) -> IResult where From 461e41b1025b0e089114512b123220d0e65621b6 Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Sun, 31 Aug 2025 19:18:16 +1000 Subject: [PATCH 3/5] Add tests from #1679, use bytes always when testing with bytes. --- src/bytes/tests.rs | 50 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/src/bytes/tests.rs b/src/bytes/tests.rs index 9e8b9d38d..1f3c421f7 100644 --- a/src/bytes/tests.rs +++ b/src/bytes/tests.rs @@ -147,17 +147,17 @@ fn to_s(i: Vec) -> String { #[cfg(feature = "alloc")] #[test] fn escape_transform() { - use crate::Parser; + use crate::{bytes::complete::is_not, Parser}; fn esc(i: &[u8]) -> IResult<&[u8], String> { map( escaped_transform( alpha, - '\\', + b'\\', alt(( - value(&b"\\"[..], tag("\\")), - value(&b"\""[..], tag("\"")), - value(&b"\n"[..], tag("n")), + value(b"\\".as_slice(), tag(b"\\".as_slice())), + value(b"\"".as_slice(), tag(b"\"".as_slice())), + value(b"\n".as_slice(), tag(b"n".as_slice())), )), ), to_s, @@ -199,10 +199,10 @@ fn escape_transform() { map( escaped_transform( alpha, - '&', + b'&', alt(( - value("è".as_bytes(), tag("egrave;")), - value("à".as_bytes(), tag("agrave;")), + value("è".as_bytes(), tag(b"egrave;".as_slice())), + value("à".as_bytes(), tag(b"agrave;".as_slice())), )), ), to_s, @@ -217,6 +217,40 @@ fn escape_transform() { esc2(&b"abèDàEF;"[..]), Ok((&b";"[..], String::from("abèDàEF"))) ); + + const FEND: u8 = 0xC0; + const FESC: u8 = 0xDB; + const TFEND: u8 = 0xDC; + const TFESC: u8 = 0xDD; + + // Escapes containing invalid UTF-8 sequences + // https://github.com/rust-bakery/nom/issues/1679 + fn esc3(i: &[u8]) -> IResult<&[u8], Vec> { + escaped_transform( + is_not([FESC].as_slice()), + FESC, + alt(( + value(&[FEND][..], tag(&[TFEND][..])), + value(&[FESC][..], tag(&[TFESC][..])), + )), + )(i) + } + + assert_eq!( + esc3(&[0x61, 0x62, FESC, TFEND, 0x63, 0x64, 0x65]), + Ok((&[][..], vec![0x61, 0x62, FEND, 0x63, 0x64, 0x65])), + ); + assert_eq!( + esc3(&[0x61, 0x62, 0x63]), + Ok((&[][..], vec![0x61, 0x62, 0x63])), + ); + assert_eq!( + esc3(&[0x61, FESC, 0x00, TFEND, 0x63, 0x64]), + Err(Err::Error(error_position!( + &[0x00, TFEND, 0x63, 0x64][..], + ErrorKind::Tag + ))), + ); } #[cfg(feature = "std")] From 3af9cb6203a8e2823db070867da0643140b9d9d7 Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Sun, 31 Aug 2025 20:10:12 +1000 Subject: [PATCH 4/5] Make `satisfy`, `one_of` and `none_of` use `AsChar` for their predicate (#1679). This is potentially API breaking, as it won't silently coerece `char` to `u8` when working with `[u8]`. --- benchmarks/benches/arithmetic.rs | 4 ++-- src/bytes/tests.rs | 22 ++++++++++++++++++++-- src/character/complete.rs | 18 ++++++++++++------ src/character/mod.rs | 31 +++++++++++++++++-------------- src/character/streaming.rs | 18 ++++++++++++------ src/character/tests.rs | 12 ++++++------ 6 files changed, 69 insertions(+), 36 deletions(-) diff --git a/benchmarks/benches/arithmetic.rs b/benchmarks/benches/arithmetic.rs index cc497ccf8..fee7649f9 100644 --- a/benchmarks/benches/arithmetic.rs +++ b/benchmarks/benches/arithmetic.rs @@ -40,7 +40,7 @@ fn term(input: &[u8]) -> IResult<&[u8], i64> { pair(one_of("*/"), factor), move || init, |acc, (op, val)| { - if op == '*' { + if op == b'*' { acc * val } else { acc / val @@ -57,7 +57,7 @@ fn expr(input: &[u8]) -> IResult<&[u8], i64> { pair(one_of("+-"), term), move || init, |acc, (op, val)| { - if op == '+' { + if op == b'+' { acc + val } else { acc - val diff --git a/src/bytes/tests.rs b/src/bytes/tests.rs index 1f3c421f7..b9e6c9029 100644 --- a/src/bytes/tests.rs +++ b/src/bytes/tests.rs @@ -73,7 +73,7 @@ fn escaping() { use crate::character::streaming::one_of; fn esc(i: &[u8]) -> IResult<&[u8], &[u8]> { - escaped(alpha, '\\', one_of("\"n\\"))(i) + escaped(alpha, b'\\', one_of(b"\"n\\".as_slice()))(i) } assert_eq!(esc(&b"abcd;"[..]), Ok((&b";"[..], &b"abcd"[..]))); assert_eq!(esc(&b"ab\\\"cd;"[..]), Ok((&b";"[..], &b"ab\\\"cd"[..]))); @@ -97,9 +97,27 @@ fn escaping() { ); fn esc2(i: &[u8]) -> IResult<&[u8], &[u8]> { - escaped(digit, '\\', one_of("\"n\\"))(i) + escaped(digit, b'\\', one_of(&b"\"n\\"[..]))(i) } assert_eq!(esc2(&b"12\\nnn34"[..]), Ok((&b"nn34"[..], &b"12\\n"[..]))); + + // Escapes containing invalid UTF-8 sequences + // https://github.com/rust-bakery/nom/issues/1679 + fn esc3(i: &[u8]) -> IResult<&[u8], &[u8]> { + escaped(digit, 0xDB, one_of(&b"\xDB\xDC\xDD"[..]))(i) + } + assert_eq!( + esc3(&b"12\xDB\xDC34;"[..]), + Ok((b";".as_slice(), b"12\xDB\xDC34".as_slice())) + ); + assert_eq!( + esc3(&b"12\xDC34;"[..]), + Ok((b"\xDC34;".as_slice(), b"12".as_slice())) + ); + assert_eq!( + esc3(&b"12\xDB\xDC\xDC\xDC34"[..]), + Ok((b"\xDC\xDC34".as_slice(), b"12\xDB\xDC".as_slice())) + ); } #[cfg(feature = "alloc")] diff --git a/src/character/complete.rs b/src/character/complete.rs index f794d9d11..a6dcba2c6 100644 --- a/src/character/complete.rs +++ b/src/character/complete.rs @@ -54,11 +54,13 @@ where /// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); /// assert_eq!(parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); /// ``` -pub fn satisfy>(predicate: F) -> impl FnMut(I) -> IResult +pub fn satisfy>( + predicate: F, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - F: Fn(char) -> bool, + F: Fn(::Item) -> bool, { let mut parser = super::satisfy(predicate); move |i: I| parser.process::>(i) @@ -76,11 +78,13 @@ where /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::OneOf)))); /// ``` -pub fn one_of>(list: T) -> impl FnMut(I) -> IResult +pub fn one_of>( + list: T, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { let mut parser = super::one_of(list); move |i: I| parser.process::>(i) @@ -98,11 +102,13 @@ where /// assert_eq!(none_of::<_, _, (&str, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::NoneOf)))); /// ``` -pub fn none_of>(list: T) -> impl FnMut(I) -> IResult +pub fn none_of>( + list: T, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { let mut parser = super::none_of(list); move |i: I| parser.process::>(i) diff --git a/src/character/mod.rs b/src/character/mod.rs index 8bdae4312..0b60983ac 100644 --- a/src/character/mod.rs +++ b/src/character/mod.rs @@ -154,11 +154,11 @@ where /// ``` pub fn satisfy>( predicate: F, -) -> impl Parser +) -> impl Parser::Item, Error = Error> where I: Input, ::Item: AsChar, - F: Fn(char) -> bool, + F: Fn(::Item) -> bool, { Satisfy { predicate, @@ -176,10 +176,10 @@ impl, F, MakeError> Parser for Satisfy where I: Input, ::Item: AsChar, - F: Fn(char) -> bool, + F: Fn(::Item) -> bool, MakeError: Fn(I) -> Error, { - type Output = char; + type Output = ::Item; type Error = Error; #[inline(always)] @@ -188,9 +188,8 @@ where i: I, ) -> crate::PResult { match (i).iter_elements().next().map(|t| { - let c = t.as_char(); - let b = (self.predicate)(c); - (c, b) + let b = (self.predicate)(t); + (t, b) }) { None => { if OM::Incomplete::is_streaming() { @@ -200,7 +199,7 @@ where } } Some((_, false)) => Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))), - Some((c, true)) => Ok((i.take_from(c.len()), OM::Output::bind(|| c.as_char()))), + Some((c, true)) => Ok((i.take_from(c.len()), OM::Output::bind(|| c))), } } } @@ -216,14 +215,16 @@ where /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::OneOf)))); /// ``` -pub fn one_of>(list: T) -> impl Parser +pub fn one_of>( + list: T, +) -> impl Parser::Item, Error = Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { Satisfy { - predicate: move |c: char| list.find_token(c), + predicate: move |c| list.find_token(c), make_error: move |i| Error::from_error_kind(i, ErrorKind::OneOf), } } @@ -239,14 +240,16 @@ where /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn none_of>(list: T) -> impl Parser +pub fn none_of>( + list: T, +) -> impl Parser::Item, Error = Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { Satisfy { - predicate: move |c: char| !list.find_token(c), + predicate: move |c| !list.find_token(c), make_error: move |i| Error::from_error_kind(i, ErrorKind::NoneOf), } } diff --git a/src/character/streaming.rs b/src/character/streaming.rs index 9ee48d49b..5b310d42e 100644 --- a/src/character/streaming.rs +++ b/src/character/streaming.rs @@ -53,11 +53,13 @@ where /// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); /// assert_eq!(parser(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn satisfy>(cond: F) -> impl FnMut(I) -> IResult +pub fn satisfy>( + cond: F, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - F: Fn(char) -> bool, + F: Fn(::Item) -> bool, { let mut parser = super::satisfy(cond); move |i: I| parser.process::>(i) @@ -75,11 +77,13 @@ where /// assert_eq!(one_of::<_, _, (_, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn one_of>(list: T) -> impl FnMut(I) -> IResult +pub fn one_of>( + list: T, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { let mut parser = super::one_of(list); move |i: I| parser.process::>(i) @@ -97,11 +101,13 @@ where /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn none_of>(list: T) -> impl FnMut(I) -> IResult +pub fn none_of>( + list: T, +) -> impl FnMut(I) -> IResult::Item, Error> where I: Input, ::Item: AsChar, - T: FindToken, + T: FindToken<::Item>, { let mut parser = super::none_of(list); move |i: I| parser.process::>(i) diff --git a/src/character/tests.rs b/src/character/tests.rs index ef3b02a0d..769255e1f 100644 --- a/src/character/tests.rs +++ b/src/character/tests.rs @@ -4,12 +4,12 @@ use crate::internal::{Err, IResult}; #[test] fn one_of_test() { - fn f(i: &[u8]) -> IResult<&[u8], char> { - one_of("ab")(i) + fn f(i: &[u8]) -> IResult<&[u8], u8> { + one_of(&"ab"[..])(i) } let a = &b"abcd"[..]; - assert_eq!(f(a), Ok((&b"bcd"[..], 'a'))); + assert_eq!(f(a), Ok((&b"bcd"[..], b'a'))); let b = &b"cde"[..]; assert_eq!(f(b), Err(Err::Error(error_position!(b, ErrorKind::OneOf)))); @@ -24,15 +24,15 @@ fn one_of_test() { #[test] fn none_of_test() { - fn f(i: &[u8]) -> IResult<&[u8], char> { - none_of("ab")(i) + fn f(i: &[u8]) -> IResult<&[u8], u8> { + none_of(&b"ab"[..])(i) } let a = &b"abcd"[..]; assert_eq!(f(a), Err(Err::Error(error_position!(a, ErrorKind::NoneOf)))); let b = &b"cde"[..]; - assert_eq!(f(b), Ok((&b"de"[..], 'c'))); + assert_eq!(f(b), Ok((&b"de"[..], b'c'))); } #[test] From 8d8509dabd60a97d6d4a5c3672d259bfa804871f Mon Sep 17 00:00:00 2001 From: Michael Farrell Date: Wed, 3 Sep 2025 11:36:21 +1000 Subject: [PATCH 5/5] Roll back `satisfy`, `one_of` and `none_of` to using `char` as a predicate, while fixing the issue of silent UTF-8 usage (#1679) --- benchmarks/benches/arithmetic.rs | 4 +- src/character/complete.rs | 18 ++-- src/character/mod.rs | 60 ++++++----- src/character/streaming.rs | 18 ++-- src/character/tests.rs | 180 +++++++++++++++++++++++++++++-- 5 files changed, 221 insertions(+), 59 deletions(-) diff --git a/benchmarks/benches/arithmetic.rs b/benchmarks/benches/arithmetic.rs index fee7649f9..cc497ccf8 100644 --- a/benchmarks/benches/arithmetic.rs +++ b/benchmarks/benches/arithmetic.rs @@ -40,7 +40,7 @@ fn term(input: &[u8]) -> IResult<&[u8], i64> { pair(one_of("*/"), factor), move || init, |acc, (op, val)| { - if op == b'*' { + if op == '*' { acc * val } else { acc / val @@ -57,7 +57,7 @@ fn expr(input: &[u8]) -> IResult<&[u8], i64> { pair(one_of("+-"), term), move || init, |acc, (op, val)| { - if op == b'+' { + if op == '+' { acc + val } else { acc - val diff --git a/src/character/complete.rs b/src/character/complete.rs index a6dcba2c6..f794d9d11 100644 --- a/src/character/complete.rs +++ b/src/character/complete.rs @@ -54,13 +54,11 @@ where /// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); /// assert_eq!(parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); /// ``` -pub fn satisfy>( - predicate: F, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn satisfy>(predicate: F) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - F: Fn(::Item) -> bool, + F: Fn(char) -> bool, { let mut parser = super::satisfy(predicate); move |i: I| parser.process::>(i) @@ -78,13 +76,11 @@ where /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::OneOf)))); /// ``` -pub fn one_of>( - list: T, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn one_of>(list: T) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { let mut parser = super::one_of(list); move |i: I| parser.process::>(i) @@ -102,13 +98,11 @@ where /// assert_eq!(none_of::<_, _, (&str, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::NoneOf)))); /// ``` -pub fn none_of>( - list: T, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn none_of>(list: T) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { let mut parser = super::none_of(list); move |i: I| parser.process::>(i) diff --git a/src/character/mod.rs b/src/character/mod.rs index 0b60983ac..2d47b951e 100644 --- a/src/character/mod.rs +++ b/src/character/mod.rs @@ -138,27 +138,34 @@ where } } -/// Recognizes one character and checks that it satisfies a predicate +/// Recognizes one character or byte and checks that it satisfies a predicate. /// /// # Example /// /// ``` /// # use nom::{Err, error::{ErrorKind, Error}, Needed, IResult}; /// # use nom::character::complete::satisfy; -/// fn parser(i: &str) -> IResult<&str, char> { +/// fn char_parser(i: &str) -> IResult<&str, char> { /// satisfy(|c| c == 'a' || c == 'b')(i) /// } -/// assert_eq!(parser("abc"), Ok(("bc", 'a'))); -/// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); -/// assert_eq!(parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); +/// assert_eq!(char_parser("abc"), Ok(("bc", 'a'))); +/// assert_eq!(char_parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); +/// assert_eq!(char_parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); +/// +/// fn byte_parser(i: &[u8]) -> IResult<&[u8], char> { +/// satisfy(|c| c == 'a' || c == 'b')(i) +/// } +/// assert_eq!(byte_parser(b"abc"), Ok((&b"bc"[..], 'a'))); +/// assert_eq!(byte_parser(b"cd"), Err(Err::Error(Error::new(&b"cd"[..], ErrorKind::Satisfy)))); +/// assert_eq!(byte_parser(b""), Err(Err::Error(Error::new(&b""[..], ErrorKind::Satisfy)))); /// ``` pub fn satisfy>( predicate: F, -) -> impl Parser::Item, Error = Error> +) -> impl Parser where I: Input, ::Item: AsChar, - F: Fn(::Item) -> bool, + F: Fn(char) -> bool, { Satisfy { predicate, @@ -176,10 +183,10 @@ impl, F, MakeError> Parser for Satisfy where I: Input, ::Item: AsChar, - F: Fn(::Item) -> bool, + F: Fn(char) -> bool, MakeError: Fn(I) -> Error, { - type Output = ::Item; + type Output = char; type Error = Error; #[inline(always)] @@ -188,8 +195,9 @@ where i: I, ) -> crate::PResult { match (i).iter_elements().next().map(|t| { - let b = (self.predicate)(t); - (t, b) + let c = t.as_char(); + let b = (self.predicate)(c); + (c, t.len(), b) }) { None => { if OM::Incomplete::is_streaming() { @@ -198,13 +206,13 @@ where Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))) } } - Some((_, false)) => Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))), - Some((c, true)) => Ok((i.take_from(c.len()), OM::Output::bind(|| c))), + Some((_, _, false)) => Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))), + Some((c, len, true)) => Ok((i.take_from(len), OM::Output::bind(|| c))), } } } -/// Recognizes one of the provided characters. +/// Recognizes one of the provided characters or bytes. /// /// # Example /// @@ -214,14 +222,16 @@ where /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("abc")("b"), Ok(("", 'b'))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::OneOf)))); +/// +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"abc"[..])(b"b"), Ok((&b""[..], 'b'))); +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b"bc"), Err(Err::Error((&b"bc"[..], ErrorKind::OneOf)))); +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b""), Err(Err::Error((&b""[..], ErrorKind::OneOf)))); /// ``` -pub fn one_of>( - list: T, -) -> impl Parser::Item, Error = Error> +pub fn one_of>(list: T) -> impl Parser where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { Satisfy { predicate: move |c| list.find_token(c), @@ -229,7 +239,7 @@ where } } -//. Recognizes a character that is not in the provided characters. +/// Recognizes a character or byte that is not in the provided characters or bytes. /// /// # Example /// @@ -239,14 +249,16 @@ where /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("abc")("z"), Ok(("", 'z'))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); +/// +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"abc"[..])(b"z"), Ok((&b""[..], 'z'))); +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"ab"[..])(b"a"), Err(Err::Error((&b"a"[..], ErrorKind::NoneOf)))); +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn none_of>( - list: T, -) -> impl Parser::Item, Error = Error> +pub fn none_of>(list: T) -> impl Parser where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { Satisfy { predicate: move |c| !list.find_token(c), @@ -254,7 +266,7 @@ where } } -// Matches one byte as a character. Note that the input type will +/// Matches one byte as a character. Note that the input type will /// accept a `str`, but not a `&[u8]`, unlike many other nom parsers. /// /// # Example diff --git a/src/character/streaming.rs b/src/character/streaming.rs index 5b310d42e..9ee48d49b 100644 --- a/src/character/streaming.rs +++ b/src/character/streaming.rs @@ -53,13 +53,11 @@ where /// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); /// assert_eq!(parser(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn satisfy>( - cond: F, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn satisfy>(cond: F) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - F: Fn(::Item) -> bool, + F: Fn(char) -> bool, { let mut parser = super::satisfy(cond); move |i: I| parser.process::>(i) @@ -77,13 +75,11 @@ where /// assert_eq!(one_of::<_, _, (_, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn one_of>( - list: T, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn one_of>(list: T) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { let mut parser = super::one_of(list); move |i: I| parser.process::>(i) @@ -101,13 +97,11 @@ where /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); /// ``` -pub fn none_of>( - list: T, -) -> impl FnMut(I) -> IResult::Item, Error> +pub fn none_of>(list: T) -> impl FnMut(I) -> IResult where I: Input, ::Item: AsChar, - T: FindToken<::Item>, + T: FindToken, { let mut parser = super::none_of(list); move |i: I| parser.process::>(i) diff --git a/src/character/tests.rs b/src/character/tests.rs index 769255e1f..aa36e6a09 100644 --- a/src/character/tests.rs +++ b/src/character/tests.rs @@ -4,35 +4,197 @@ use crate::internal::{Err, IResult}; #[test] fn one_of_test() { - fn f(i: &[u8]) -> IResult<&[u8], u8> { - one_of(&"ab"[..])(i) + // Tip: matching a [u8] with a one_of(str) is probably an error, and won't handle multi-byte + // UTF-8 sequences in the `list`, and may lead to unexpected behaviour. + fn f(i: &[u8]) -> IResult<&[u8], char> { + one_of("abç")(i) } let a = &b"abcd"[..]; - assert_eq!(f(a), Ok((&b"bcd"[..], b'a'))); + assert_eq!(f(a), Ok((&b"bcd"[..], 'a'))); let b = &b"cde"[..]; assert_eq!(f(b), Err(Err::Error(error_position!(b, ErrorKind::OneOf)))); - fn utf8(i: &str) -> IResult<&str, char> { + // This doesn't match because b"ç" == [0xC3, 0xA7] but 'ç' == 0xe7. + let c = "çde".as_bytes(); + assert_eq!(f(c), Err(Err::Error(error_position!(c, ErrorKind::OneOf)))); + + // ...but this will match. + let d = &b"\xE7de"[..]; + assert_eq!(f(d), Ok((&d[1..], '\u{E7}'))); + + // A one_of([u8]) should be used with [u8] inputs. + fn f2(i: &[u8]) -> IResult<&[u8], char> { + // abç + one_of(&b"ab\xC3\xA7"[..])(i) + } + + let a = &b"abcd"[..]; + assert_eq!(f2(a), Ok((&b"bcd"[..], 'a'))); + + let b = &b"cde"[..]; + assert_eq!(f2(b), Err(Err::Error(error_position!(b, ErrorKind::OneOf)))); + + // This will match, but only on the first byte of the ç sequence + let c = &b"\xC3\xA7de"[..]; + assert_eq!(f2(c), Ok((&c[1..], '\u{C3}'))); + + // Using from the second byte of the ç sequence should also match + assert_eq!(f2(&c[1..]), Ok((&c[2..], '\u{A7}'))); + + // A one_of(str) should match multi-byte sequences on str inputs. + fn utf8_str(i: &str) -> IResult<&str, char> { + one_of("+\u{FF0B}")(i) + } + assert_eq!(utf8_str("+"), Ok(("", '+'))); + assert_eq!(utf8_str("\u{FF0B}"), Ok(("", '\u{FF0B}'))); + + // Don't match on multi-byte sequences with the same prefix + let ff0c = "\u{FF0C}"; + assert_eq!( + utf8_str(ff0c), + Err(Err::Error(error_position!(ff0c, ErrorKind::OneOf))) + ); + + // Matching a [u8] with one_of(str) should only match for single-byte + // codepoints. Using one_of in this way is probably an error. + fn utf8_bytes_with_str_list(i: &[u8]) -> IResult<&[u8], char> { one_of("+\u{FF0B}")(i) } - assert!(utf8("+").is_ok()); - assert!(utf8("\u{FF0B}").is_ok()); + assert_eq!(utf8_bytes_with_str_list(&b"+"[..]), Ok((&b""[..], '+'))); + let ff0b = &b"\xEF\xBC\x8B"[..]; + assert_eq!( + utf8_bytes_with_str_list(ff0b), + Err(Err::Error(error_position!(ff0b, ErrorKind::OneOf))) + ); + + // one_of([u8]) should not match on UTF-8 sequences, and should allow invalid sequences + fn utf8_bytes(i: &[u8]) -> IResult<&[u8], char> { + one_of(&b"+\xDC\xEF\xBC\x8B\0\x8C"[..])(i) + } + + assert_eq!(utf8_bytes(&b"+"[..]), Ok((&b""[..], '+'))); + assert_eq!(utf8_bytes(&b"\0"[..]), Ok((&b""[..], '\0'))); + // Because this is a one_of([u8]), we can match on bytes + assert_eq!(utf8_bytes(ff0b), Ok((&ff0b[1..], '\u{EF}'))); + + let dc00 = &b"\xDC\01234"[..]; + assert_eq!(utf8_bytes(dc00), Ok((&dc00[1..], '\u{DC}'))); } #[test] fn none_of_test() { - fn f(i: &[u8]) -> IResult<&[u8], u8> { - none_of(&b"ab"[..])(i) + // Tip: matching a [u8] with a none_of(str) is probably an error, and won't handle multi-byte + // UTF-8 sequences in the `list`. + fn f(i: &[u8]) -> IResult<&[u8], char> { + none_of("ab\u{E7}")(i) } let a = &b"abcd"[..]; assert_eq!(f(a), Err(Err::Error(error_position!(a, ErrorKind::NoneOf)))); let b = &b"cde"[..]; - assert_eq!(f(b), Ok((&b"de"[..], b'c'))); + assert_eq!(f(b), Ok((&b"de"[..], 'c'))); + + // This doesn't match none_of because b"ç" == [0xC3, 0xA7] but 'ç' == 0xe7. + let c = "\u{E7}de".as_bytes(); + assert_eq!(f(c), Ok((&c[1..], '\u{C3}'))); + + // This will match none_of because b"\u{7f00}" == [0xE7, 0xBC, 0x80]. + let d = "\u{7f00}".as_bytes(); + assert_eq!(f(d), Err(Err::Error(error_position!(d, ErrorKind::NoneOf)))); + + // A one_of([u8]) should be used with [u8] inputs. + fn f2(i: &[u8]) -> IResult<&[u8], char> { + // abç + none_of(&b"ab\xC3\xA7"[..])(i) + } + + let a = &b"abcd"[..]; + assert_eq!( + f2(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = &b"cde"[..]; + assert_eq!(f2(b), Ok((&b"de"[..], 'c'))); + + let c = "\u{E7}de".as_bytes(); + assert_eq!( + f2(c), + Err(Err::Error(error_position!(c, ErrorKind::NoneOf))) + ); + + let d = &b"\xE7de"[..]; + assert_eq!(f(d), Err(Err::Error(error_position!(d, ErrorKind::NoneOf)))); + + let e = "\u{A7}de".as_bytes(); + assert_eq!(f2(e), Ok((&e[1..], '\u{C2}'))); + + // A none_of(str) should match multi-byte sequences on str inputs. + fn utf8_str(i: &str) -> IResult<&str, char> { + none_of("+\u{FF0B}")(i) + } + + let a = "+"; + assert_eq!( + utf8_str(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = "\u{FF0B}"; + assert_eq!( + utf8_str(b), + Err(Err::Error(error_position!(b, ErrorKind::NoneOf))) + ); + + // Multi-byte sequence with the same prefix + let ff0c = "\u{FF0C}"; + assert_eq!(utf8_str(ff0c), Ok(("", '\u{FF0C}'))); + + // Matching a [u8] with none_of(str) should only match for single-byte + // codepoints. Using one_of in this way is probably an error. + fn utf8_bytes_with_str_list(i: &[u8]) -> IResult<&[u8], char> { + none_of("+\u{FF0B}")(i) + } + + let a = &b"+"[..]; + assert_eq!( + utf8_bytes_with_str_list(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = "\u{FF0B}".as_bytes(); + assert_eq!(utf8_bytes_with_str_list(b), Ok((&b[1..], '\u{EF}'))); + + // one_of([u8]) should not match on UTF-8 sequences, and should allow invalid sequences + fn utf8_bytes(i: &[u8]) -> IResult<&[u8], char> { + none_of(&b"+\xDC\xEF\xBC\x8B\0\x8C"[..])(i) + } + + let a = &b"+"[..]; + assert_eq!( + utf8_bytes(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = &b"\0"[..]; + assert_eq!( + utf8_bytes(b), + Err(Err::Error(error_position!(b, ErrorKind::NoneOf))) + ); + + // b'\xBC' is in the none_of list, but '\u{BC}' is encoded as [0xC2, 0xBC]. + let c = "\u{BC}".as_bytes(); + assert_eq!(utf8_bytes(c), Ok((&c[1..], '\u{C2}'))); + + let d = &b"\xDC\01234"[..]; + assert_eq!( + utf8_bytes(d), + Err(Err::Error(error_position!(d, ErrorKind::NoneOf))) + ); } #[test]