diff --git a/src/bytes/complete.rs b/src/bytes/complete.rs index 28cbb717d..0a5ee7448 100644 --- a/src/bytes/complete.rs +++ b/src/bytes/complete.rs @@ -417,7 +417,7 @@ where /// pub fn escaped<'a, I, Error, F, G>( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, escapable: G, ) -> impl FnMut(I) -> IResult where @@ -467,7 +467,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, transform: G, ) -> impl FnMut(I) -> IResult where diff --git a/src/bytes/mod.rs b/src/bytes/mod.rs index 96732cd57..7bede24d0 100644 --- a/src/bytes/mod.rs +++ b/src/bytes/mod.rs @@ -725,7 +725,7 @@ where /// pub fn escaped( normal: F, - control_char: char, + control_char: impl AsChar, escapable: G, ) -> impl Parser where @@ -744,17 +744,18 @@ where } /// Parser implementation for [escaped] -pub struct Escaped { +pub struct Escaped { normal: F, escapable: G, - control_char: char, + control_char: C, e: PhantomData, } -impl, F, G> Parser for Escaped +impl, F, G, C> Parser for Escaped where I: Input + Clone + crate::traits::Offset, ::Item: crate::traits::AsChar, + C: crate::traits::AsChar, F: Parser, G: Parser, Error: ParseError, @@ -798,8 +799,8 @@ where } Err(Err::Error(_)) => { // unwrap() should be safe here since index < $i.input_len() - if i.iter_elements().next().unwrap().as_char() == self.control_char { - let next = self.control_char.len_utf8(); + if i.iter_elements().next().unwrap().as_char() == self.control_char.as_char() { + let next = self.control_char.len(); if next >= i.input_len() { if OM::Incomplete::is_streaming() { return Err(Err::Incomplete(Needed::new(1))); @@ -900,7 +901,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl AsChar, transform: G, ) -> impl Parser where @@ -924,23 +925,24 @@ where } /// Parser implementation for [escaped_transform] -pub struct EscapedTransform { +pub struct EscapedTransform { normal: F, transform: G, - control_char: char, + control_char: C, e: PhantomData, extend: PhantomData, o: PhantomData, } -impl, F, G, ExtendItem, Output> Parser - for EscapedTransform +impl, F, G, C, ExtendItem, Output> Parser + for EscapedTransform where I: Clone + crate::traits::Offset + Input, I: crate::traits::ExtendInto, >::Output: crate::traits::ExtendInto, >::Output: crate::traits::ExtendInto, ::Item: crate::traits::AsChar, + C: crate::traits::AsChar, F: Parser, G: Parser, Error: ParseError, @@ -979,8 +981,8 @@ where } Err(Err::Error(_)) => { // unwrap() should be safe here since index < $i.input_len() - if remainder.iter_elements().next().unwrap().as_char() == self.control_char { - let next = index + self.control_char.len_utf8(); + if remainder.iter_elements().next().unwrap().as_char() == self.control_char.as_char() { + let next = index + self.control_char.len(); let input_len = input.input_len(); if next >= input_len { diff --git a/src/bytes/streaming.rs b/src/bytes/streaming.rs index e240b5e52..905e9fe85 100644 --- a/src/bytes/streaming.rs +++ b/src/bytes/streaming.rs @@ -431,7 +431,7 @@ where /// pub fn escaped( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, escapable: G, ) -> impl FnMut(I) -> IResult where @@ -480,7 +480,7 @@ where #[cfg_attr(feature = "docsrs", doc(cfg(feature = "alloc")))] pub fn escaped_transform( normal: F, - control_char: char, + control_char: impl crate::traits::AsChar, transform: G, ) -> impl FnMut(I) -> IResult where diff --git a/src/bytes/tests.rs b/src/bytes/tests.rs index 9e8b9d38d..b9e6c9029 100644 --- a/src/bytes/tests.rs +++ b/src/bytes/tests.rs @@ -73,7 +73,7 @@ fn escaping() { use crate::character::streaming::one_of; fn esc(i: &[u8]) -> IResult<&[u8], &[u8]> { - escaped(alpha, '\\', one_of("\"n\\"))(i) + escaped(alpha, b'\\', one_of(b"\"n\\".as_slice()))(i) } assert_eq!(esc(&b"abcd;"[..]), Ok((&b";"[..], &b"abcd"[..]))); assert_eq!(esc(&b"ab\\\"cd;"[..]), Ok((&b";"[..], &b"ab\\\"cd"[..]))); @@ -97,9 +97,27 @@ fn escaping() { ); fn esc2(i: &[u8]) -> IResult<&[u8], &[u8]> { - escaped(digit, '\\', one_of("\"n\\"))(i) + escaped(digit, b'\\', one_of(&b"\"n\\"[..]))(i) } assert_eq!(esc2(&b"12\\nnn34"[..]), Ok((&b"nn34"[..], &b"12\\n"[..]))); + + // Escapes containing invalid UTF-8 sequences + // https://github.com/rust-bakery/nom/issues/1679 + fn esc3(i: &[u8]) -> IResult<&[u8], &[u8]> { + escaped(digit, 0xDB, one_of(&b"\xDB\xDC\xDD"[..]))(i) + } + assert_eq!( + esc3(&b"12\xDB\xDC34;"[..]), + Ok((b";".as_slice(), b"12\xDB\xDC34".as_slice())) + ); + assert_eq!( + esc3(&b"12\xDC34;"[..]), + Ok((b"\xDC34;".as_slice(), b"12".as_slice())) + ); + assert_eq!( + esc3(&b"12\xDB\xDC\xDC\xDC34"[..]), + Ok((b"\xDC\xDC34".as_slice(), b"12\xDB\xDC".as_slice())) + ); } #[cfg(feature = "alloc")] @@ -147,17 +165,17 @@ fn to_s(i: Vec) -> String { #[cfg(feature = "alloc")] #[test] fn escape_transform() { - use crate::Parser; + use crate::{bytes::complete::is_not, Parser}; fn esc(i: &[u8]) -> IResult<&[u8], String> { map( escaped_transform( alpha, - '\\', + b'\\', alt(( - value(&b"\\"[..], tag("\\")), - value(&b"\""[..], tag("\"")), - value(&b"\n"[..], tag("n")), + value(b"\\".as_slice(), tag(b"\\".as_slice())), + value(b"\"".as_slice(), tag(b"\"".as_slice())), + value(b"\n".as_slice(), tag(b"n".as_slice())), )), ), to_s, @@ -199,10 +217,10 @@ fn escape_transform() { map( escaped_transform( alpha, - '&', + b'&', alt(( - value("è".as_bytes(), tag("egrave;")), - value("à".as_bytes(), tag("agrave;")), + value("è".as_bytes(), tag(b"egrave;".as_slice())), + value("à".as_bytes(), tag(b"agrave;".as_slice())), )), ), to_s, @@ -217,6 +235,40 @@ fn escape_transform() { esc2(&b"abèDàEF;"[..]), Ok((&b";"[..], String::from("abèDàEF"))) ); + + const FEND: u8 = 0xC0; + const FESC: u8 = 0xDB; + const TFEND: u8 = 0xDC; + const TFESC: u8 = 0xDD; + + // Escapes containing invalid UTF-8 sequences + // https://github.com/rust-bakery/nom/issues/1679 + fn esc3(i: &[u8]) -> IResult<&[u8], Vec> { + escaped_transform( + is_not([FESC].as_slice()), + FESC, + alt(( + value(&[FEND][..], tag(&[TFEND][..])), + value(&[FESC][..], tag(&[TFESC][..])), + )), + )(i) + } + + assert_eq!( + esc3(&[0x61, 0x62, FESC, TFEND, 0x63, 0x64, 0x65]), + Ok((&[][..], vec![0x61, 0x62, FEND, 0x63, 0x64, 0x65])), + ); + assert_eq!( + esc3(&[0x61, 0x62, 0x63]), + Ok((&[][..], vec![0x61, 0x62, 0x63])), + ); + assert_eq!( + esc3(&[0x61, FESC, 0x00, TFEND, 0x63, 0x64]), + Err(Err::Error(error_position!( + &[0x00, TFEND, 0x63, 0x64][..], + ErrorKind::Tag + ))), + ); } #[cfg(feature = "std")] diff --git a/src/character/mod.rs b/src/character/mod.rs index 8bdae4312..2d47b951e 100644 --- a/src/character/mod.rs +++ b/src/character/mod.rs @@ -138,19 +138,26 @@ where } } -/// Recognizes one character and checks that it satisfies a predicate +/// Recognizes one character or byte and checks that it satisfies a predicate. /// /// # Example /// /// ``` /// # use nom::{Err, error::{ErrorKind, Error}, Needed, IResult}; /// # use nom::character::complete::satisfy; -/// fn parser(i: &str) -> IResult<&str, char> { +/// fn char_parser(i: &str) -> IResult<&str, char> { /// satisfy(|c| c == 'a' || c == 'b')(i) /// } -/// assert_eq!(parser("abc"), Ok(("bc", 'a'))); -/// assert_eq!(parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); -/// assert_eq!(parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); +/// assert_eq!(char_parser("abc"), Ok(("bc", 'a'))); +/// assert_eq!(char_parser("cd"), Err(Err::Error(Error::new("cd", ErrorKind::Satisfy)))); +/// assert_eq!(char_parser(""), Err(Err::Error(Error::new("", ErrorKind::Satisfy)))); +/// +/// fn byte_parser(i: &[u8]) -> IResult<&[u8], char> { +/// satisfy(|c| c == 'a' || c == 'b')(i) +/// } +/// assert_eq!(byte_parser(b"abc"), Ok((&b"bc"[..], 'a'))); +/// assert_eq!(byte_parser(b"cd"), Err(Err::Error(Error::new(&b"cd"[..], ErrorKind::Satisfy)))); +/// assert_eq!(byte_parser(b""), Err(Err::Error(Error::new(&b""[..], ErrorKind::Satisfy)))); /// ``` pub fn satisfy>( predicate: F, @@ -190,7 +197,7 @@ where match (i).iter_elements().next().map(|t| { let c = t.as_char(); let b = (self.predicate)(c); - (c, b) + (c, t.len(), b) }) { None => { if OM::Incomplete::is_streaming() { @@ -199,13 +206,13 @@ where Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))) } } - Some((_, false)) => Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))), - Some((c, true)) => Ok((i.take_from(c.len()), OM::Output::bind(|| c.as_char()))), + Some((_, _, false)) => Err(Err::Error(OM::Error::bind(|| (self.make_error)(i)))), + Some((c, len, true)) => Ok((i.take_from(len), OM::Output::bind(|| c))), } } } -/// Recognizes one of the provided characters. +/// Recognizes one of the provided characters or bytes. /// /// # Example /// @@ -215,6 +222,10 @@ where /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("abc")("b"), Ok(("", 'b'))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")("bc"), Err(Err::Error(("bc", ErrorKind::OneOf)))); /// assert_eq!(one_of::<_, _, (&str, ErrorKind)>("a")(""), Err(Err::Error(("", ErrorKind::OneOf)))); +/// +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"abc"[..])(b"b"), Ok((&b""[..], 'b'))); +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b"bc"), Err(Err::Error((&b"bc"[..], ErrorKind::OneOf)))); +/// assert_eq!(one_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b""), Err(Err::Error((&b""[..], ErrorKind::OneOf)))); /// ``` pub fn one_of>(list: T) -> impl Parser where @@ -223,12 +234,12 @@ where T: FindToken, { Satisfy { - predicate: move |c: char| list.find_token(c), + predicate: move |c| list.find_token(c), make_error: move |i| Error::from_error_kind(i, ErrorKind::OneOf), } } -//. Recognizes a character that is not in the provided characters. +/// Recognizes a character or byte that is not in the provided characters or bytes. /// /// # Example /// @@ -238,6 +249,10 @@ where /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("abc")("z"), Ok(("", 'z'))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("ab")("a"), Err(Err::Error(("a", ErrorKind::NoneOf)))); /// assert_eq!(none_of::<_, _, (_, ErrorKind)>("a")(""), Err(Err::Incomplete(Needed::Unknown))); +/// +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"abc"[..])(b"z"), Ok((&b""[..], 'z'))); +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"ab"[..])(b"a"), Err(Err::Error((&b"a"[..], ErrorKind::NoneOf)))); +/// assert_eq!(none_of::<_, _, (&[u8], ErrorKind)>(&b"a"[..])(b""), Err(Err::Incomplete(Needed::Unknown))); /// ``` pub fn none_of>(list: T) -> impl Parser where @@ -246,12 +261,12 @@ where T: FindToken, { Satisfy { - predicate: move |c: char| !list.find_token(c), + predicate: move |c| !list.find_token(c), make_error: move |i| Error::from_error_kind(i, ErrorKind::NoneOf), } } -// Matches one byte as a character. Note that the input type will +/// Matches one byte as a character. Note that the input type will /// accept a `str`, but not a `&[u8]`, unlike many other nom parsers. /// /// # Example diff --git a/src/character/tests.rs b/src/character/tests.rs index ef3b02a0d..aa36e6a09 100644 --- a/src/character/tests.rs +++ b/src/character/tests.rs @@ -4,8 +4,10 @@ use crate::internal::{Err, IResult}; #[test] fn one_of_test() { + // Tip: matching a [u8] with a one_of(str) is probably an error, and won't handle multi-byte + // UTF-8 sequences in the `list`, and may lead to unexpected behaviour. fn f(i: &[u8]) -> IResult<&[u8], char> { - one_of("ab")(i) + one_of("abç")(i) } let a = &b"abcd"[..]; @@ -14,18 +16,80 @@ fn one_of_test() { let b = &b"cde"[..]; assert_eq!(f(b), Err(Err::Error(error_position!(b, ErrorKind::OneOf)))); - fn utf8(i: &str) -> IResult<&str, char> { + // This doesn't match because b"ç" == [0xC3, 0xA7] but 'ç' == 0xe7. + let c = "çde".as_bytes(); + assert_eq!(f(c), Err(Err::Error(error_position!(c, ErrorKind::OneOf)))); + + // ...but this will match. + let d = &b"\xE7de"[..]; + assert_eq!(f(d), Ok((&d[1..], '\u{E7}'))); + + // A one_of([u8]) should be used with [u8] inputs. + fn f2(i: &[u8]) -> IResult<&[u8], char> { + // abç + one_of(&b"ab\xC3\xA7"[..])(i) + } + + let a = &b"abcd"[..]; + assert_eq!(f2(a), Ok((&b"bcd"[..], 'a'))); + + let b = &b"cde"[..]; + assert_eq!(f2(b), Err(Err::Error(error_position!(b, ErrorKind::OneOf)))); + + // This will match, but only on the first byte of the ç sequence + let c = &b"\xC3\xA7de"[..]; + assert_eq!(f2(c), Ok((&c[1..], '\u{C3}'))); + + // Using from the second byte of the ç sequence should also match + assert_eq!(f2(&c[1..]), Ok((&c[2..], '\u{A7}'))); + + // A one_of(str) should match multi-byte sequences on str inputs. + fn utf8_str(i: &str) -> IResult<&str, char> { + one_of("+\u{FF0B}")(i) + } + assert_eq!(utf8_str("+"), Ok(("", '+'))); + assert_eq!(utf8_str("\u{FF0B}"), Ok(("", '\u{FF0B}'))); + + // Don't match on multi-byte sequences with the same prefix + let ff0c = "\u{FF0C}"; + assert_eq!( + utf8_str(ff0c), + Err(Err::Error(error_position!(ff0c, ErrorKind::OneOf))) + ); + + // Matching a [u8] with one_of(str) should only match for single-byte + // codepoints. Using one_of in this way is probably an error. + fn utf8_bytes_with_str_list(i: &[u8]) -> IResult<&[u8], char> { one_of("+\u{FF0B}")(i) } - assert!(utf8("+").is_ok()); - assert!(utf8("\u{FF0B}").is_ok()); + assert_eq!(utf8_bytes_with_str_list(&b"+"[..]), Ok((&b""[..], '+'))); + let ff0b = &b"\xEF\xBC\x8B"[..]; + assert_eq!( + utf8_bytes_with_str_list(ff0b), + Err(Err::Error(error_position!(ff0b, ErrorKind::OneOf))) + ); + + // one_of([u8]) should not match on UTF-8 sequences, and should allow invalid sequences + fn utf8_bytes(i: &[u8]) -> IResult<&[u8], char> { + one_of(&b"+\xDC\xEF\xBC\x8B\0\x8C"[..])(i) + } + + assert_eq!(utf8_bytes(&b"+"[..]), Ok((&b""[..], '+'))); + assert_eq!(utf8_bytes(&b"\0"[..]), Ok((&b""[..], '\0'))); + // Because this is a one_of([u8]), we can match on bytes + assert_eq!(utf8_bytes(ff0b), Ok((&ff0b[1..], '\u{EF}'))); + + let dc00 = &b"\xDC\01234"[..]; + assert_eq!(utf8_bytes(dc00), Ok((&dc00[1..], '\u{DC}'))); } #[test] fn none_of_test() { + // Tip: matching a [u8] with a none_of(str) is probably an error, and won't handle multi-byte + // UTF-8 sequences in the `list`. fn f(i: &[u8]) -> IResult<&[u8], char> { - none_of("ab")(i) + none_of("ab\u{E7}")(i) } let a = &b"abcd"[..]; @@ -33,6 +97,104 @@ fn none_of_test() { let b = &b"cde"[..]; assert_eq!(f(b), Ok((&b"de"[..], 'c'))); + + // This doesn't match none_of because b"ç" == [0xC3, 0xA7] but 'ç' == 0xe7. + let c = "\u{E7}de".as_bytes(); + assert_eq!(f(c), Ok((&c[1..], '\u{C3}'))); + + // This will match none_of because b"\u{7f00}" == [0xE7, 0xBC, 0x80]. + let d = "\u{7f00}".as_bytes(); + assert_eq!(f(d), Err(Err::Error(error_position!(d, ErrorKind::NoneOf)))); + + // A one_of([u8]) should be used with [u8] inputs. + fn f2(i: &[u8]) -> IResult<&[u8], char> { + // abç + none_of(&b"ab\xC3\xA7"[..])(i) + } + + let a = &b"abcd"[..]; + assert_eq!( + f2(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = &b"cde"[..]; + assert_eq!(f2(b), Ok((&b"de"[..], 'c'))); + + let c = "\u{E7}de".as_bytes(); + assert_eq!( + f2(c), + Err(Err::Error(error_position!(c, ErrorKind::NoneOf))) + ); + + let d = &b"\xE7de"[..]; + assert_eq!(f(d), Err(Err::Error(error_position!(d, ErrorKind::NoneOf)))); + + let e = "\u{A7}de".as_bytes(); + assert_eq!(f2(e), Ok((&e[1..], '\u{C2}'))); + + // A none_of(str) should match multi-byte sequences on str inputs. + fn utf8_str(i: &str) -> IResult<&str, char> { + none_of("+\u{FF0B}")(i) + } + + let a = "+"; + assert_eq!( + utf8_str(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = "\u{FF0B}"; + assert_eq!( + utf8_str(b), + Err(Err::Error(error_position!(b, ErrorKind::NoneOf))) + ); + + // Multi-byte sequence with the same prefix + let ff0c = "\u{FF0C}"; + assert_eq!(utf8_str(ff0c), Ok(("", '\u{FF0C}'))); + + // Matching a [u8] with none_of(str) should only match for single-byte + // codepoints. Using one_of in this way is probably an error. + fn utf8_bytes_with_str_list(i: &[u8]) -> IResult<&[u8], char> { + none_of("+\u{FF0B}")(i) + } + + let a = &b"+"[..]; + assert_eq!( + utf8_bytes_with_str_list(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = "\u{FF0B}".as_bytes(); + assert_eq!(utf8_bytes_with_str_list(b), Ok((&b[1..], '\u{EF}'))); + + // one_of([u8]) should not match on UTF-8 sequences, and should allow invalid sequences + fn utf8_bytes(i: &[u8]) -> IResult<&[u8], char> { + none_of(&b"+\xDC\xEF\xBC\x8B\0\x8C"[..])(i) + } + + let a = &b"+"[..]; + assert_eq!( + utf8_bytes(a), + Err(Err::Error(error_position!(a, ErrorKind::NoneOf))) + ); + + let b = &b"\0"[..]; + assert_eq!( + utf8_bytes(b), + Err(Err::Error(error_position!(b, ErrorKind::NoneOf))) + ); + + // b'\xBC' is in the none_of list, but '\u{BC}' is encoded as [0xC2, 0xBC]. + let c = "\u{BC}".as_bytes(); + assert_eq!(utf8_bytes(c), Ok((&c[1..], '\u{C2}'))); + + let d = &b"\xDC\01234"[..]; + assert_eq!( + utf8_bytes(d), + Err(Err::Error(error_position!(d, ErrorKind::NoneOf))) + ); } #[test]