utf8_iter/
report.rs

Help
1// Copyright Mozilla Foundation
2//
3// Licensed under the Apache License (Version 2.0), or the MIT license,
4// (the "Licenses") at your option. You may not use this file except in
5// compliance with one of the Licenses. You may obtain copies of the
6// Licenses at:
7//
8//    https://www.apache.org/licenses/LICENSE-2.0
9//    https://opensource.org/licenses/MIT
10//
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the Licenses is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the Licenses for the specific language governing permissions and
15// limitations under the Licenses.
16
17use crate::in_inclusive_range8;
18use crate::UTF8_DATA;
19use core::fmt::Formatter;
20use core::iter::FusedIterator;
21
22/// A type for signaling UTF-8 errors.
23///
24/// Note: `core::error::Error` is not implemented due to implementing it
25/// being an [unstable feature][1] at the time of writing.
26///
27/// [1]: https://github.com/rust-lang/rust/issues/103765
28#[derive(Debug, PartialEq)]
29#[non_exhaustive]
30pub struct Utf8CharsError;
31
32impl core::fmt::Display for Utf8CharsError {
33    fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), core::fmt::Error> {
34        write!(f, "byte sequence not well-formed UTF-8")
35    }
36}
37
38/// Iterator by `Result<char,Utf8CharsError>` over `&[u8]` that contains
39/// potentially-invalid UTF-8. There is exactly one `Utf8CharsError` per
40/// each error as defined by the WHATWG Encoding Standard.
41///
42/// ```
43/// let s = b"a\xFFb\xFF\x80c\xF0\x9F\xA4\xA6\xF0\x9F\xA4\xF0\x9F\xF0d";
44/// let plain = utf8_iter::Utf8Chars::new(s);
45/// let reporting = utf8_iter::ErrorReportingUtf8Chars::new(s);
46/// assert!(plain.eq(reporting.map(|r| r.unwrap_or('\u{FFFD}'))));
47/// ```
48#[derive(Debug, Clone)]
49pub struct ErrorReportingUtf8Chars<'a> {
50    remaining: &'a [u8],
51}
52
53impl<'a> ErrorReportingUtf8Chars<'a> {
54    #[inline(always)]
55    /// Creates the iterator from a byte slice.
56    pub fn new(bytes: &'a [u8]) -> Self {
57        ErrorReportingUtf8Chars::<'a> { remaining: bytes }
58    }
59
60    /// Views the current remaining data in the iterator as a subslice
61    /// of the original slice.
62    #[inline(always)]
63    pub fn as_slice(&self) -> &'a [u8] {
64        self.remaining
65    }
66
67    #[inline(never)]
68    fn next_fallback(&mut self) -> Option<Result<char, Utf8CharsError>> {
69        if self.remaining.is_empty() {
70            return None;
71        }
72        let first = self.remaining[0];
73        if first < 0x80 {
74            self.remaining = &self.remaining[1..];
75            return Some(Ok(char::from(first)));
76        }
77        if !in_inclusive_range8(first, 0xC2, 0xF4) || self.remaining.len() == 1 {
78            self.remaining = &self.remaining[1..];
79            return Some(Err(Utf8CharsError));
80        }
81        let second = self.remaining[1];
82        let (lower_bound, upper_bound) = match first {
83            0xE0 => (0xA0, 0xBF),
84            0xED => (0x80, 0x9F),
85            0xF0 => (0x90, 0xBF),
86            0xF4 => (0x80, 0x8F),
87            _ => (0x80, 0xBF),
88        };
89        if !in_inclusive_range8(second, lower_bound, upper_bound) {
90            self.remaining = &self.remaining[1..];
91            return Some(Err(Utf8CharsError));
92        }
93        if first < 0xE0 {
94            self.remaining = &self.remaining[2..];
95            let point = ((u32::from(first) & 0x1F) << 6) | (u32::from(second) & 0x3F);
96            return Some(Ok(unsafe { char::from_u32_unchecked(point) }));
97        }
98        if self.remaining.len() == 2 {
99            self.remaining = &self.remaining[2..];
100            return Some(Err(Utf8CharsError));
101        }
102        let third = self.remaining[2];
103        if !in_inclusive_range8(third, 0x80, 0xBF) {
104            self.remaining = &self.remaining[2..];
105            return Some(Err(Utf8CharsError));
106        }
107        if first < 0xF0 {
108            self.remaining = &self.remaining[3..];
109            let point = ((u32::from(first) & 0xF) << 12)
110                | ((u32::from(second) & 0x3F) << 6)
111                | (u32::from(third) & 0x3F);
112            return Some(Ok(unsafe { char::from_u32_unchecked(point) }));
113        }
114        // At this point, we have a valid 3-byte prefix of a
115        // four-byte sequence that has to be incomplete, because
116        // otherwise `next()` would have succeeded.
117        self.remaining = &self.remaining[3..];
118        Some(Err(Utf8CharsError))
119    }
120}
121
122impl<'a> Iterator for ErrorReportingUtf8Chars<'a> {
123    type Item = Result<char, Utf8CharsError>;
124
125    #[inline]
126    fn next(&mut self) -> Option<Result<char, Utf8CharsError>> {
127        // This loop is only broken out of as goto forward
128        #[allow(clippy::never_loop)]
129        loop {
130            if self.remaining.len() < 4 {
131                break;
132            }
133            let first = self.remaining[0];
134            if first < 0x80 {
135                self.remaining = &self.remaining[1..];
136                return Some(Ok(char::from(first)));
137            }
138            let second = self.remaining[1];
139            if in_inclusive_range8(first, 0xC2, 0xDF) {
140                if !in_inclusive_range8(second, 0x80, 0xBF) {
141                    break;
142                }
143                let point = ((u32::from(first) & 0x1F) << 6) | (u32::from(second) & 0x3F);
144                self.remaining = &self.remaining[2..];
145                return Some(Ok(unsafe { char::from_u32_unchecked(point) }));
146            }
147            // This table-based formulation was benchmark-based in encoding_rs,
148            // but it hasn't been re-benchmarked in this iterator context.
149            let third = self.remaining[2];
150            if first < 0xF0 {
151                if ((UTF8_DATA.table[usize::from(second)]
152                    & UTF8_DATA.table[usize::from(first) + 0x80])
153                    | (third >> 6))
154                    != 2
155                {
156                    break;
157                }
158                let point = ((u32::from(first) & 0xF) << 12)
159                    | ((u32::from(second) & 0x3F) << 6)
160                    | (u32::from(third) & 0x3F);
161                self.remaining = &self.remaining[3..];
162                return Some(Ok(unsafe { char::from_u32_unchecked(point) }));
163            }
164            let fourth = self.remaining[3];
165            if (u16::from(
166                UTF8_DATA.table[usize::from(second)] & UTF8_DATA.table[usize::from(first) + 0x80],
167            ) | u16::from(third >> 6)
168                | (u16::from(fourth & 0xC0) << 2))
169                != 0x202
170            {
171                break;
172            }
173            let point = ((u32::from(first) & 0x7) << 18)
174                | ((u32::from(second) & 0x3F) << 12)
175                | ((u32::from(third) & 0x3F) << 6)
176                | (u32::from(fourth) & 0x3F);
177            self.remaining = &self.remaining[4..];
178            return Some(Ok(unsafe { char::from_u32_unchecked(point) }));
179        }
180        self.next_fallback()
181    }
182}
183
184impl<'a> DoubleEndedIterator for ErrorReportingUtf8Chars<'a> {
185    #[inline]
186    fn next_back(&mut self) -> Option<Result<char, Utf8CharsError>> {
187        if self.remaining.is_empty() {
188            return None;
189        }
190        let mut attempt = 1;
191        for b in self.remaining.iter().rev() {
192            if b & 0xC0 != 0x80 {
193                let (head, tail) = self.remaining.split_at(self.remaining.len() - attempt);
194                let mut inner = ErrorReportingUtf8Chars::new(tail);
195                let candidate = inner.next();
196                if inner.as_slice().is_empty() {
197                    self.remaining = head;
198                    return candidate;
199                }
200                break;
201            }
202            if attempt == 4 {
203                break;
204            }
205            attempt += 1;
206        }
207
208        self.remaining = &self.remaining[..self.remaining.len() - 1];
209        Some(Err(Utf8CharsError))
210    }
211}
212
213impl FusedIterator for ErrorReportingUtf8Chars<'_> {}
214
215#[cfg(test)]
216mod tests {
217    use crate::ErrorReportingUtf8Chars;
218
219    // Should be a static assert, but not taking a dependency for this.
220    #[test]
221    fn test_size() {
222        assert_eq!(
223            core::mem::size_of::<Option<<ErrorReportingUtf8Chars<'_> as Iterator>::Item>>(),
224            core::mem::size_of::<Option<char>>()
225        );
226    }
227
228    #[test]
229    fn test_eq() {
230        let a: <ErrorReportingUtf8Chars<'_> as Iterator>::Item = Ok('a');
231        let a_again: <ErrorReportingUtf8Chars<'_> as Iterator>::Item = Ok('a');
232        assert_eq!(a, a_again);
233    }
234}
utf8_iter/report.rs

utf8_iter/
report.rs
