utf8_iter/

str.rs

// Copyright Mozilla Foundation
//
// Licensed under the Apache License (Version 2.0), or the MIT license,
// (the "Licenses") at your option. You may not use this file except in
// compliance with one of the Licenses. You may obtain copies of the
// Licenses at:
//
//    https://www.apache.org/licenses/LICENSE-2.0
//    https://opensource.org/licenses/MIT
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the Licenses is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Licenses for the specific language governing permissions and
// limitations under the Licenses.

#![allow(unsafe_code)] // Needed to make efficient use of the invariant of `str`

use crate::Utf8Handler;

use utf_types::utf8::Ascii;
use utf_types::utf8::FourByteSequence;
use utf_types::utf8::LeadForFourByte;
use utf_types::utf8::LeadForMultiByte;
use utf_types::utf8::LeadForThreeByte;
use utf_types::utf8::ThreeByteSequence;
use utf_types::utf8::TwoByteSequence;
use utf_types::utf8::UnconstrainedContinuation;

/// Iterator over guaranteed-well-formed UTF-8 with `Utf8Handler`.
/// The `Utf8Handler::error()` is, of course, never called.
#[derive(Debug)]
pub struct CharsWithHandler<'a, H>
where
    H: Utf8Handler,
{
    /// # Safety invariant
    ///
    /// `self.inner.as_slice()` is well-formed UTF-8, except when a method that
    /// advances the `CharsWithHandler` as an iterator ()`next`, `next_back`,
    /// `next_with_minimum_lead`) is in the process of reading a UTF-8 sequence
    /// from `self.inner`.
    ///
    /// Note: For panic-safety, methods that change the state of `self.inner`
    /// must first consume a full UTF-8 sequence (such that the above-stated
    /// invariant holds again) before calling into `self.handler`, so that
    /// the invariant of `self.inner` is upheld even if `self.handler`
    /// panics.
    inner: core::slice::Iter<'a, u8>,
    handler: H,
}

impl<'a, H> CharsWithHandler<'a, H>
where
    H: Utf8Handler,
{
    #[inline(always)]
    /// Creates the iterator from a `&str` and handler.
    pub fn new(s: &'a str, handler: H) -> Self {
        CharsWithHandler::<'a, H> {
            inner: s.as_bytes().iter(),
            handler,
        }
    }

    /// Views the current remaining data in the iterator as a subslice
    /// of the original slice.
    #[inline(always)]
    pub fn as_str(&self) -> &'a str {
        // SAFETY: OK per the safety invariant of `self.inner`.
        unsafe { core::str::from_utf8_unchecked(self.inner.as_slice()) }
    }

    /// Obtains a reference to the handler.
    #[inline(always)]
    pub fn handler(&self) -> &H {
        &self.handler
    }

    /// Obtains the output for the next UTF-8 sequence whose lead byte
    /// is greater than or equal to `multi_byte_lead` ignoring UTF-8
    /// sequences before such a UTF-8 sequence. (Or `None` if the iterator
    /// is consumed without finding such a UTF-8 sequence.)
    #[inline(always)]
    pub fn next_with_minimum_lead(
        &mut self,
        multi_byte_lead: LeadForMultiByte,
    ) -> Option<H::Output> {
        let bound = multi_byte_lead.to_u8();
        loop {
            let byte = *self.inner.next()?;
            if byte < bound {
                continue;
            }
            // SAFETY: From the safety invariant of `self.inner`
            // we know that upon entry into this method `self.inner`
            // represented well-formed UTF-8.
            //
            // We know that well-formed UTF-8 consists of three kinds
            // of bytes:
            // 1. ASCII bytes
            // 2. Continuation bytes
            // 3. Lead bytes for multi-byte sequences.
            //
            // We know that ASCII bytes and continuation bytes are
            // below lead bytes for multi-byte sequences in unsigned
            // integer order.
            //
            // Since `bound` is a lead byte for a multi-byte sequence,
            // we know that ASCII bytes and continuation bytes take
            // the `continue` path above.
            //
            // Therefore, we know that `byte` must now be a lead byte
            // for a multibyte sequence. Since we know that `self.inner`
            // represented well-formed UTF-8 upon entry into this method,
            // we know that `self.inner` will yield the continuation bytes
            // that are appropriate given `byte` as the lead byte for
            // a multibyte sequence.
            //
            // INVARIANT UPHELD for `self.inner`:
            // `consume_continuations` consumes the rest of the
            // multi-byte UTF-8 sequence causing a complete sequence
            // to be consumed.
            return Some(unsafe { self.consume_continuations(byte) });
        }
    }

    /// Consumes continuations given multi-byte lead byte `first`.
    ///
    /// # Safety
    ///
    /// `first` must be a lead byte for a multi-byte UTF-8 sequence and
    /// the concatenation of `first` and `self.inner` must be well-formed
    /// UTF-8.
    ///
    /// # Safety-usable invariant
    ///
    /// This method consumes from `self.inner` the continuation bytes that
    /// belong to the multi-byte sequence started by `first`.
    #[inline(always)]
    unsafe fn consume_continuations(&mut self, first: u8) -> H::Output {
        debug_assert!(LeadForMultiByte::is(first));
        // SAFETY: Since we don't have a single-byte sequence (per the
        // safety invariant of this method) and we know the concatenation
        // of `first` and `self.inner` is well-formed UTF-8, a second
        // byte must exist.
        let second = *unsafe { self.inner.next().unwrap_unchecked() };
        if first < LeadForThreeByte::MIN as u8 {
            // INVARIANT UPHELD for `self.inner`:
            // We consumed a complete UTF-8 sequence; see "SAFETY"
            // below.
            return self.handler.two_byte(
                // SAFETY: From the invariant of this method we know that
                // `first` is a multibyte lead and `second` must be a
                // continuation since the concatenation of `first` and
                // `self.inner` was well-formed UTF-8 upon entry into
                // the method. Since `first` is also less than the
                // smallest lead for a three-byte sequence, it must be
                // a lead for a two-byte sequence. Therefore, we have
                // a two-byte UTF-8 sequence, which satisfies the invariant
                // of `TwoByteSequence::new_unchecked()`.
                unsafe { TwoByteSequence::new_unchecked(first, second) },
            );
        }
        // SAFETY: Per the above two "SAFETY" comments, we don't have
        // a single-byte or a two-byte sequence if we get this far, so
        // a third byte must exist given that the concanation of `first`
        // and `self.inner` was well-formed UTF-8 at the start of this
        // method.
        let third = *unsafe { self.inner.next().unwrap_unchecked() };
        if first < LeadForFourByte::MIN as u8 {
            // INVARIANT UPHELD for `self.inner`:
            // We consumed a complete UTF-8 sequence; see "SAFETY"
            // below.
            return self.handler.three_byte(unsafe {
                // SAFETY: From the safety invariant of this method
                // we know that the concatenation of `first` and
                // `self.inner` was well-formed UTF-8 at the start of
                // this method. The safety invariant of this method
                // rules out a single-byte sequence and we have already
                // checked for and ruled out a two-byte sequence. Most
                // recently, we ruled out a four-byte sequence. Therefore,
                // we must have a three-byte sequence, which satisfies
                // the safety invariant of `ThreeByteSequence::new_unchecked()`.
                ThreeByteSequence::new_unchecked(first, second, third)
            });
        }
        // SAFETY: Since we don't have a single-byte sequence, a two-byte
        // sequence, or a three-byte sequence (per above) and we know
        // from the safety invariant of this method that the concatenation
        // of `first` and `self.inner` was well-formed UTF-8 upon entry
        // into this method. Therefore, a fourth byte must exist.
        let fourth = *unsafe { self.inner.next().unwrap_unchecked() };
        // INVARIANT UPHELD for `self.inner`:
        // We consumed a complete UTF-8 sequence; see "SAFETY"
        // below.
        self.handler.four_byte(
            // SAFETY: From the safety invariant of this method
            // we know that the concatenation of `first` and
            // `self.inner` was well-formed UTF-8 at the start of
            // this method. The safety invariant of this method
            // rules out a single-byte sequence and we have already
            // checked for and ruled out a two-byte and three-byte
            // sequences. Therefore, we must have a four-byte
            // sequence, which satisfies the safety invariant of
            // `FourByteSequence::new_unchecked()`.
            unsafe { FourByteSequence::new_unchecked(first, second, third, fourth) },
        )
    }
}

impl<'a, H> Clone for CharsWithHandler<'a, H>
where
    H: Utf8Handler + Clone,
{
    fn clone(&self) -> Self {
        Self {
            inner: self.inner.clone(),
            handler: self.handler.clone(),
        }
    }
}

impl<'a, H> Iterator for CharsWithHandler<'a, H>
where
    H: Utf8Handler,
{
    type Item = H::Output;

    #[inline(always)]
    fn next(&mut self) -> Option<H::Output> {
        let first = *self.inner.next()?;
        if let Ok(ascii) = Ascii::try_new(first) {
            return Some(self.handler.single_byte(ascii));
        }
        // SAFETY: Since `self.inner` represented well-formed UTF-8
        // upon entry into this method and `first` is not ASCII,
        // first must be a valid lead byte for a multi-byte UTF-8
        // sequence.
        //
        // INVARIANT UPHELD for `self.inner`:
        // `consume_continuations` consumes the rest of the
        // multi-byte UTF-8 sequence causing a complete sequence
        // to be consumed.
        Some(unsafe { self.consume_continuations(first) })
    }

    #[inline(always)]
    fn count(self) -> usize {
        self.as_str().chars().count()
    }

    #[inline(always)]
    fn size_hint(&self) -> (usize, Option<usize>) {
        let len = self.as_str().len();
        (len.div_ceil(4), Some(len))
    }

    #[inline(always)]
    fn last(mut self) -> Option<Self::Item> {
        // No need to go through the entire string.
        self.next_back()
    }
}

impl<'a, H> DoubleEndedIterator for CharsWithHandler<'a, H>
where
    H: Utf8Handler,
{
    #[inline(always)]
    fn next_back(&mut self) -> Option<H::Output> {
        let last = *self.inner.next_back()?;
        if let Ok(ascii) = Ascii::try_new(last) {
            return Some(self.handler.single_byte(ascii));
        }
        debug_assert!(UnconstrainedContinuation::is(last));
        // SAFETY: Since the last byte was not a single-byte sequence
        // given that `self.inner` represented well-formed UTF-8 upon
        // entry into this method, an earlier byte must exist.
        let second_last = *unsafe { self.inner.next_back().unwrap_unchecked() };
        if !UnconstrainedContinuation::is(second_last) {
            // INVARIANT UPHELD for `self.inner`:
            // We consumed a complete UTF-8 sequence; see "SAFETY" below.
            return Some(self.handler.two_byte(
                // SAFETY: Given that `self.inner` represented well-formed
                // UTF-8 upon entry into this method and that `last` is a continuation
                // but `second_last` is not, they must form a two-byte UTF-8
                // sequence, which satisfies the safety invariant of
                // `TwoByteSequence::new_unchecked()`.
                unsafe { TwoByteSequence::new_unchecked(second_last, last) },
            ));
        }
        // SAFETY: Given that `self.inner` represented well-formed
        // UTF-8 upon entry into this method and we've ruled out a
        // single-byte sequence and a two-byte sequence, an earlier
        // byte must exist.
        let third_last = *unsafe { self.inner.next_back().unwrap_unchecked() };
        if !UnconstrainedContinuation::is(third_last) {
            // INVARIANT UPHELD for `self.inner`:
            // We consumed a complete UTF-8 sequence; see "SAFETY" below.
            return Some(self.handler.three_byte(
                // SAFETY: Given that `self.inner` represented well-formed
                // UTF-8 upon entry into this method and that `last` and `second_last`
                // are continuations but `third_last` is not, they must form a three-byte UTF-8
                // sequence, which satisfies the safety invariant of `ThreeByteSequence::new_unchecked()`.
                unsafe { ThreeByteSequence::new_unchecked(third_last, second_last, last) },
            ));
        }
        // SAFETY: Given that `self.inner` represented well-formed
        // UTF-8 upon entry into this method and we've ruled out a
        // single-byte sequence, a two-byte sequence, and a three-byte
        // sequences, an earlier byte must exist.
        let fourth_last = *unsafe { self.inner.next_back().unwrap_unchecked() };
        // INVARIANT UPHELD for `self.inner`:
        // We consumed a complete UTF-8 sequence; see "SAFETY" below.
        Some(self.handler.four_byte(
            // SAFETY: Given that `self.inner` represented well-formed
            // UTF-8 upon entry into this method and we've ruled out a
            // single-byte sequence, a two-byte sequence, and a three-byte
            // sequences, this must be a four-byte sequence, which satisfies
            // the safety invariant of `FourByteSequence::new_unchecked()`.
            unsafe { FourByteSequence::new_unchecked(fourth_last, third_last, second_last, last) },
        ))
    }
}

impl<H> core::iter::FusedIterator for CharsWithHandler<'_, H> where H: Utf8Handler {}