Files

aho_corasick

autiter.rsfull.rslib.rs

bigdecimal

lib.rsmacros.rs

bitflags

lib.rs

byteorder

io.rslib.rs

cfg_if

lib.rs

chrono

format

mod.rsparse.rsparsed.rsscan.rsstrftime.rs

naive

date.rsdatetime.rsinternals.rsisoweek.rstime.rs

offset

fixed.rslocal.rsmod.rsutc.rs

sys

unix.rs

date.rsdatetime.rsdiv.rslib.rsround.rssys.rs

diesel

associations

belongs_to.rsmod.rs

connection

mod.rsstatement_cache.rstransaction_manager.rs

expression

functions

aggregate_folding.rsaggregate_ordering.rsdate_and_time.rshelper_types.rsmod.rs

ops

mod.rsnumeric.rs

array_comparison.rsbound.rscoerce.rscount.rsexists.rsgrouped.rshelper_types.rsmod.rsnot.rsnullable.rsoperators.rssql_literal.rssubselect.rs

expression_methods

bool_expression_methods.rseq_all.rsescape_expression_methods.rsglobal_expression_methods.rsmod.rstext_expression_methods.rs

macros

internal.rsmod.rsops.rsquery_id.rsstatic_cond.rstuples.rs

migration

errors.rsmod.rs

mysql

connection

stmt

iterator.rsmetadata.rsmod.rs

bind.rsmod.rsraw.rsurl.rs

query_builder

mod.rsquery_fragment_impls.rs

types

date_and_time.rsmod.rsnumeric.rs

backend.rsbind_collector.rsmod.rs

pg

connection

stmt

mod.rs

cursor.rsmod.rsraw.rsresult.rsrow.rs

expression

extensions

interval_dsl.rsmod.rs

array.rsarray_comparison.rsdate_and_time.rsexpression_methods.rshelper_types.rsmod.rsoperators.rs

query_builder

distinct_on.rsmod.rsquery_fragment_impls.rs

serialize

mod.rswrite_tuple.rs

types

date_and_time

chrono.rsdeprecated_time.rsmod.rsquickcheck_impls.rsstd_time.rs

floats

mod.rsquickcheck_impls.rs

array.rsintegers.rsjson.rsmod.rsmoney.rsnetwork_address.rsnumeric.rsprimitives.rsranges.rsrecord.rsuuid.rsuuid_v0_7.rs

upsert

mod.rson_conflict_actions.rson_conflict_clause.rson_conflict_extension.rson_conflict_target.rs

backend.rsmetadata_lookup.rsmod.rstransaction.rs

query_builder

delete_statement

mod.rs

insert_statement

column_list.rsinsert_from_select.rsmod.rs

nodes

mod.rs

select_statement

boxed.rsdsl_impls.rsmod.rs

update_statement

changeset.rsmod.rstarget.rs

ast_pass.rsbind_collector.rsclause_macro.rsdebug_query.rsdistinct_clause.rsfunctions.rsgroup_by_clause.rslimit_clause.rslocking_clause.rsmod.rsoffset_clause.rsorder_clause.rsquery_id.rsreturning_clause.rsselect_clause.rssql_query.rswhere_clause.rs

query_dsl

belonging_to_dsl.rsboxed_dsl.rsdistinct_dsl.rsfilter_dsl.rsgroup_by_dsl.rsjoin_dsl.rslimit_dsl.rsload_dsl.rslocking_dsl.rsmod.rsnullable_select_dsl.rsoffset_dsl.rsorder_dsl.rssave_changes_dsl.rsselect_dsl.rssingle_value_dsl.rs

query_source

joins.rsmod.rspeano_numbers.rs

sql_types

fold.rsmod.rsops.rsord.rs

sqlite

connection

functions.rsmod.rsraw.rsserialized_value.rssqlite_value.rsstatement_iterator.rsstmt.rs

query_builder

mod.rs

types

date_and_time

chrono.rsmod.rs

mod.rsnumeric.rs

backend.rsmod.rs

type_impls

date_and_time.rsdecimal.rsfloats.rsintegers.rsmod.rsoption.rsprimitives.rstuples.rs

types

mod.rs

backend.rsdata_types.rsdeserialize.rsinsertable.rslib.rsr2d2.rsresult.rsrow.rsserialize.rsutil.rs

diesel_derives

as_changeset.rsas_expression.rsassociations.rsdiagnostic_shim.rsdiesel_numeric_ops.rsfield.rsfrom_sql_row.rsidentifiable.rsinsertable.rslib.rsmeta.rsmodel.rsquery_id.rsqueryable.rsqueryable_by_name.rsresolved_at_shim.rssql_type.rsutil.rs

diesel_migrations

lib.rs

env_logger

lib.rsregex.rs

idna

lib.rspunycode.rsuts46.rs

instant

lib.rsnative.rs

ipnetwork

common.rsipv4.rsipv6.rslib.rs

itoa

lib.rs

kernel32

lib.rs

libc

unix

linux_like

linux

arch

generic

mod.rs

mod.rs

gnu

b64

x86_64

align.rsmod.rsnot_x32.rs

mod.rs

align.rsmod.rs

align.rsmod.rs

mod.rs

align.rsmod.rs

fixed_width_ints.rslib.rsmacros.rs

libsqlite3_sys

error.rslib.rs

lock_api

lib.rsmutex.rsremutex.rsrwlock.rs

log

lib.rsmacros.rs

matches

lib.rs

memchr

lib.rs

migrations_internals

connection.rslib.rsmigration.rsschema.rs

migrations_macros

embed_migrations.rslib.rsmigrations.rsutil.rs

mysqlclient_sys

bindings_macos.rslib.rs

num_bigint

algorithms.rsbigint.rsbiguint.rslib.rsmacros.rsmonty.rs

num_integer

average.rslib.rsroots.rs

num_traits

ops

checked.rsinv.rsmod.rsmul_add.rsoverflowing.rssaturating.rswrapping.rs

bounds.rscast.rsfloat.rsidentities.rsint.rslib.rsmacros.rspow.rsreal.rssign.rs

parking_lot

condvar.rsdeadlock.rselision.rsfair_mutex.rslib.rsmutex.rsonce.rsraw_fair_mutex.rsraw_mutex.rsraw_rwlock.rsremutex.rsrwlock.rsutil.rs

parking_lot_core

thread_parker

linux.rsmod.rs

lib.rsparking_lot.rsspinwait.rsutil.rsword_lock.rs

percent_encoding

lib.rs

pq_sys

lib.rs

proc_macro2

detection.rsfallback.rslib.rsmarker.rsparse.rswrapper.rs

quickcheck

arbitrary.rslib.rstester.rs

quote

ext.rsformat.rsident_fragment.rslib.rsruntime.rsspanned.rsto_tokens.rs

r2d2

config.rsevent.rsextensions.rslib.rs

regex

simd_fallback

mod.rsteddy128.rs

backtrack.rscompile.rsdfa.rserror.rsexec.rsexpand.rsfreqs.rsinput.rslib.rsliterals.rspikevm.rsprog.rsre_builder.rsre_bytes.rsre_plugin.rsre_set.rsre_trait.rsre_unicode.rssparse.rsutf8.rs

regex_syntax

lib.rsliterals.rsparser.rsunicode.rs

ryu

buffer

mod.rs

pretty

exponent.rsmantissa.rsmod.rs

common.rsd2s.rsd2s_full_table.rsd2s_intrinsics.rsdigit_table.rsf2s.rsf2s_intrinsics.rslib.rs

scheduled_thread_pool

lib.rsthunk.rs

scopeguard

lib.rs

serde

de

ignored_any.rsimpls.rsmod.rsseed.rsutf8.rsvalue.rs

private

de.rsdoc.rsmod.rsser.rssize_hint.rs

ser

fmt.rsimpls.rsimpossible.rsmod.rs

integer128.rslib.rsmacros.rs

serde_derive

internals

ast.rsattr.rscase.rscheck.rsctxt.rsmod.rsreceiver.rsrespan.rssymbol.rs

bound.rsde.rsdummy.rsfragment.rslib.rspretend.rsser.rstry.rs

serde_json

features_check

mod.rs

io

mod.rs

value

de.rsfrom.rsindex.rsmod.rspartial_eq.rsser.rs

de.rserror.rsiter.rslib.rsmacros.rsmap.rsnumber.rsread.rsser.rs

smallvec

lib.rs

syn

gen

clone.rsdebug.rseq.rsfold.rsgen_helper.rshash.rs

attr.rsawait.rsbigint.rsbuffer.rscustom_keyword.rscustom_punctuation.rsdata.rsderive.rsdiscouraged.rserror.rsexport.rsexpr.rsext.rsfile.rsgenerics.rsgroup.rsident.rsitem.rslib.rslifetime.rslit.rslookahead.rsmac.rsmacros.rsop.rsparse.rsparse_macro_input.rsparse_quote.rspat.rspath.rsprint.rspunctuated.rsreserved.rssealed.rsspan.rsspanned.rsstmt.rsthread.rstoken.rstt.rsty.rsverbatim.rswhitespace.rs

thread_id

lib.rs

thread_local

lib.rs

time

display.rsduration.rslib.rsparse.rssys.rs

tinyvec

array

generated_impl.rs

array.rsarrayvec.rsarrayvec_drain.rslib.rsslicevec.rstinyvec.rs

tinyvec_macros

lib.rs

unicode_bidi

char_data

mod.rstables.rs

deprecated.rsexplicit.rsformat_chars.rsimplicit.rslevel.rslib.rsprepare.rs

unicode_normalization

__test_api.rsdecompose.rslib.rslookups.rsno_std_prelude.rsnormalize.rsperfect_hash.rsquick_check.rsrecompose.rsreplace.rsstream_safe.rstables.rs

unicode_xid

lib.rstables.rs

url

encoding.rsform_urlencoded.rshost.rslib.rsorigin.rsparser.rspath_segments.rsquirks.rsslicing.rs

utf8_ranges

char_utf8.rslib.rs

uuid

adapter

mod.rs

builder

error.rsmod.rs

parser

error.rsmod.rs

error.rslib.rsprelude.rs

winapi

lib.rs

>

use crate::lookups::{
    canonical_combining_class, canonical_fully_decomposed, compatibility_fully_decomposed,
    stream_safe_trailing_nonstarters,
};
use crate::normalize::{hangul_decomposition_length, is_hangul_syllable};
use crate::tables::stream_safe_leading_nonstarters;

pub(crate) const MAX_NONSTARTERS: usize = 30;
const COMBINING_GRAPHEME_JOINER: char = '\u{034F}';

/// UAX15-D4: This iterator keeps track of how many non-starters there have been
/// since the last starter in *NFKD* and will emit a Combining Grapheme Joiner
/// (U+034F) if the count exceeds 30.
pub struct StreamSafe<I> {
    iter: I,
    nonstarter_count: usize,
    buffer: Option<char>,
}

impl<I> StreamSafe<I> {
    pub(crate) fn new(iter: I) -> Self {
        Self {
            iter,
            nonstarter_count: 0,
            buffer: None,
        }
    }
}

impl<I: Iterator<Item = char>> Iterator for StreamSafe<I> {
    type Item = char;

    #[inline]
    fn next(&mut self) -> Option<char> {
        let next_ch = match self.buffer.take().or_else(|| self.iter.next()) {
            None => return None,
            Some(c) => c,
        };
        let d = classify_nonstarters(next_ch);
        if self.nonstarter_count + d.leading_nonstarters > MAX_NONSTARTERS {
            // Since we're emitting a CGJ, the suffix of the emitted string in NFKD has no trailing
            // nonstarters, so we can reset the counter to zero. Put `next_ch` back into the
            // iterator (via `self.buffer`), and we'll reclassify it next iteration.
            self.nonstarter_count = 0;
            self.buffer = Some(next_ch);
            return Some(COMBINING_GRAPHEME_JOINER);
        }

        // Is the character all nonstarters in NFKD? If so, increment our counter of contiguous
        // nonstarters in NKFD.
        if d.leading_nonstarters == d.decomposition_len {
            self.nonstarter_count += d.decomposition_len;
        }
        // Otherwise, reset the counter to the decomposition's number of trailing nonstarters.
        else {
            self.nonstarter_count = d.trailing_nonstarters;
        }
        Some(next_ch)
    }
}

#[derive(Debug)]
pub(crate) struct Decomposition {
    pub(crate) leading_nonstarters: usize,
    pub(crate) trailing_nonstarters: usize,
    pub(crate) decomposition_len: usize,
}

#[inline]
pub(crate) fn classify_nonstarters(c: char) -> Decomposition {
    // As usual, fast path for ASCII (which is always a starter)
    if c <= '\x7f' {
        return Decomposition {
            leading_nonstarters: 0,
            trailing_nonstarters: 0,
            decomposition_len: 1,
        };
    }
    // Next, special case Hangul, since it's not handled by our tables.
    if is_hangul_syllable(c) {
        return Decomposition {
            leading_nonstarters: 0,
            trailing_nonstarters: 0,
            decomposition_len: hangul_decomposition_length(c),
        };
    }
    let decomp = compatibility_fully_decomposed(c).or_else(|| canonical_fully_decomposed(c));
    match decomp {
        Some(decomp) => Decomposition {
            leading_nonstarters: stream_safe_leading_nonstarters(c),
            trailing_nonstarters: stream_safe_trailing_nonstarters(c),
            decomposition_len: decomp.len(),
        },
        None => {
            let is_nonstarter = canonical_combining_class(c) != 0;
            let nonstarter = if is_nonstarter { 1 } else { 0 };
            Decomposition {
                leading_nonstarters: nonstarter,
                trailing_nonstarters: nonstarter,
                decomposition_len: 1,
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::{classify_nonstarters, StreamSafe};
    use crate::lookups::canonical_combining_class;
    use crate::normalize::decompose_compatible;

    #[cfg(not(feature = "std"))]
    use crate::no_std_prelude::*;

    use core::char;

    fn stream_safe(s: &str) -> String {
        StreamSafe::new(s.chars()).collect()
    }

    #[test]
    fn test_simple() {
        let technically_okay = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}ngerzone";
        assert_eq!(stream_safe(technically_okay), technically_okay);

        let too_much = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone";
        let fixed_it = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{034f}\u{032e}ngerzone";
        assert_eq!(stream_safe(too_much), fixed_it);

        let woah_nelly = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{032e}ngerzone";
        let its_cool = "Da\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{031d}\u{034f}\u{032e}\u{0300}\u{0301}\u{0302}\u{0303}\u{0304}\u{0305}\u{0306}\u{0307}\u{0308}\u{0309}\u{030a}\u{030b}\u{030c}\u{030d}\u{030e}\u{030f}\u{0310}\u{0311}\u{0312}\u{0313}\u{0314}\u{0315}\u{0316}\u{0317}\u{0318}\u{0319}\u{031a}\u{031b}\u{031c}\u{034f}\u{031d}\u{032e}ngerzone";
        assert_eq!(stream_safe(woah_nelly), its_cool);
    }

    #[test]
    fn test_all_nonstarters() {
        let s = "\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}\u{0300}";
        let expected = "\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{034F}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}\u{300}";
        assert_eq!(stream_safe(s), expected);
    }

    #[test]
    fn test_classify_nonstarters() {
        // Highest character in the `compat_fully_decomp` table is 2FA1D
        for ch in 0..0x2FA1E {
            let ch = match char::from_u32(ch) {
                Some(c) => c,
                None => continue,
            };
            let c = classify_nonstarters(ch);
            let mut s = Vec::new();
            decompose_compatible(ch, |c| s.push(c));

            assert_eq!(s.len(), c.decomposition_len);

            let num_leading = s
                .iter()
                .take_while(|&c| canonical_combining_class(*c) != 0)
                .count();
            let num_trailing = s
                .iter()
                .rev()
                .take_while(|&c| canonical_combining_class(*c) != 0)
                .count();

            assert_eq!(num_leading, c.leading_nonstarters);
            assert_eq!(num_trailing, c.trailing_nonstarters);
        }
    }
}