icu_locale_core/
data.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::ParseError;
6use crate::extensions::unicode as unicode_ext;
7use crate::parser::{
8    ParserMode, SubtagIterator,
9    parse_locale_with_single_variant_single_keyword_unicode_extension_from_iter,
10};
11use crate::preferences::{LocalePreferences, extensions::unicode::keywords::RegionalSubdivision};
12use crate::subtags::{Language, Region, Script, Subtag, Variant};
13use crate::{LanguageIdentifier, Locale};
14use core::cmp::Ordering;
15use core::default::Default;
16use core::fmt;
17use core::hash::Hash;
18use core::str::FromStr;
19
20/// A locale type optimized for use in fallbacking and the ICU4X data pipeline.
21///
22/// [`DataLocale`] contains less functionality than [`Locale`] but more than
23/// [`LanguageIdentifier`] for better size and performance while still meeting
24/// the needs of the ICU4X data pipeline.
25///
26/// In general, you should not need to construct one of these directly. If you do,
27/// even though there is a direct `From<Locale>` conversion, you should
28/// convert through the [`LocalePreferences`] type:
29///
30/// ```
31/// use icu_locale_core::locale;
32/// use icu_locale_core::preferences::LocalePreferences;
33/// use icu_provider::DataLocale;
34/// use writeable::assert_writeable_eq;
35///
36/// // Locale: American English with British user preferences
37/// let locale = locale!("en-US-u-rg-gbzzzz");
38///
39/// // For language-priority fallback, the region override is ignored
40/// let data_locale =
41///     LocalePreferences::from(&locale).to_data_locale_language_priority();
42/// assert_writeable_eq!(data_locale, "en-US");
43///
44/// // The direct conversion implicitly uses language-priority fallback
45/// // (which is incorrect for some use cases).
46/// assert_eq!(data_locale, DataLocale::from(&locale));
47///
48/// // For region-priority fallback, the region override is applied
49/// let data_locale =
50///     LocalePreferences::from(&locale).to_data_locale_region_priority();
51/// assert_writeable_eq!(data_locale, "en-GB");
52/// ```
53///
54/// [`DataLocale`] only supports `-u-sd` keywords, to reflect the current state of CLDR data
55/// lookup and fallback. This may change in the future.
56///
57/// ```
58/// use icu_locale_core::{Locale, locale};
59/// use icu_provider::DataLocale;
60///
61/// let locale = "hi-IN-t-en-h0-hybrid-u-attr-ca-buddhist-sd-inas"
62///     .parse::<Locale>()
63///     .unwrap();
64///
65/// assert_eq!(
66///     DataLocale::from(locale),
67///     DataLocale::from(locale!("hi-IN-u-sd-inas"))
68/// );
69/// ```
70///
71/// [`LocalePreferences`]: crate::preferences::LocalePreferences
72#[derive(#[automatically_derived]
impl ::core::clone::Clone for DataLocale {
    #[inline]
    fn clone(&self) -> DataLocale {
        let _: ::core::clone::AssertParamIsClone<Language>;
        let _: ::core::clone::AssertParamIsClone<Option<Script>>;
        let _: ::core::clone::AssertParamIsClone<Option<Region>>;
        let _: ::core::clone::AssertParamIsClone<Option<Variant>>;
        let _: ::core::clone::AssertParamIsClone<Option<Subtag>>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for DataLocale { }Copy)]
73#[non_exhaustive]
74pub struct DataLocale {
75    /// Language subtag
76    pub language: Language,
77    /// Script subtag
78    pub script: Option<Script>,
79    /// Region subtag
80    pub region: Option<Region>,
81    /// Variant subtag
82    pub variant: Option<Variant>,
83    /// Subivision (-u-sd-) subtag
84    // TODO(3.0): Use `SubdivisionSuffix` type
85    pub subdivision: Option<Subtag>,
86}
87
88impl PartialEq for DataLocale {
89    fn eq(&self, other: &Self) -> bool {
90        self.as_tuple() == other.as_tuple()
91    }
92}
93
94impl Eq for DataLocale {}
95
96impl Hash for DataLocale {
97    fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
98        self.as_tuple().hash(state);
99    }
100}
101
102impl Default for DataLocale {
103    fn default() -> Self {
104        Self {
105            language: Language::UNKNOWN,
106            script: None,
107            region: None,
108            variant: None,
109            subdivision: None,
110        }
111    }
112}
113
114impl DataLocale {
115    /// `const` version of `Default::default`
116    pub const fn default() -> Self {
117        DataLocale {
118            language: Language::UNKNOWN,
119            script: None,
120            region: None,
121            variant: None,
122            subdivision: None,
123        }
124    }
125}
126
127impl Default for &DataLocale {
128    fn default() -> Self {
129        static DEFAULT: DataLocale = DataLocale::default();
130        &DEFAULT
131    }
132}
133
134impl fmt::Debug for DataLocale {
135    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
136        f.write_fmt(format_args!("DataLocale{{{0}}}", self))write!(f, "DataLocale{{{self}}}")
137    }
138}
139
140impl writeable::Writeable for DataLocale {
    fn write_to<W: core::fmt::Write + ?Sized>(&self, sink: &mut W)
        -> core::fmt::Result {
        let mut initial = true;
        self.for_each_subtag_str(&mut |subtag|
                    {
                        if initial {
                            initial = false;
                        } else { sink.write_char('-')?; }
                        sink.write_str(subtag)
                    })
    }
    #[inline]
    fn writeable_length_hint(&self) -> writeable::LengthHint {
        let mut result = writeable::LengthHint::exact(0);
        let mut initial = true;
        self.for_each_subtag_str::<core::convert::Infallible,
                _>(&mut |subtag|
                        {
                            if initial { initial = false; } else { result += 1; }
                            result += subtag.len();
                            Ok(())
                        }).expect("infallible");
        result
    }
    fn writeable_borrow(&self) -> Option<&str> {
        let selff = self;
        if selff.script.is_none() && selff.region.is_none() &&
                    selff.variant.is_none() && selff.subdivision.is_none() {
            Some(selff.language.as_str())
        } else { None }
    }
}
/// This trait is implemented for compatibility with [`fmt!`](core::fmt).
/// To create a string, [`Writeable::write_to_string`] is usually more efficient.
impl core::fmt::Display for DataLocale {
    #[inline]
    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
        ::writeable::Writeable::write_to(&self, f)
    }
}impl_writeable_for_each_subtag_str_no_test!(DataLocale, selff, selff.script.is_none() && selff.region.is_none() && selff.variant.is_none() && selff.subdivision.is_none() => Some(selff.language.as_str()));
141
142impl From<LanguageIdentifier> for DataLocale {
143    fn from(langid: LanguageIdentifier) -> Self {
144        Self::from(&langid)
145    }
146}
147
148impl From<Locale> for DataLocale {
149    fn from(locale: Locale) -> Self {
150        Self::from(&locale)
151    }
152}
153
154impl From<&LanguageIdentifier> for DataLocale {
155    fn from(langid: &LanguageIdentifier) -> Self {
156        Self {
157            language: langid.language,
158            script: langid.script,
159            region: langid.region,
160            variant: langid.variants.iter().copied().next(),
161            subdivision: None,
162        }
163    }
164}
165
166impl From<&Locale> for DataLocale {
167    fn from(locale: &Locale) -> Self {
168        LocalePreferences::from(locale).to_data_locale_language_priority()
169    }
170}
171
172impl From<(Language, Option<Script>, Option<Region>)> for DataLocale {
173    fn from((l, s, r): (Language, Option<Script>, Option<Region>)) -> Self {
174        Self::from_parts(
175            l,
176            s,
177            r.map(|r| unicode_ext::SubdivisionId::new(r, unicode_ext::SubdivisionSuffix::UNKNOWN)),
178            None,
179        )
180    }
181}
182
183impl FromStr for DataLocale {
184    type Err = ParseError;
185    #[inline]
186    fn from_str(s: &str) -> Result<Self, Self::Err> {
187        Self::try_from_str(s)
188    }
189}
190
191impl DataLocale {
192    #[inline]
193    /// Parses a [`DataLocale`].
194    pub const fn try_from_str(s: &str) -> Result<Self, ParseError> {
195        Self::try_from_utf8(s.as_bytes())
196    }
197
198    /// Parses a [`DataLocale`] from a UTF-8 byte slice.
199    pub const fn try_from_utf8(code_units: &[u8]) -> Result<Self, ParseError> {
200        let (language, script, region, variant, keyword) =
201            match parse_locale_with_single_variant_single_keyword_unicode_extension_from_iter(
202                SubtagIterator::new(code_units),
203                ParserMode::Locale,
204            ) {
205                Ok(o) => o,
206                Err(e) => return Err(e),
207            };
208
209        let subdivision = if let Some((key, value)) = keyword {
210            if let Some(value) = value
211                && let RegionalSubdivision::UNICODE_EXTENSION_KEY = key
212            {
213                let Ok(subdivision) = unicode_ext::SubdivisionId::try_from_subtag(value) else {
214                    return Err(ParseError::InvalidExtension);
215                };
216                if let Some(region) = region
217                    && (region.into_raw()[0] != subdivision.region.into_raw()[0]
218                        || region.into_raw()[1] != subdivision.region.into_raw()[1]
219                        || region.into_raw()[2] != subdivision.region.into_raw()[2])
220                {
221                    Some(unicode_ext::SubdivisionId::new(
222                        region,
223                        unicode_ext::SubdivisionSuffix::UNKNOWN,
224                    ))
225                } else {
226                    Some(subdivision)
227                }
228            } else {
229                return Err(ParseError::InvalidExtension);
230            }
231        } else if let Some(region) = region {
232            Some(unicode_ext::SubdivisionId::new(
233                region,
234                unicode_ext::SubdivisionSuffix::UNKNOWN,
235            ))
236        } else {
237            None
238        };
239
240        Ok(Self::from_parts(language, script, subdivision, variant))
241    }
242
243    pub(crate) fn for_each_subtag_str<E, F>(&self, f: &mut F) -> Result<(), E>
244    where
245        F: FnMut(&str) -> Result<(), E>,
246    {
247        f(self.language.as_str())?;
248        if let Some(ref script) = self.script {
249            f(script.as_str())?;
250        }
251        if let Some(ref region) = self.region {
252            f(region.as_str())?;
253        }
254        if let Some(ref single_variant) = self.variant {
255            f(single_variant.as_str())?;
256        }
257        if let Some(extensions) = self.extensions() {
258            extensions.for_each_subtag_str(f)?;
259        }
260        Ok(())
261    }
262
263    fn region_and_subdivision(&self) -> Option<unicode_ext::SubdivisionId> {
264        self.subdivision
265            .and_then(|s| unicode_ext::SubdivisionId::try_from_str(s.as_str()).ok())
266            .or_else(|| {
267                self.region.map(|region| unicode_ext::SubdivisionId {
268                    region,
269                    suffix: unicode_ext::SubdivisionSuffix::UNKNOWN,
270                })
271            })
272    }
273
274    fn as_tuple(
275        &self,
276    ) -> (
277        Language,
278        Option<Script>,
279        Option<unicode_ext::SubdivisionId>,
280        Option<Variant>,
281    ) {
282        (
283            self.language,
284            self.script,
285            self.region_and_subdivision(),
286            self.variant,
287        )
288    }
289
290    pub(crate) const fn from_parts(
291        language: Language,
292        script: Option<Script>,
293        region: Option<unicode_ext::SubdivisionId>,
294        variant: Option<Variant>,
295    ) -> Self {
296        Self {
297            language,
298            script,
299            region: if let Some(r) = region {
300                Some(r.region)
301            } else {
302                None
303            },
304            variant,
305            subdivision: if let Some(r) = region {
306                Some(r.into_subtag())
307            } else {
308                None
309            },
310        }
311    }
312
313    /// Returns an ordering suitable for use in [`BTreeSet`].
314    ///
315    /// [`BTreeSet`]: alloc::collections::BTreeSet
316    pub fn total_cmp(&self, other: &Self) -> Ordering {
317        self.as_tuple().cmp(&other.as_tuple())
318    }
319
320    /// Compare this [`DataLocale`] with BCP-47 bytes.
321    ///
322    /// The return value is equivalent to what would happen if you first converted this
323    /// [`DataLocale`] to a BCP-47 string and then performed a byte comparison.
324    ///
325    /// This function is case-sensitive and results in a *total order*, so it is appropriate for
326    /// binary search. The only argument producing [`Ordering::Equal`] is `self.to_string()`.
327    ///
328    /// # Examples
329    ///
330    /// ```
331    /// use core::cmp::Ordering;
332    /// use icu_provider::DataLocale;
333    ///
334    /// let bcp47_strings: &[&str] = &[
335    ///     "ca",
336    ///     "ca-ES",
337    ///     "ca-ES-u-sd-esct",
338    ///     "ca-ES-valencia",
339    ///     "cat",
340    ///     "pl-Latn-PL",
341    ///     "und",
342    ///     "und-fonipa",
343    ///     "zh",
344    /// ];
345    ///
346    /// for ab in bcp47_strings.windows(2) {
347    ///     let a = ab[0];
348    ///     let b = ab[1];
349    ///     assert_eq!(a.cmp(b), Ordering::Less, "strings: {} < {}", a, b);
350    ///     let a_loc: DataLocale = a.parse().unwrap();
351    ///     assert_eq!(
352    ///         a_loc.strict_cmp(a.as_bytes()),
353    ///         Ordering::Equal,
354    ///         "strict_cmp: {} == {}",
355    ///         a_loc,
356    ///         a
357    ///     );
358    ///     assert_eq!(
359    ///         a_loc.strict_cmp(b.as_bytes()),
360    ///         Ordering::Less,
361    ///         "strict_cmp: {} < {}",
362    ///         a_loc,
363    ///         b
364    ///     );
365    ///     let b_loc: DataLocale = b.parse().unwrap();
366    ///     assert_eq!(
367    ///         b_loc.strict_cmp(b.as_bytes()),
368    ///         Ordering::Equal,
369    ///         "strict_cmp: {} == {}",
370    ///         b_loc,
371    ///         b
372    ///     );
373    ///     assert_eq!(
374    ///         b_loc.strict_cmp(a.as_bytes()),
375    ///         Ordering::Greater,
376    ///         "strict_cmp: {} > {}",
377    ///         b_loc,
378    ///         a
379    ///     );
380    /// }
381    /// ```
382    ///
383    /// Comparison against invalid strings:
384    ///
385    /// ```
386    /// use icu_provider::DataLocale;
387    ///
388    /// let invalid_strings: &[&str] = &[
389    ///     // Less than "ca-ES"
390    ///     "CA",
391    ///     "ar-x-gbp-FOO",
392    ///     // Greater than "ca-AR"
393    ///     "ca_ES",
394    ///     "ca-ES-x-gbp-FOO",
395    /// ];
396    ///
397    /// let data_locale = "ca-ES".parse::<DataLocale>().unwrap();
398    ///
399    /// for s in invalid_strings.iter() {
400    ///     let expected_ordering = "ca-AR".cmp(s);
401    ///     let actual_ordering = data_locale.strict_cmp(s.as_bytes());
402    ///     assert_eq!(expected_ordering, actual_ordering, "{}", s);
403    /// }
404    /// ```
405    pub fn strict_cmp(&self, other: &[u8]) -> Ordering {
406        writeable::cmp_utf8(self, other)
407    }
408
409    /// Returns whether this [`DataLocale`] is `und` in the locale and extensions portion.
410    ///
411    /// # Examples
412    ///
413    /// ```
414    /// use icu_provider::DataLocale;
415    ///
416    /// assert!("und".parse::<DataLocale>().unwrap().is_unknown());
417    /// assert!(!"de-u-sd-denw".parse::<DataLocale>().unwrap().is_unknown());
418    /// assert!(!"und-ES".parse::<DataLocale>().unwrap().is_unknown());
419    /// ```
420    pub fn is_unknown(&self) -> bool {
421        self.language.is_unknown()
422            && self.script.is_none()
423            && self.region.is_none()
424            && self.variant.is_none()
425            && self.subdivision.is_none()
426    }
427
428    /// Converts this `DataLocale` into a [`Locale`].
429    pub fn into_locale(self) -> Locale {
430        Locale {
431            id: LanguageIdentifier {
432                language: self.language,
433                script: self.script,
434                region: self.region,
435                variants: self
436                    .variant
437                    .map(crate::subtags::Variants::from_variant)
438                    .unwrap_or_default(),
439            },
440            extensions: self.extensions().unwrap_or_default(),
441        }
442    }
443
444    fn extensions(&self) -> Option<crate::extensions::Extensions> {
445        Some(crate::extensions::Extensions {
446            unicode: unicode_ext::Unicode {
447                keywords: unicode_ext::Keywords::new_single(
448                    RegionalSubdivision::UNICODE_EXTENSION_KEY,
449                    RegionalSubdivision(
450                        self.region_and_subdivision()
451                            .filter(|sd| !sd.suffix.is_unknown())?,
452                    )
453                    .into(),
454                ),
455                ..Default::default()
456            },
457            ..Default::default()
458        })
459    }
460}
461
462#[test]
463fn test_data_locale_to_string() {
464    struct TestCase {
465        pub locale: &'static str,
466        pub expected: &'static str,
467    }
468
469    for cas in [
470        TestCase {
471            locale: "und",
472            expected: "und",
473        },
474        TestCase {
475            locale: "und-u-sd-sdd",
476            expected: "und-SD-u-sd-sdd",
477        },
478        TestCase {
479            locale: "en-ZA-u-sd-zaa",
480            expected: "en-ZA-u-sd-zaa",
481        },
482        TestCase {
483            locale: "en-ZA-u-sd-sdd",
484            expected: "en-ZA",
485        },
486    ] {
487        let locale = cas.locale.parse::<DataLocale>().unwrap();
488        writeable::assert_writeable_eq!(locale, cas.expected);
489    }
490}
491
492#[test]
493fn test_data_locale_from_string() {
494    #[derive(Debug)]
495    struct TestCase {
496        pub input: &'static str,
497        pub success: bool,
498    }
499
500    for cas in [
501        TestCase {
502            input: "und",
503            success: true,
504        },
505        TestCase {
506            input: "und-u-cu-gbp",
507            success: false,
508        },
509        TestCase {
510            input: "en-ZA-u-sd-zaa",
511            success: true,
512        },
513        TestCase {
514            input: "en...",
515            success: false,
516        },
517    ] {
518        let data_locale = match (DataLocale::from_str(cas.input), cas.success) {
519            (Ok(l), true) => l,
520            (Err(_), false) => {
521                continue;
522            }
523            (Ok(_), false) => {
524                panic!("DataLocale parsed but it was supposed to fail: {cas:?}");
525            }
526            (Err(_), true) => {
527                panic!("DataLocale was supposed to parse but it failed: {cas:?}");
528            }
529        };
530        writeable::assert_writeable_eq!(data_locale, cas.input);
531    }
532}
icu_locale_core/data.rs

icu_locale_core/
data.rs
