icu_locid_transform/fallback/
algorithms.rs

Help
1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use icu_locid::extensions::unicode::{key, Key};
6use icu_locid::subtags::Language;
7use icu_locid::LanguageIdentifier;
8use icu_provider::FallbackPriority;
9
10use super::*;
11
12const SUBDIVISION_KEY: Key = key!("sd");
13
14impl<'a> LocaleFallbackerWithConfig<'a> {
15    pub(crate) fn normalize(&self, locale: &mut DataLocale) {
16        let language = locale.language();
17        // 1. Populate the region (required for region fallback only)
18        if self.config.priority == FallbackPriority::Region && locale.region().is_none() {
19            // 1a. First look for region based on language+script
20            if let Some(script) = locale.script() {
21                locale.set_region(
22                    self.likely_subtags
23                        .ls2r
24                        .get_2d(
25                            &language.into_tinystr().to_unvalidated(),
26                            &script.into_tinystr().to_unvalidated(),
27                        )
28                        .copied(),
29                );
30            }
31            // 1b. If that fails, try language only
32            if locale.region().is_none() {
33                locale.set_region(
34                    self.likely_subtags
35                        .l2r
36                        .get(&language.into_tinystr().to_unvalidated())
37                        .copied(),
38                );
39            }
40        }
41        // 2. Remove the script if it is implied by the other subtags
42        if let Some(script) = locale.script() {
43            let default_script = self
44                .likely_subtags
45                .l2s
46                .get_copied(&language.into_tinystr().to_unvalidated())
47                .unwrap_or(DEFAULT_SCRIPT);
48            if let Some(region) = locale.region() {
49                if script
50                    == self
51                        .likely_subtags
52                        .lr2s
53                        .get_copied_2d(
54                            &language.into_tinystr().to_unvalidated(),
55                            &region.into_tinystr().to_unvalidated(),
56                        )
57                        .unwrap_or(default_script)
58                {
59                    locale.set_script(None);
60                }
61            } else if script == default_script {
62                locale.set_script(None);
63            }
64        }
65        // 3. Remove irrelevant extension subtags
66        locale.retain_unicode_ext(|key| {
67            match *key {
68                // Always retain -u-sd
69                SUBDIVISION_KEY => true,
70                // Retain the query-specific keyword
71                _ if Some(*key) == self.config.extension_key => true,
72                // Drop all others
73                _ => false,
74            }
75        });
76        // 4. If there is an invalid "sd" subtag, drop it
77        // For now, ignore it, and let fallback do it for us
78    }
79}
80
81impl<'a> LocaleFallbackIteratorInner<'a> {
82    pub fn step(&mut self, locale: &mut DataLocale) {
83        match self.config.priority {
84            FallbackPriority::Language => self.step_language(locale),
85            FallbackPriority::Region => self.step_region(locale),
86            // TODO(#1964): Change the collation fallback rules to be different
87            // from the language fallback fules.
88            FallbackPriority::Collation => self.step_language(locale),
89            // This case should not normally happen, but `FallbackPriority` is non_exhaustive.
90            // Make it go directly to `und`.
91            _ => {
92                debug_assert!(
93                    false,
94                    "Unknown FallbackPriority: {:?}",
95                    self.config.priority
96                );
97                *locale = Default::default()
98            }
99        }
100    }
101
102    fn step_language(&mut self, locale: &mut DataLocale) {
103        // 1. Remove the extension fallback keyword
104        if let Some(extension_key) = self.config.extension_key {
105            if let Some(value) = locale.remove_unicode_ext(&extension_key) {
106                self.backup_extension = Some(value);
107                return;
108            }
109        }
110        // 2. Remove the subdivision keyword
111        if let Some(value) = locale.remove_unicode_ext(&SUBDIVISION_KEY) {
112            self.backup_subdivision = Some(value);
113            return;
114        }
115        // 3. Assert that the locale is a language identifier
116        debug_assert!(!locale.has_unicode_ext());
117        // 4. Remove variants
118        if locale.has_variants() {
119            self.backup_variants = Some(locale.clear_variants());
120            return;
121        }
122        // 5. Check for parent override
123        if let Some(parent) = self.get_explicit_parent(locale) {
124            locale.set_langid(parent);
125            self.restore_extensions_variants(locale);
126            return;
127        }
128        // 6. Add the script subtag if necessary
129        if locale.script().is_none() {
130            if let Some(region) = locale.region() {
131                let language = locale.language();
132                if let Some(script) = self.likely_subtags.lr2s.get_copied_2d(
133                    &language.into_tinystr().to_unvalidated(),
134                    &region.into_tinystr().to_unvalidated(),
135                ) {
136                    locale.set_script(Some(script));
137                    self.restore_extensions_variants(locale);
138                    return;
139                }
140            }
141        }
142        // 7. Remove region
143        if locale.region().is_some() {
144            locale.set_region(None);
145            self.restore_extensions_variants(locale);
146            return;
147        }
148        // 8. Remove language+script
149        debug_assert!(!locale.language().is_empty()); // don't call .step() on und
150        locale.set_script(None);
151        locale.set_language(Language::UND);
152    }
153
154    fn step_region(&mut self, locale: &mut DataLocale) {
155        // 1. Remove the extension fallback keyword
156        if let Some(extension_key) = self.config.extension_key {
157            if let Some(value) = locale.remove_unicode_ext(&extension_key) {
158                self.backup_extension = Some(value);
159                return;
160            }
161        }
162        // 2. Remove the subdivision keyword
163        if let Some(value) = locale.remove_unicode_ext(&SUBDIVISION_KEY) {
164            self.backup_subdivision = Some(value);
165            return;
166        }
167        // 3. Assert that the locale is a language identifier
168        debug_assert!(!locale.has_unicode_ext());
169        // 4. Remove variants
170        if locale.has_variants() {
171            self.backup_variants = Some(locale.clear_variants());
172            return;
173        }
174        // 5. Remove language+script
175        if !locale.language().is_empty() || locale.script().is_some() {
176            locale.set_script(None);
177            locale.set_language(Language::UND);
178            self.restore_extensions_variants(locale);
179            return;
180        }
181        // 6. Remove region
182        debug_assert!(locale.region().is_some()); // don't call .step() on und
183        locale.set_region(None);
184    }
185
186    fn restore_extensions_variants(&mut self, locale: &mut DataLocale) {
187        if let Some(value) = self.backup_extension.take() {
188            #[allow(clippy::unwrap_used)] // not reachable unless extension_key is present
189            locale.set_unicode_ext(self.config.extension_key.unwrap(), value);
190        }
191        if let Some(value) = self.backup_subdivision.take() {
192            locale.set_unicode_ext(SUBDIVISION_KEY, value);
193        }
194        if let Some(variants) = self.backup_variants.take() {
195            locale.set_variants(variants);
196        }
197    }
198
199    fn get_explicit_parent(&self, locale: &DataLocale) -> Option<LanguageIdentifier> {
200        self.supplement
201            .and_then(|supplement| {
202                supplement
203                    .parents
204                    .get_copied_by(|uvstr| locale.strict_cmp(uvstr).reverse())
205            })
206            .or_else(|| {
207                self.parents
208                    .parents
209                    .get_copied_by(|uvstr| locale.strict_cmp(uvstr).reverse())
210            })
211            .map(LanguageIdentifier::from)
212    }
213}
214
215#[cfg(test)]
216mod tests {
217    use super::*;
218    use writeable::Writeable;
219
220    /// Unicode extension keywords take part in fallback, but [auxiliary keys] are not modified.
221    ///
222    /// [auxiliary keys]: icu_provider::AuxiliaryKeys
223    #[test]
224    fn test_aux_key_fallback() {
225        use super::LocaleFallbacker;
226
227        let fallbacker = LocaleFallbacker::new();
228        let mut fallback_iterator = fallbacker
229            .for_config(Default::default())
230            .fallback_for("en-US-u-sd-usca-x-aux".parse().unwrap());
231
232        assert_eq!(fallback_iterator.get().to_string(), "en-US-u-sd-usca-x-aux");
233        fallback_iterator.step();
234        assert_eq!(fallback_iterator.get().to_string(), "en-US-x-aux");
235        fallback_iterator.step();
236        assert_eq!(fallback_iterator.get().to_string(), "en-u-sd-usca-x-aux");
237        fallback_iterator.step();
238        assert_eq!(fallback_iterator.get().to_string(), "en-x-aux");
239        fallback_iterator.step();
240        assert_eq!(fallback_iterator.get().to_string(), "und-x-aux");
241        assert!(fallback_iterator.get().is_und());
242    }
243
244    struct TestCase {
245        input: &'static str,
246        requires_data: bool,
247        extension_key: Option<Key>,
248        fallback_supplement: Option<LocaleFallbackSupplement>,
249        // Note: The first entry in the chain is the normalized locale
250        expected_language_chain: &'static [&'static str],
251        expected_region_chain: &'static [&'static str],
252    }
253
254    // TODO: Consider loading these from a JSON file
255    const TEST_CASES: &[TestCase] = &[
256        TestCase {
257            input: "en-u-hc-h12-sd-usca",
258            requires_data: false,
259            extension_key: None,
260            fallback_supplement: None,
261            expected_language_chain: &["en-u-sd-usca", "en"],
262            expected_region_chain: &["en-u-sd-usca", "en", "und-u-sd-usca"],
263        },
264        TestCase {
265            input: "en-US-u-hc-h12-sd-usca",
266            requires_data: false,
267            extension_key: None,
268            fallback_supplement: None,
269            expected_language_chain: &["en-US-u-sd-usca", "en-US", "en-u-sd-usca", "en"],
270            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
271        },
272        TestCase {
273            input: "en-US-fonipa-u-hc-h12-sd-usca",
274            requires_data: false,
275            extension_key: Some(key!("hc")),
276            fallback_supplement: None,
277            expected_language_chain: &[
278                "en-US-fonipa-u-hc-h12-sd-usca",
279                "en-US-fonipa-u-sd-usca",
280                "en-US-fonipa",
281                "en-US",
282                "en-fonipa-u-hc-h12-sd-usca",
283                "en-fonipa-u-sd-usca",
284                "en-fonipa",
285                "en",
286            ],
287            expected_region_chain: &[
288                "en-US-fonipa-u-hc-h12-sd-usca",
289                "en-US-fonipa-u-sd-usca",
290                "en-US-fonipa",
291                "en-US",
292                "und-US-fonipa-u-hc-h12-sd-usca",
293                "und-US-fonipa-u-sd-usca",
294                "und-US-fonipa",
295                "und-US",
296            ],
297        },
298        TestCase {
299            input: "en-u-hc-h12-sd-usca",
300            requires_data: true,
301            extension_key: None,
302            fallback_supplement: None,
303            expected_language_chain: &["en-u-sd-usca", "en"],
304            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
305        },
306        TestCase {
307            input: "en-Latn-u-sd-usca",
308            requires_data: true,
309            extension_key: None,
310            fallback_supplement: None,
311            expected_language_chain: &["en-u-sd-usca", "en"],
312            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
313        },
314        TestCase {
315            input: "en-Latn-US-u-sd-usca",
316            requires_data: true,
317            extension_key: None,
318            fallback_supplement: None,
319            expected_language_chain: &["en-US-u-sd-usca", "en-US", "en-u-sd-usca", "en"],
320            expected_region_chain: &["en-US-u-sd-usca", "en-US", "und-US-u-sd-usca", "und-US"],
321        },
322        TestCase {
323            // TODO(#4413): -u-rg is not yet supported; when it is, this test should be updated
324            input: "en-u-rg-gbxxxx",
325            requires_data: false,
326            extension_key: None,
327            fallback_supplement: None,
328            expected_language_chain: &["en"],
329            expected_region_chain: &["en"],
330        },
331        TestCase {
332            input: "sr-ME",
333            requires_data: true,
334            extension_key: None,
335            fallback_supplement: None,
336            expected_language_chain: &["sr-ME", "sr-Latn-ME", "sr-Latn"],
337            expected_region_chain: &["sr-ME", "und-ME"],
338        },
339        TestCase {
340            input: "sr-Latn-ME",
341            requires_data: true,
342            extension_key: None,
343            fallback_supplement: None,
344            expected_language_chain: &["sr-ME", "sr-Latn-ME", "sr-Latn"],
345            expected_region_chain: &["sr-ME", "und-ME"],
346        },
347        TestCase {
348            input: "sr-ME-fonipa",
349            requires_data: true,
350            extension_key: None,
351            fallback_supplement: None,
352            expected_language_chain: &[
353                "sr-ME-fonipa",
354                "sr-ME",
355                "sr-Latn-ME-fonipa",
356                "sr-Latn-ME",
357                "sr-Latn-fonipa",
358                "sr-Latn",
359            ],
360            expected_region_chain: &["sr-ME-fonipa", "sr-ME", "und-ME-fonipa", "und-ME"],
361        },
362        TestCase {
363            input: "sr-RS",
364            requires_data: true,
365            extension_key: None,
366            fallback_supplement: None,
367            expected_language_chain: &["sr-RS", "sr"],
368            expected_region_chain: &["sr-RS", "und-RS"],
369        },
370        TestCase {
371            input: "sr-Cyrl-RS",
372            requires_data: true,
373            extension_key: None,
374            fallback_supplement: None,
375            expected_language_chain: &["sr-RS", "sr"],
376            expected_region_chain: &["sr-RS", "und-RS"],
377        },
378        TestCase {
379            input: "sr-Latn-RS",
380            requires_data: true,
381            extension_key: None,
382            fallback_supplement: None,
383            expected_language_chain: &["sr-Latn-RS", "sr-Latn"],
384            expected_region_chain: &["sr-Latn-RS", "und-RS"],
385        },
386        TestCase {
387            input: "de-Latn-LI",
388            requires_data: true,
389            extension_key: None,
390            fallback_supplement: None,
391            expected_language_chain: &["de-LI", "de"],
392            expected_region_chain: &["de-LI", "und-LI"],
393        },
394        TestCase {
395            input: "ca-ES-valencia",
396            requires_data: true,
397            extension_key: None,
398            fallback_supplement: None,
399            expected_language_chain: &["ca-ES-valencia", "ca-ES", "ca-valencia", "ca"],
400            expected_region_chain: &["ca-ES-valencia", "ca-ES", "und-ES-valencia", "und-ES"],
401        },
402        TestCase {
403            input: "es-AR",
404            requires_data: true,
405            extension_key: None,
406            fallback_supplement: None,
407            expected_language_chain: &["es-AR", "es-419", "es"],
408            expected_region_chain: &["es-AR", "und-AR"],
409        },
410        TestCase {
411            input: "hi-IN",
412            requires_data: true,
413            extension_key: None,
414            fallback_supplement: None,
415            expected_language_chain: &["hi-IN", "hi"],
416            expected_region_chain: &["hi-IN", "und-IN"],
417        },
418        TestCase {
419            input: "hi-Latn-IN",
420            requires_data: true,
421            extension_key: None,
422            fallback_supplement: None,
423            expected_language_chain: &["hi-Latn-IN", "hi-Latn", "en-IN", "en-001", "en"],
424            expected_region_chain: &["hi-Latn-IN", "und-IN"],
425        },
426        TestCase {
427            input: "zh-CN",
428            requires_data: true,
429            extension_key: None,
430            fallback_supplement: None,
431            // Note: "zh-Hans" is not reachable because it is the default script for "zh".
432            // The fallback algorithm does not visit the language-script bundle when the
433            // script is the default for the language
434            expected_language_chain: &["zh-CN", "zh"],
435            expected_region_chain: &["zh-CN", "und-CN"],
436        },
437        TestCase {
438            input: "zh-TW",
439            requires_data: true,
440            extension_key: None,
441            fallback_supplement: None,
442            expected_language_chain: &["zh-TW", "zh-Hant-TW", "zh-Hant"],
443            expected_region_chain: &["zh-TW", "und-TW"],
444        },
445        TestCase {
446            input: "yue-HK",
447            requires_data: true,
448            extension_key: None,
449            fallback_supplement: None,
450            expected_language_chain: &["yue-HK", "yue"],
451            expected_region_chain: &["yue-HK", "und-HK"],
452        },
453        TestCase {
454            input: "yue-HK",
455            requires_data: true,
456            extension_key: None,
457            fallback_supplement: Some(LocaleFallbackSupplement::Collation),
458            expected_language_chain: &["yue-HK", "yue", "zh-Hant", "zh"],
459            expected_region_chain: &["yue-HK", "und-HK"],
460        },
461    ];
462
463    #[test]
464    fn test_fallback() {
465        let fallbacker_no_data = LocaleFallbacker::new_without_data();
466        let fallbacker_no_data = fallbacker_no_data.as_borrowed();
467        let fallbacker_with_data = LocaleFallbacker::new();
468        for cas in TEST_CASES {
469            for (priority, expected_chain) in [
470                (
471                    LocaleFallbackPriority::Language,
472                    cas.expected_language_chain,
473                ),
474                (LocaleFallbackPriority::Region, cas.expected_region_chain),
475            ] {
476                let mut config = LocaleFallbackConfig::default();
477                config.priority = priority;
478                config.extension_key = cas.extension_key;
479                config.fallback_supplement = cas.fallback_supplement;
480                let fallbacker = if cas.requires_data {
481                    fallbacker_with_data
482                } else {
483                    fallbacker_no_data
484                };
485                let mut it = fallbacker
486                    .for_config(config)
487                    .fallback_for(cas.input.parse().unwrap());
488                for &expected in expected_chain {
489                    assert_eq!(
490                        expected,
491                        &*it.get().write_to_string(),
492                        "{:?} ({:?})",
493                        cas.input,
494                        priority
495                    );
496                    it.step();
497                }
498                assert_eq!(
499                    "und",
500                    &*it.get().write_to_string(),
501                    "{:?} ({:?})",
502                    cas.input,
503                    priority
504                );
505            }
506        }
507    }
508}
icu_locid_transform/fallback/algorithms.rs

icu_locid_transform/fallback/
algorithms.rs
