icu_properties/
trievalue.rs

1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5use crate::bidi::BidiMirroringGlyph;
6use crate::props::{
7    BidiClass, Block, CanonicalCombiningClass, EastAsianWidth, GeneralCategory,
8    GeneralCategoryGroup, GraphemeClusterBreak, HangulSyllableType, IndicConjunctBreak,
9    IndicSyllabicCategory, JoiningGroup, JoiningType, LineBreak, NumericType, Script,
10    SentenceBreak, VerticalOrientation, WordBreak,
11};
12use crate::script::ScriptWithExt;
13use core::convert::TryInto;
14use core::num::TryFromIntError;
15use zerovec::ule::{AsULE, RawBytesULE};
16
17use icu_collections::codepointtrie::TrieValue;
18
19use core::convert::TryFrom;
20
21impl TrieValue for CanonicalCombiningClass {
22    type TryFromU32Error = TryFromIntError;
23
24    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
25        u8::try_from(i).map(Self)
26    }
27
28    fn to_u32(self) -> u32 {
29        u32::from(self.0)
30    }
31}
32
33impl TrieValue for NumericType {
34    type TryFromU32Error = TryFromIntError;
35
36    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
37        u8::try_from(i).map(Self)
38    }
39
40    fn to_u32(self) -> u32 {
41        u32::from(self.0)
42    }
43}
44
45impl TrieValue for BidiClass {
46    type TryFromU32Error = TryFromIntError;
47
48    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
49        u8::try_from(i).map(Self)
50    }
51
52    fn to_u32(self) -> u32 {
53        u32::from(self.0)
54    }
55}
56
57impl TrieValue for Block {
58    type TryFromU32Error = TryFromIntError;
59
60    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
61        u16::try_from(i).map(Self)
62    }
63
64    fn to_u32(self) -> u32 {
65        u32::from(self.0)
66    }
67}
68
69impl TrieValue for GeneralCategory {
70    type TryFromU32Error = &'static str;
71
72    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
73        // If the u32 is out of range, fall back to u8::MAX, which is out of range of the GeneralCategory enum.
74        GeneralCategory::new_from_u8(i.try_into().unwrap_or(u8::MAX))
75            .ok_or("Cannot parse GeneralCategory from integer")
76    }
77
78    fn to_u32(self) -> u32 {
79        u32::from(self as u8)
80    }
81}
82
83impl TrieValue for Script {
84    type TryFromU32Error = TryFromIntError;
85
86    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
87        u16::try_from(i).map(Script)
88    }
89
90    fn to_u32(self) -> u32 {
91        u32::from(self.0)
92    }
93}
94
95impl TrieValue for HangulSyllableType {
96    type TryFromU32Error = TryFromIntError;
97
98    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
99        u8::try_from(i).map(Self)
100    }
101
102    fn to_u32(self) -> u32 {
103        u32::from(self.0)
104    }
105}
106
107impl TrieValue for ScriptWithExt {
108    type TryFromU32Error = TryFromIntError;
109
110    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
111        u16::try_from(i).map(Self)
112    }
113
114    fn to_u32(self) -> u32 {
115        u32::from(self.0)
116    }
117}
118
119impl TrieValue for EastAsianWidth {
120    type TryFromU32Error = TryFromIntError;
121
122    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
123        u8::try_from(i).map(Self)
124    }
125
126    fn to_u32(self) -> u32 {
127        u32::from(self.0)
128    }
129}
130
131impl TrieValue for LineBreak {
132    type TryFromU32Error = TryFromIntError;
133
134    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
135        u8::try_from(i).map(Self)
136    }
137
138    fn to_u32(self) -> u32 {
139        u32::from(self.0)
140    }
141}
142
143impl TrieValue for GraphemeClusterBreak {
144    type TryFromU32Error = TryFromIntError;
145
146    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
147        u8::try_from(i).map(Self)
148    }
149
150    fn to_u32(self) -> u32 {
151        u32::from(self.0)
152    }
153}
154
155impl TrieValue for WordBreak {
156    type TryFromU32Error = TryFromIntError;
157
158    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
159        u8::try_from(i).map(Self)
160    }
161
162    fn to_u32(self) -> u32 {
163        u32::from(self.0)
164    }
165}
166
167impl TrieValue for SentenceBreak {
168    type TryFromU32Error = TryFromIntError;
169
170    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
171        u8::try_from(i).map(Self)
172    }
173
174    fn to_u32(self) -> u32 {
175        u32::from(self.0)
176    }
177}
178
179impl TrieValue for IndicConjunctBreak {
180    type TryFromU32Error = TryFromIntError;
181
182    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
183        u8::try_from(i).map(Self)
184    }
185
186    fn to_u32(self) -> u32 {
187        u32::from(self.0)
188    }
189}
190
191impl TrieValue for IndicSyllabicCategory {
192    type TryFromU32Error = TryFromIntError;
193
194    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
195        u8::try_from(i).map(Self)
196    }
197
198    fn to_u32(self) -> u32 {
199        u32::from(self.0)
200    }
201}
202
203impl TrieValue for VerticalOrientation {
204    type TryFromU32Error = TryFromIntError;
205
206    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
207        u8::try_from(i).map(Self)
208    }
209
210    fn to_u32(self) -> u32 {
211        u32::from(self.0)
212    }
213}
214
215// GCG is not used inside tries, but it is used in the name lookup type, and we want
216// to squeeze it into a u16 for storage. Its named mask values are specced so we can
217// do this in code.
218//
219// This is done by:
220// - Single-value masks are translated to their corresponding GeneralCategory values
221// - we know all of the multi-value masks and we give them special values
222// - Anything else goes to 0xFF00, though this code path shouldn't be hit unless working with malformed icuexportdata
223//
224// In the reverse direction, unknown values go to the empty mask, but this codepath should not be hit except
225// with malformed ICU4X generated data.
226impl AsULE for GeneralCategoryGroup {
227    type ULE = RawBytesULE<2>;
228    fn to_unaligned(self) -> Self::ULE {
229        let value = gcg_to_packed_u16(self);
230        value.to_unaligned()
231    }
232    fn from_unaligned(ule: Self::ULE) -> Self {
233        let value = ule.as_unsigned_int();
234        packed_u16_to_gcg(value)
235    }
236}
237
238fn packed_u16_to_gcg(value: u16) -> GeneralCategoryGroup {
239    match value {
240        0xFFFF => GeneralCategoryGroup::CasedLetter,
241        0xFFFE => GeneralCategoryGroup::Letter,
242        0xFFFD => GeneralCategoryGroup::Mark,
243        0xFFFC => GeneralCategoryGroup::Number,
244        0xFFFB => GeneralCategoryGroup::Separator,
245        0xFFFA => GeneralCategoryGroup::Other,
246        0xFFF9 => GeneralCategoryGroup::Punctuation,
247        0xFFF8 => GeneralCategoryGroup::Symbol,
248        v if v < 32 => GeneralCategory::new_from_u8(v as u8)
249            .map(|gc| gc.into())
250            .unwrap_or(GeneralCategoryGroup(0)),
251        // unknown values produce an empty mask
252        _ => GeneralCategoryGroup(0),
253    }
254}
255
256fn gcg_to_packed_u16(gcg: GeneralCategoryGroup) -> u16 {
257    // if it's a single property, translate to that property
258    if gcg.0.is_power_of_two() {
259        // inverse operation of a bitshift
260        gcg.0.trailing_zeros() as u16
261    } else {
262        match gcg {
263            GeneralCategoryGroup::CasedLetter => 0xFFFF,
264            GeneralCategoryGroup::Letter => 0xFFFE,
265            GeneralCategoryGroup::Mark => 0xFFFD,
266            GeneralCategoryGroup::Number => 0xFFFC,
267            GeneralCategoryGroup::Separator => 0xFFFB,
268            GeneralCategoryGroup::Other => 0xFFFA,
269            GeneralCategoryGroup::Punctuation => 0xFFF9,
270            GeneralCategoryGroup::Symbol => 0xFFF8,
271            _ => 0xFF00, // random sentinel value
272        }
273    }
274}
275
276impl TrieValue for GeneralCategoryGroup {
277    type TryFromU32Error = TryFromIntError;
278    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
279        // Even though we're dealing with u32s here, TrieValue is about converting
280        // trie storage types to the actual type. This type will always be a packed u16
281        // in our case since the names map upcasts from u16
282        u16::try_from(i).map(packed_u16_to_gcg)
283    }
284
285    fn to_u32(self) -> u32 {
286        u32::from(gcg_to_packed_u16(self))
287    }
288}
289
290impl TrieValue for BidiMirroringGlyph {
291    type TryFromU32Error = u32;
292
293    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
294        let code_point = i & 0x1FFFFF;
295        let mirroring_glyph = if code_point == 0 {
296            None
297        } else {
298            Some(char::try_from_u32(code_point).map_err(|_| i)?)
299        };
300        let mirrored = ((i >> 21) & 0x1) == 1;
301        let paired_bracket_type = {
302            let value = ((i >> 22) & 0x3) as u8;
303            match value {
304                0 => crate::bidi::BidiPairedBracketType::None,
305                1 => crate::bidi::BidiPairedBracketType::Open,
306                2 => crate::bidi::BidiPairedBracketType::Close,
307                _ => return Err(i),
308            }
309        };
310        Ok(Self {
311            mirrored,
312            mirroring_glyph,
313            paired_bracket_type,
314        })
315    }
316
317    fn to_u32(self) -> u32 {
318        self.mirroring_glyph.unwrap_or_default() as u32
319            | ((self.mirrored as u32) << 21)
320            | (match self.paired_bracket_type {
321                crate::bidi::BidiPairedBracketType::None => 0,
322                crate::bidi::BidiPairedBracketType::Open => 1,
323                crate::bidi::BidiPairedBracketType::Close => 2,
324            } << 22)
325    }
326}
327
328impl TrieValue for JoiningType {
329    type TryFromU32Error = TryFromIntError;
330
331    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
332        u8::try_from(i).map(Self)
333    }
334
335    fn to_u32(self) -> u32 {
336        u32::from(self.0)
337    }
338}
339
340impl TrieValue for JoiningGroup {
341    type TryFromU32Error = TryFromIntError;
342
343    fn try_from_u32(i: u32) -> Result<Self, Self::TryFromU32Error> {
344        u8::try_from(i).map(Self)
345    }
346    fn to_u32(self) -> u32 {
347        u32::from(self.0)
348    }
349}
icu_properties/trievalue.rs

icu_properties/
trievalue.rs
