time_macros/format_description/
lexer.rs

Help
1use core::iter;
2
3use super::{Error, Location, Spanned, SpannedValue};
4
5pub(super) struct Lexed<I: Iterator> {
6    iter: iter::Peekable<I>,
7}
8
9impl<I: Iterator> Iterator for Lexed<I> {
10    type Item = I::Item;
11
12    fn next(&mut self) -> Option<Self::Item> {
13        self.iter.next()
14    }
15}
16
17impl<'iter, 'token: 'iter, I: Iterator<Item = Result<Token<'token>, Error>> + 'iter> Lexed<I> {
18    pub(super) fn peek(&mut self) -> Option<&I::Item> {
19        self.iter.peek()
20    }
21
22    pub(super) fn next_if_whitespace(&mut self) -> Option<Spanned<&'token [u8]>> {
23        if let Some(&Ok(Token::ComponentPart {
24            kind: ComponentKind::Whitespace,
25            value,
26        })) = self.peek()
27        {
28            self.next(); // consume
29            Some(value)
30        } else {
31            None
32        }
33    }
34
35    pub(super) fn next_if_not_whitespace(&mut self) -> Option<Spanned<&'token [u8]>> {
36        if let Some(&Ok(Token::ComponentPart {
37            kind: ComponentKind::NotWhitespace,
38            value,
39        })) = self.peek()
40        {
41            self.next();
42            Some(value)
43        } else {
44            None
45        }
46    }
47
48    pub(super) fn next_if_opening_bracket(&mut self) -> Option<Location> {
49        if let Some(&Ok(Token::Bracket {
50            kind: BracketKind::Opening,
51            location,
52        })) = self.peek()
53        {
54            self.next();
55            Some(location)
56        } else {
57            None
58        }
59    }
60
61    pub(super) fn peek_closing_bracket(&'iter mut self) -> Option<&'iter Location> {
62        if let Some(Ok(Token::Bracket {
63            kind: BracketKind::Closing,
64            location,
65        })) = self.peek()
66        {
67            Some(location)
68        } else {
69            None
70        }
71    }
72
73    pub(super) fn next_if_closing_bracket(&mut self) -> Option<Location> {
74        if let Some(&Ok(Token::Bracket {
75            kind: BracketKind::Closing,
76            location,
77        })) = self.peek()
78        {
79            self.next();
80            Some(location)
81        } else {
82            None
83        }
84    }
85}
86
87pub(super) enum Token<'a> {
88    Literal(Spanned<&'a [u8]>),
89    Bracket {
90        kind: BracketKind,
91        location: Location,
92    },
93    ComponentPart {
94        kind: ComponentKind,
95        value: Spanned<&'a [u8]>,
96    },
97}
98
99pub(super) enum BracketKind {
100    Opening,
101    Closing,
102}
103
104pub(super) enum ComponentKind {
105    #[allow(clippy::missing_docs_in_private_items)]
106    Whitespace,
107    #[allow(clippy::missing_docs_in_private_items)]
108    NotWhitespace,
109}
110
111fn attach_location<'item>(
112    iter: impl Iterator<Item = &'item u8>,
113    proc_span: proc_macro::Span,
114) -> impl Iterator<Item = (&'item u8, Location)> {
115    let mut byte_pos = 0;
116
117    iter.map(move |byte| {
118        let location = Location {
119            byte: byte_pos,
120            proc_span,
121        };
122        byte_pos += 1;
123        (byte, location)
124    })
125}
126
127#[allow(clippy::unused_peekable)] // false positive
128pub(super) fn lex<const VERSION: u8>(
129    mut input: &[u8],
130    proc_span: proc_macro::Span,
131) -> Lexed<impl Iterator<Item = Result<Token<'_>, Error>>> {
132    assert!(version!(1..=2));
133
134    let mut depth: u8 = 0;
135    let mut iter = attach_location(input.iter(), proc_span).peekable();
136    let mut second_bracket_location = None;
137
138    let iter = iter::from_fn(move || {
139        if version!(..=1) {
140            if let Some(location) = second_bracket_location.take() {
141                return Some(Ok(Token::Bracket {
142                    kind: BracketKind::Opening,
143                    location,
144                }));
145            }
146        }
147
148        Some(Ok(match iter.next()? {
149            (b'\\', backslash_loc) if version!(2..) => match iter.next() {
150                Some((b'\\' | b'[' | b']', char_loc)) => {
151                    let char = &input[1..2];
152                    input = &input[2..];
153                    if depth == 0 {
154                        Token::Literal(char.spanned(backslash_loc.to(char_loc)))
155                    } else {
156                        Token::ComponentPart {
157                            kind: ComponentKind::NotWhitespace,
158                            value: char.spanned(backslash_loc.to(char_loc)),
159                        }
160                    }
161                }
162                Some((_, loc)) => {
163                    return Some(Err(loc.error("invalid escape sequence")));
164                }
165                None => {
166                    return Some(Err(backslash_loc.error("unexpected end of input")));
167                }
168            },
169            (b'[', location) if version!(..=1) => {
170                if let Some((_, second_location)) = iter.next_if(|&(&byte, _)| byte == b'[') {
171                    second_bracket_location = Some(second_location);
172                    input = &input[2..];
173                } else {
174                    depth += 1;
175                    input = &input[1..];
176                }
177
178                Token::Bracket {
179                    kind: BracketKind::Opening,
180                    location,
181                }
182            }
183            (b'[', location) => {
184                depth += 1;
185                input = &input[1..];
186
187                Token::Bracket {
188                    kind: BracketKind::Opening,
189                    location,
190                }
191            }
192            (b']', location) if depth > 0 => {
193                depth -= 1;
194                input = &input[1..];
195
196                Token::Bracket {
197                    kind: BracketKind::Closing,
198                    location,
199                }
200            }
201            (_, start_location) if depth == 0 => {
202                let mut bytes = 1;
203                let mut end_location = start_location;
204
205                while let Some((_, location)) =
206                    iter.next_if(|&(&byte, _)| !((version!(2..) && byte == b'\\') || byte == b'['))
207                {
208                    end_location = location;
209                    bytes += 1;
210                }
211
212                let value = &input[..bytes];
213                input = &input[bytes..];
214
215                Token::Literal(value.spanned(start_location.to(end_location)))
216            }
217            (byte, start_location) => {
218                let mut bytes = 1;
219                let mut end_location = start_location;
220                let is_whitespace = byte.is_ascii_whitespace();
221
222                while let Some((_, location)) = iter.next_if(|&(byte, _)| {
223                    !matches!(byte, b'\\' | b'[' | b']')
224                        && is_whitespace == byte.is_ascii_whitespace()
225                }) {
226                    end_location = location;
227                    bytes += 1;
228                }
229
230                let value = &input[..bytes];
231                input = &input[bytes..];
232
233                Token::ComponentPart {
234                    kind: if is_whitespace {
235                        ComponentKind::Whitespace
236                    } else {
237                        ComponentKind::NotWhitespace
238                    },
239                    value: value.spanned(start_location.to(end_location)),
240                }
241            }
242        }))
243    });
244
245    Lexed {
246        iter: iter.peekable(),
247    }
248}
time_macros/format_description/lexer.rs

time_macros/format_description/
lexer.rs
