winnow/stream/
token.rs

Help
1use crate::error::Needed;
2use crate::lib::std::iter::Enumerate;
3use crate::lib::std::slice::Iter;
4use crate::stream::Checkpoint;
5use crate::stream::Compare;
6use crate::stream::CompareResult;
7use crate::stream::Location;
8use crate::stream::Offset;
9#[cfg(feature = "unstable-recover")]
10#[cfg(feature = "std")]
11use crate::stream::Recover;
12use crate::stream::SliceLen;
13use crate::stream::Stream;
14use crate::stream::StreamIsPartial;
15use crate::stream::UpdateSlice;
16
17/// Specialized input for parsing lexed tokens
18///
19/// Helpful impls
20/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with
21///   [`literal`][crate::token::literal]
22/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens
23///
24/// See also [Lexing and Parsing][crate::_topic::lexing].
25#[derive(Debug, Copy, Clone, PartialEq, Eq)]
26pub struct TokenSlice<'t, T> {
27    initial: &'t [T],
28    input: &'t [T],
29}
30
31impl<'t, T> TokenSlice<'t, T>
32where
33    T: crate::lib::std::fmt::Debug + Clone,
34{
35    /// Make a stream to parse tokens
36    #[inline]
37    pub fn new(input: &'t [T]) -> Self {
38        Self {
39            initial: input,
40            input,
41        }
42    }
43
44    /// Reset the stream to the start
45    ///
46    /// This is useful for formats that encode a graph with addresses relative to the start of the
47    /// input.
48    #[doc(alias = "fseek")]
49    #[inline]
50    pub fn reset_to_start(&mut self) {
51        let start = self.initial.checkpoint();
52        self.input.reset(&start);
53    }
54}
55
56/// Track locations by implementing [`Location`] on the Token.
57impl<T> TokenSlice<'_, T>
58where
59    T: Location,
60{
61    #[inline(always)]
62    fn previous_token_end(&self) -> Option<usize> {
63        let index = self.input.offset_from(&self.initial);
64        index
65            .checked_sub(1)
66            .map(|i| self.initial[i].previous_token_end())
67    }
68
69    #[inline(always)]
70    fn current_token_start(&self) -> Option<usize> {
71        self.input.first().map(|t| t.current_token_start())
72    }
73}
74
75impl<T> Default for TokenSlice<'_, T>
76where
77    T: crate::lib::std::fmt::Debug + Clone,
78{
79    fn default() -> Self {
80        Self::new(&[])
81    }
82}
83
84impl<T> crate::lib::std::ops::Deref for TokenSlice<'_, T> {
85    type Target = [T];
86
87    fn deref(&self) -> &Self::Target {
88        self.input
89    }
90}
91
92impl<T> SliceLen for TokenSlice<'_, T> {
93    #[inline(always)]
94    fn slice_len(&self) -> usize {
95        self.input.slice_len()
96    }
97}
98
99impl<'t, T> Stream for TokenSlice<'t, T>
100where
101    T: crate::lib::std::fmt::Debug + Clone,
102{
103    type Token = &'t T;
104    type Slice = &'t [T];
105
106    type IterOffsets = Enumerate<Iter<'t, T>>;
107
108    type Checkpoint = Checkpoint<&'t [T], Self>;
109
110    #[inline(always)]
111    fn iter_offsets(&self) -> Self::IterOffsets {
112        self.input.iter().enumerate()
113    }
114    #[inline(always)]
115    fn eof_offset(&self) -> usize {
116        self.input.eof_offset()
117    }
118
119    #[inline(always)]
120    fn next_token(&mut self) -> Option<Self::Token> {
121        let (token, next) = self.input.split_first()?;
122        self.input = next;
123        Some(token)
124    }
125
126    #[inline(always)]
127    fn peek_token(&self) -> Option<Self::Token> {
128        self.input.first()
129    }
130
131    #[inline(always)]
132    fn offset_for<P>(&self, predicate: P) -> Option<usize>
133    where
134        P: Fn(Self::Token) -> bool,
135    {
136        self.input.iter().position(predicate)
137    }
138    #[inline(always)]
139    fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
140        self.input.offset_at(tokens)
141    }
142    #[inline(always)]
143    fn next_slice(&mut self, offset: usize) -> Self::Slice {
144        self.input.next_slice(offset)
145    }
146    #[inline(always)]
147    fn peek_slice(&self, offset: usize) -> Self::Slice {
148        self.input.peek_slice(offset)
149    }
150
151    #[inline(always)]
152    fn checkpoint(&self) -> Self::Checkpoint {
153        Checkpoint::<_, Self>::new(self.input)
154    }
155    #[inline(always)]
156    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
157        self.input = checkpoint.inner;
158    }
159
160    #[inline(always)]
161    fn raw(&self) -> &dyn crate::lib::std::fmt::Debug {
162        &self.input
163    }
164}
165
166impl<T> Location for TokenSlice<'_, T>
167where
168    T: Location,
169{
170    #[inline(always)]
171    fn previous_token_end(&self) -> usize {
172        self.previous_token_end()
173            .or_else(|| self.current_token_start())
174            .unwrap_or(0)
175    }
176    #[inline(always)]
177    fn current_token_start(&self) -> usize {
178        self.current_token_start()
179            .or_else(|| self.previous_token_end())
180            .unwrap_or(0)
181    }
182}
183
184#[cfg(feature = "unstable-recover")]
185#[cfg(feature = "std")]
186impl<T, E> Recover<E> for TokenSlice<'_, T>
187where
188    T: crate::lib::std::fmt::Debug + Clone,
189{
190    #[inline(always)]
191    fn record_err(
192        &mut self,
193        _token_start: &Self::Checkpoint,
194        _err_start: &Self::Checkpoint,
195        err: E,
196    ) -> Result<(), E> {
197        Err(err)
198    }
199
200    /// Report whether the [`Stream`] can save off errors for recovery
201    #[inline(always)]
202    fn is_recovery_supported() -> bool {
203        false
204    }
205}
206
207impl<'t, T> StreamIsPartial for TokenSlice<'t, T>
208where
209    T: crate::lib::std::fmt::Debug + Clone,
210{
211    type PartialState = <&'t [T] as StreamIsPartial>::PartialState;
212
213    #[inline]
214    fn complete(&mut self) -> Self::PartialState {
215        #![allow(clippy::semicolon_if_nothing_returned)]
216        self.input.complete()
217    }
218
219    #[inline]
220    fn restore_partial(&mut self, state: Self::PartialState) {
221        self.input.restore_partial(state);
222    }
223
224    #[inline(always)]
225    fn is_partial_supported() -> bool {
226        <&[T] as StreamIsPartial>::is_partial_supported()
227    }
228
229    #[inline(always)]
230    fn is_partial(&self) -> bool {
231        self.input.is_partial()
232    }
233}
234
235impl<T> Offset for TokenSlice<'_, T>
236where
237    T: crate::lib::std::fmt::Debug + Clone,
238{
239    #[inline(always)]
240    fn offset_from(&self, other: &Self) -> usize {
241        self.offset_from(&other.checkpoint())
242    }
243}
244
245impl<T> Offset<<TokenSlice<'_, T> as Stream>::Checkpoint> for TokenSlice<'_, T>
246where
247    T: crate::lib::std::fmt::Debug + Clone,
248{
249    #[inline(always)]
250    fn offset_from(&self, other: &<TokenSlice<'_, T> as Stream>::Checkpoint) -> usize {
251        self.checkpoint().offset_from(other)
252    }
253}
254
255impl<T, O> Compare<O> for TokenSlice<'_, T>
256where
257    T: PartialEq<O> + Eq,
258{
259    #[inline]
260    fn compare(&self, t: O) -> CompareResult {
261        if let Some(token) = self.first() {
262            if *token == t {
263                CompareResult::Ok(1)
264            } else {
265                CompareResult::Error
266            }
267        } else {
268            CompareResult::Incomplete
269        }
270    }
271}
272
273impl<T> UpdateSlice for TokenSlice<'_, T>
274where
275    T: crate::lib::std::fmt::Debug + Clone,
276{
277    #[inline(always)]
278    fn update_slice(mut self, inner: Self::Slice) -> Self {
279        self.input = <&[T] as UpdateSlice>::update_slice(self.input, inner);
280        self
281    }
282}
winnow/stream/token.rs

winnow/stream/
token.rs
