winnow/stream/
token.rs

Help
1use crate::error::Needed;
2use crate::lib::std::iter::Enumerate;
3use crate::lib::std::slice::Iter;
4use crate::stream::Checkpoint;
5use crate::stream::Compare;
6use crate::stream::CompareResult;
7use crate::stream::Location;
8use crate::stream::Offset;
9#[cfg(feature = "unstable-recover")]
10#[cfg(feature = "std")]
11use crate::stream::Recover;
12use crate::stream::SliceLen;
13use crate::stream::Stream;
14use crate::stream::StreamIsPartial;
15use crate::stream::UpdateSlice;
16
17/// Specialized input for parsing lexed tokens
18#[derive(Debug, Copy, Clone, PartialEq, Eq)]
19pub struct TokenSlice<'t, T> {
20    initial: &'t [T],
21    input: &'t [T],
22}
23
24impl<'t, T> TokenSlice<'t, T>
25where
26    T: crate::lib::std::fmt::Debug + Clone,
27{
28    /// Make a stream to parse tokens
29    #[inline]
30    pub fn new(input: &'t [T]) -> Self {
31        Self {
32            initial: input,
33            input,
34        }
35    }
36
37    /// Reset the stream to the start
38    ///
39    /// This is useful for formats that encode a graph with addresses relative to the start of the
40    /// input.
41    #[doc(alias = "fseek")]
42    #[inline]
43    pub fn reset_to_start(&mut self) {
44        let start = self.initial.checkpoint();
45        self.input.reset(&start);
46    }
47}
48
49/// Track locations by implementing [`Location`] on the Token.
50impl<T> TokenSlice<'_, T>
51where
52    T: Location,
53{
54    #[inline(always)]
55    fn previous_token_end(&self) -> Option<usize> {
56        let index = self.input.offset_from(&self.initial);
57        index
58            .checked_sub(1)
59            .map(|i| self.initial[i].previous_token_end())
60    }
61
62    #[inline(always)]
63    fn current_token_start(&self) -> Option<usize> {
64        self.input.first().map(|t| t.current_token_start())
65    }
66}
67
68impl<T> Default for TokenSlice<'_, T>
69where
70    T: crate::lib::std::fmt::Debug + Clone,
71{
72    fn default() -> Self {
73        Self::new(&[])
74    }
75}
76
77impl<T> crate::lib::std::ops::Deref for TokenSlice<'_, T> {
78    type Target = [T];
79
80    fn deref(&self) -> &Self::Target {
81        self.input
82    }
83}
84
85impl<T> SliceLen for TokenSlice<'_, T> {
86    #[inline(always)]
87    fn slice_len(&self) -> usize {
88        self.input.slice_len()
89    }
90}
91
92impl<'t, T> Stream for TokenSlice<'t, T>
93where
94    T: crate::lib::std::fmt::Debug + Clone,
95{
96    type Token = &'t T;
97    type Slice = &'t [T];
98
99    type IterOffsets = Enumerate<Iter<'t, T>>;
100
101    type Checkpoint = Checkpoint<&'t [T], Self>;
102
103    #[inline(always)]
104    fn iter_offsets(&self) -> Self::IterOffsets {
105        self.input.iter().enumerate()
106    }
107    #[inline(always)]
108    fn eof_offset(&self) -> usize {
109        self.input.eof_offset()
110    }
111
112    #[inline(always)]
113    fn next_token(&mut self) -> Option<Self::Token> {
114        let (token, next) = self.input.split_first()?;
115        self.input = next;
116        Some(token)
117    }
118
119    #[inline(always)]
120    fn peek_token(&self) -> Option<Self::Token> {
121        self.input.first()
122    }
123
124    #[inline(always)]
125    fn offset_for<P>(&self, predicate: P) -> Option<usize>
126    where
127        P: Fn(Self::Token) -> bool,
128    {
129        self.input.iter().position(predicate)
130    }
131    #[inline(always)]
132    fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
133        self.input.offset_at(tokens)
134    }
135    #[inline(always)]
136    fn next_slice(&mut self, offset: usize) -> Self::Slice {
137        self.input.next_slice(offset)
138    }
139    #[inline(always)]
140    fn peek_slice(&self, offset: usize) -> Self::Slice {
141        self.input.peek_slice(offset)
142    }
143
144    #[inline(always)]
145    fn checkpoint(&self) -> Self::Checkpoint {
146        Checkpoint::<_, Self>::new(self.input)
147    }
148    #[inline(always)]
149    fn reset(&mut self, checkpoint: &Self::Checkpoint) {
150        self.input = checkpoint.inner;
151    }
152
153    #[inline(always)]
154    fn raw(&self) -> &dyn crate::lib::std::fmt::Debug {
155        &self.input
156    }
157}
158
159impl<T> Location for TokenSlice<'_, T>
160where
161    T: Location,
162{
163    #[inline(always)]
164    fn previous_token_end(&self) -> usize {
165        self.previous_token_end()
166            .or_else(|| self.current_token_start())
167            .unwrap_or(0)
168    }
169    #[inline(always)]
170    fn current_token_start(&self) -> usize {
171        self.current_token_start()
172            .or_else(|| self.previous_token_end())
173            .unwrap_or(0)
174    }
175}
176
177#[cfg(feature = "unstable-recover")]
178#[cfg(feature = "std")]
179impl<T, E> Recover<E> for TokenSlice<'_, T>
180where
181    T: crate::lib::std::fmt::Debug + Clone,
182{
183    #[inline(always)]
184    fn record_err(
185        &mut self,
186        _token_start: &Self::Checkpoint,
187        _err_start: &Self::Checkpoint,
188        err: E,
189    ) -> Result<(), E> {
190        Err(err)
191    }
192
193    /// Report whether the [`Stream`] can save off errors for recovery
194    #[inline(always)]
195    fn is_recovery_supported() -> bool {
196        false
197    }
198}
199
200impl<'t, T> StreamIsPartial for TokenSlice<'t, T>
201where
202    T: crate::lib::std::fmt::Debug + Clone,
203{
204    type PartialState = <&'t [T] as StreamIsPartial>::PartialState;
205
206    #[inline]
207    fn complete(&mut self) -> Self::PartialState {
208        #![allow(clippy::semicolon_if_nothing_returned)]
209        self.input.complete()
210    }
211
212    #[inline]
213    fn restore_partial(&mut self, state: Self::PartialState) {
214        self.input.restore_partial(state);
215    }
216
217    #[inline(always)]
218    fn is_partial_supported() -> bool {
219        <&[T] as StreamIsPartial>::is_partial_supported()
220    }
221
222    #[inline(always)]
223    fn is_partial(&self) -> bool {
224        self.input.is_partial()
225    }
226}
227
228impl<T> Offset for TokenSlice<'_, T>
229where
230    T: crate::lib::std::fmt::Debug + Clone,
231{
232    #[inline(always)]
233    fn offset_from(&self, other: &Self) -> usize {
234        self.offset_from(&other.checkpoint())
235    }
236}
237
238impl<T> Offset<<TokenSlice<'_, T> as Stream>::Checkpoint> for TokenSlice<'_, T>
239where
240    T: crate::lib::std::fmt::Debug + Clone,
241{
242    #[inline(always)]
243    fn offset_from(&self, other: &<TokenSlice<'_, T> as Stream>::Checkpoint) -> usize {
244        self.checkpoint().offset_from(other)
245    }
246}
247
248impl<T, O> Compare<O> for TokenSlice<'_, T>
249where
250    T: PartialEq<O> + Eq,
251{
252    #[inline]
253    fn compare(&self, t: O) -> CompareResult {
254        if let Some(token) = self.first() {
255            if *token == t {
256                CompareResult::Ok(1)
257            } else {
258                CompareResult::Error
259            }
260        } else {
261            CompareResult::Incomplete
262        }
263    }
264}
265
266impl<T> UpdateSlice for TokenSlice<'_, T>
267where
268    T: crate::lib::std::fmt::Debug + Clone,
269{
270    #[inline(always)]
271    fn update_slice(mut self, inner: Self::Slice) -> Self {
272        self.input = <&[T] as UpdateSlice>::update_slice(self.input, inner);
273        self
274    }
275}
winnow/stream/token.rs

winnow/stream/
token.rs
