winnow/token/
mod.rs

Help
1//! Parsers extracting tokens from the stream
2
3#[cfg(test)]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::Needed;
9use crate::error::ParserError;
10use crate::lib::std::result::Result::Ok;
11use crate::stream::Range;
12use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, Stream};
13use crate::stream::{StreamIsPartial, ToUsize};
14use crate::Parser;
15use crate::Result;
16
17/// Matches one token
18///
19/// *Complete version*: Will return an error if there's not enough input data.
20///
21/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
22///
23/// # Effective Signature
24///
25/// Assuming you are parsing a `&str` [Stream]:
26/// ```rust
27/// # use winnow::prelude::*;;
28/// pub fn any(input: &mut &str) -> ModalResult<char>
29/// # {
30/// #     winnow::token::any.parse_next(input)
31/// # }
32/// ```
33///
34/// # Example
35///
36/// ```rust
37/// # use winnow::{token::any, error::ErrMode, error::ContextError};
38/// # use winnow::prelude::*;
39/// fn parser(input: &mut &str) -> ModalResult<char> {
40///     any.parse_next(input)
41/// }
42///
43/// assert_eq!(parser.parse_peek("abc"), Ok(("bc",'a')));
44/// assert!(parser.parse_peek("").is_err());
45/// ```
46///
47/// ```rust
48/// # use winnow::{token::any, error::ErrMode, error::ContextError, error::Needed};
49/// # use winnow::prelude::*;
50/// # use winnow::Partial;
51/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
52/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
53/// ```
54#[inline(always)]
55#[doc(alias = "token")]
56pub fn any<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Token, Error>
57where
58    Input: StreamIsPartial + Stream,
59    Error: ParserError<Input>,
60{
61    trace("any", move |input: &mut Input| {
62        if <Input as StreamIsPartial>::is_partial_supported() {
63            any_::<_, _, true>(input)
64        } else {
65            any_::<_, _, false>(input)
66        }
67    })
68    .parse_next(input)
69}
70
71fn any_<I, E: ParserError<I>, const PARTIAL: bool>(input: &mut I) -> Result<<I as Stream>::Token, E>
72where
73    I: StreamIsPartial,
74    I: Stream,
75{
76    input.next_token().ok_or_else(|| {
77        if PARTIAL && input.is_partial() {
78            ParserError::incomplete(input, Needed::new(1))
79        } else {
80            ParserError::from_input(input)
81        }
82    })
83}
84
85/// Recognizes a literal
86///
87/// The input data will be compared to the literal combinator's argument and will return the part of
88/// the input that matches the argument
89///
90/// It will return `Err(ErrMode::Backtrack(_))` if the input doesn't match the literal
91///
92/// <div class="warning">
93///
94/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
95/// only)
96///
97/// </div>
98///
99/// # Effective Signature
100///
101/// Assuming you are parsing a `&str` [Stream]:
102/// ```rust
103/// # use winnow::prelude::*;;
104/// # use winnow::error::ContextError;
105/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
106/// # {
107/// #     winnow::token::literal(literal)
108/// # }
109/// ```
110///
111/// # Example
112/// ```rust
113/// # use winnow::prelude::*;
114/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
115/// #
116/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
117///   "Hello".parse_next(s)
118/// }
119///
120/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
121/// assert!(parser.parse_peek("Something").is_err());
122/// assert!(parser.parse_peek("").is_err());
123/// ```
124///
125/// ```rust
126/// # use winnow::prelude::*;
127/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
128/// # use winnow::Partial;
129///
130/// fn parser<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
131///   "Hello".parse_next(s)
132/// }
133///
134/// assert_eq!(parser.parse_peek(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
135/// assert!(parser.parse_peek(Partial::new("Something")).is_err());
136/// assert!(parser.parse_peek(Partial::new("S")).is_err());
137/// assert_eq!(parser.parse_peek(Partial::new("H")), Err(ErrMode::Incomplete(Needed::Unknown)));
138/// ```
139///
140/// ```rust
141/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
142/// # use winnow::prelude::*;
143/// use winnow::token::literal;
144/// use winnow::ascii::Caseless;
145///
146/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
147///   literal(Caseless("hello")).parse_next(s)
148/// }
149///
150/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
151/// assert_eq!(parser.parse_peek("hello, World!"), Ok((", World!", "hello")));
152/// assert_eq!(parser.parse_peek("HeLlO, World!"), Ok((", World!", "HeLlO")));
153/// assert!(parser.parse_peek("Something").is_err());
154/// assert!(parser.parse_peek("").is_err());
155/// ```
156#[inline(always)]
157#[doc(alias = "tag")]
158#[doc(alias = "bytes")]
159#[doc(alias = "just")]
160pub fn literal<Literal, Input, Error>(
161    literal: Literal,
162) -> impl Parser<Input, <Input as Stream>::Slice, Error>
163where
164    Input: StreamIsPartial + Stream + Compare<Literal>,
165    Literal: Clone + crate::lib::std::fmt::Debug,
166    Error: ParserError<Input>,
167{
168    trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
169        let t = literal.clone();
170        if <Input as StreamIsPartial>::is_partial_supported() {
171            literal_::<_, _, _, true>(i, t)
172        } else {
173            literal_::<_, _, _, false>(i, t)
174        }
175    })
176}
177
178fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
179    i: &mut I,
180    t: T,
181) -> Result<<I as Stream>::Slice, Error>
182where
183    I: StreamIsPartial,
184    I: Stream + Compare<T>,
185    T: crate::lib::std::fmt::Debug,
186{
187    match i.compare(t) {
188        CompareResult::Ok(len) => Ok(i.next_slice(len)),
189        CompareResult::Incomplete if PARTIAL && i.is_partial() => {
190            Err(ParserError::incomplete(i, Needed::Unknown))
191        }
192        CompareResult::Incomplete | CompareResult::Error => Err(ParserError::from_input(i)),
193    }
194}
195
196/// Recognize a token that matches a [set of tokens][ContainsToken]
197///
198/// <div class="warning">
199///
200/// **Note:** [`Parser`] is implemented as a convenience (complete
201/// only) for
202/// - `u8`
203/// - `char`
204///
205/// </div>
206///
207/// *Complete version*: Will return an error if there's not enough input data.
208///
209/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
210///
211/// # Effective Signature
212///
213/// Assuming you are parsing a `&str` [Stream]:
214/// ```rust
215/// # use winnow::prelude::*;;
216/// # use winnow::stream::ContainsToken;
217/// # use winnow::error::ContextError;
218/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
219/// # {
220/// #     winnow::token::one_of(set)
221/// # }
222/// ```
223///
224/// # Example
225///
226/// ```rust
227/// # use winnow::prelude::*;
228/// # use winnow::{error::ErrMode, error::ContextError};
229/// # use winnow::token::one_of;
230/// assert_eq!(one_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
231/// assert!(one_of::<_, _, ContextError>('a').parse_peek("bc").is_err());
232/// assert!(one_of::<_, _, ContextError>('a').parse_peek("").is_err());
233///
234/// fn parser_fn(i: &mut &str) -> ModalResult<char> {
235///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
236/// }
237/// assert_eq!(parser_fn.parse_peek("abc"), Ok(("bc", 'a')));
238/// assert!(parser_fn.parse_peek("cd").is_err());
239/// assert!(parser_fn.parse_peek("").is_err());
240/// ```
241///
242/// ```rust
243/// # use winnow::prelude::*;
244/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
245/// # use winnow::Partial;
246/// # use winnow::token::one_of;
247/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
248/// assert!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("bc")).is_err());
249/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
250///
251/// fn parser_fn(i: &mut Partial<&str>) -> ModalResult<char> {
252///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
253/// }
254/// assert_eq!(parser_fn.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
255/// assert!(parser_fn.parse_peek(Partial::new("cd")).is_err());
256/// assert_eq!(parser_fn.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
257/// ```
258#[inline(always)]
259#[doc(alias = "char")]
260#[doc(alias = "token")]
261#[doc(alias = "satisfy")]
262pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
263where
264    Input: StreamIsPartial + Stream,
265    <Input as Stream>::Token: Clone,
266    Set: ContainsToken<<Input as Stream>::Token>,
267    Error: ParserError<Input>,
268{
269    trace(
270        "one_of",
271        any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
272    )
273}
274
275/// Recognize a token that does not match a [set of tokens][ContainsToken]
276///
277/// *Complete version*: Will return an error if there's not enough input data.
278///
279/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
280///
281/// # Effective Signature
282///
283/// Assuming you are parsing a `&str` [Stream]:
284/// ```rust
285/// # use winnow::prelude::*;;
286/// # use winnow::stream::ContainsToken;
287/// # use winnow::error::ContextError;
288/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
289/// # {
290/// #     winnow::token::none_of(set)
291/// # }
292/// ```
293///
294/// # Example
295///
296/// ```rust
297/// # use winnow::{error::ErrMode, error::ContextError};
298/// # use winnow::prelude::*;
299/// # use winnow::token::none_of;
300/// assert_eq!(none_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
301/// assert!(none_of::<_, _, ContextError>(['a', 'b']).parse_peek("a").is_err());
302/// assert!(none_of::<_, _, ContextError>('a').parse_peek("").is_err());
303/// ```
304///
305/// ```rust
306/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
307/// # use winnow::prelude::*;
308/// # use winnow::Partial;
309/// # use winnow::token::none_of;
310/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
311/// assert!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b']).parse_peek(Partial::new("a")).is_err());
312/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
313/// ```
314#[inline(always)]
315pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
316where
317    Input: StreamIsPartial + Stream,
318    <Input as Stream>::Token: Clone,
319    Set: ContainsToken<<Input as Stream>::Token>,
320    Error: ParserError<Input>,
321{
322    trace(
323        "none_of",
324        any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
325    )
326}
327
328/// Recognize the longest (m <= len <= n) input slice that matches a [set of tokens][ContainsToken]
329///
330/// It will return an `ErrMode::Backtrack(_)` if the set of tokens wasn't met or is out
331/// of range (m <= len <= n).
332///
333/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
334///
335/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
336///
337/// # Effective Signature
338///
339/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
340/// ```rust
341/// # use std::ops::RangeFrom;
342/// # use winnow::prelude::*;
343/// # use winnow::stream::ContainsToken;
344/// # use winnow::error::ContextError;
345/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
346/// # {
347/// #     winnow::token::take_while(occurrences, set)
348/// # }
349/// ```
350///
351/// # Example
352///
353/// Zero or more tokens:
354/// ```rust
355/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
356/// # use winnow::prelude::*;
357/// use winnow::token::take_while;
358/// use winnow::stream::AsChar;
359///
360/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
361///   take_while(0.., AsChar::is_alpha).parse_next(s)
362/// }
363///
364/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
365/// assert_eq!(alpha.parse_peek(b"12345"), Ok((&b"12345"[..], &b""[..])));
366/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
367/// assert_eq!(alpha.parse_peek(b""), Ok((&b""[..], &b""[..])));
368/// ```
369///
370/// ```rust
371/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
372/// # use winnow::prelude::*;
373/// # use winnow::Partial;
374/// use winnow::token::take_while;
375/// use winnow::stream::AsChar;
376///
377/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
378///   take_while(0.., AsChar::is_alpha).parse_next(s)
379/// }
380///
381/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
382/// assert_eq!(alpha.parse_peek(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
383/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
384/// assert_eq!(alpha.parse_peek(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
385/// ```
386///
387/// One or more tokens:
388/// ```rust
389/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
390/// # use winnow::prelude::*;
391/// use winnow::token::take_while;
392/// use winnow::stream::AsChar;
393///
394/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
395///   take_while(1.., AsChar::is_alpha).parse_next(s)
396/// }
397///
398/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
399/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
400/// assert!(alpha.parse_peek(b"12345").is_err());
401///
402/// fn hex<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
403///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
404/// }
405///
406/// assert_eq!(hex.parse_peek("123 and voila"), Ok((" and voila", "123")));
407/// assert_eq!(hex.parse_peek("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
408/// assert_eq!(hex.parse_peek("BADBABEsomething"), Ok(("something", "BADBABE")));
409/// assert_eq!(hex.parse_peek("D15EA5E"), Ok(("", "D15EA5E")));
410/// assert!(hex.parse_peek("").is_err());
411/// ```
412///
413/// ```rust
414/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
415/// # use winnow::prelude::*;
416/// # use winnow::Partial;
417/// use winnow::token::take_while;
418/// use winnow::stream::AsChar;
419///
420/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
421///   take_while(1.., AsChar::is_alpha).parse_next(s)
422/// }
423///
424/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
425/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
426/// assert!(alpha.parse_peek(Partial::new(b"12345")).is_err());
427///
428/// fn hex<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
429///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
430/// }
431///
432/// assert_eq!(hex.parse_peek(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
433/// assert_eq!(hex.parse_peek(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
434/// assert_eq!(hex.parse_peek(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
435/// assert_eq!(hex.parse_peek(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
436/// assert_eq!(hex.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
437/// ```
438///
439/// Arbitrary amount of tokens:
440/// ```rust
441/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
442/// # use winnow::prelude::*;
443/// use winnow::token::take_while;
444/// use winnow::stream::AsChar;
445///
446/// fn short_alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
447///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
448/// }
449///
450/// assert_eq!(short_alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
451/// assert_eq!(short_alpha.parse_peek(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
452/// assert_eq!(short_alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
453/// assert!(short_alpha.parse_peek(b"ed").is_err());
454/// assert!(short_alpha.parse_peek(b"12345").is_err());
455/// ```
456///
457/// ```rust
458/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
459/// # use winnow::prelude::*;
460/// # use winnow::Partial;
461/// use winnow::token::take_while;
462/// use winnow::stream::AsChar;
463///
464/// fn short_alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
465///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
466/// }
467///
468/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
469/// assert_eq!(short_alpha.parse_peek(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
470/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
471/// assert_eq!(short_alpha.parse_peek(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
472/// assert!(short_alpha.parse_peek(Partial::new(b"12345")).is_err());
473/// ```
474#[inline(always)]
475#[doc(alias = "is_a")]
476#[doc(alias = "take_while0")]
477#[doc(alias = "take_while1")]
478pub fn take_while<Set, Input, Error>(
479    occurrences: impl Into<Range>,
480    set: Set,
481) -> impl Parser<Input, <Input as Stream>::Slice, Error>
482where
483    Input: StreamIsPartial + Stream,
484    Set: ContainsToken<<Input as Stream>::Token>,
485    Error: ParserError<Input>,
486{
487    let Range {
488        start_inclusive,
489        end_inclusive,
490    } = occurrences.into();
491    trace("take_while", move |i: &mut Input| {
492        match (start_inclusive, end_inclusive) {
493            (0, None) => {
494                if <Input as StreamIsPartial>::is_partial_supported() {
495                    take_till0::<_, _, _, true>(i, |c| !set.contains_token(c))
496                } else {
497                    take_till0::<_, _, _, false>(i, |c| !set.contains_token(c))
498                }
499            }
500            (1, None) => {
501                if <Input as StreamIsPartial>::is_partial_supported() {
502                    take_till1::<_, _, _, true>(i, |c| !set.contains_token(c))
503                } else {
504                    take_till1::<_, _, _, false>(i, |c| !set.contains_token(c))
505                }
506            }
507            (start, end) => {
508                let end = end.unwrap_or(usize::MAX);
509                if <Input as StreamIsPartial>::is_partial_supported() {
510                    take_till_m_n::<_, _, _, true>(i, start, end, |c| !set.contains_token(c))
511                } else {
512                    take_till_m_n::<_, _, _, false>(i, start, end, |c| !set.contains_token(c))
513                }
514            }
515        }
516    })
517}
518
519fn take_till0<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
520    input: &mut I,
521    predicate: P,
522) -> Result<<I as Stream>::Slice, E>
523where
524    P: Fn(I::Token) -> bool,
525{
526    let offset = match input.offset_for(predicate) {
527        Some(offset) => offset,
528        None if PARTIAL && input.is_partial() => {
529            return Err(ParserError::incomplete(input, Needed::new(1)));
530        }
531        None => input.eof_offset(),
532    };
533    Ok(input.next_slice(offset))
534}
535
536fn take_till1<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
537    input: &mut I,
538    predicate: P,
539) -> Result<<I as Stream>::Slice, E>
540where
541    P: Fn(I::Token) -> bool,
542{
543    let offset = match input.offset_for(predicate) {
544        Some(offset) => offset,
545        None if PARTIAL && input.is_partial() => {
546            return Err(ParserError::incomplete(input, Needed::new(1)));
547        }
548        None => input.eof_offset(),
549    };
550    if offset == 0 {
551        Err(ParserError::from_input(input))
552    } else {
553        Ok(input.next_slice(offset))
554    }
555}
556
557fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
558    input: &mut I,
559    m: usize,
560    n: usize,
561    predicate: P,
562) -> Result<<I as Stream>::Slice, Error>
563where
564    I: StreamIsPartial,
565    I: Stream,
566    P: Fn(I::Token) -> bool,
567{
568    if n < m {
569        return Err(ParserError::assert(
570            input,
571            "`occurrences` should be ascending, rather than descending",
572        ));
573    }
574
575    let mut final_count = 0;
576    for (processed, (offset, token)) in input.iter_offsets().enumerate() {
577        if predicate(token) {
578            if processed < m {
579                return Err(ParserError::from_input(input));
580            } else {
581                return Ok(input.next_slice(offset));
582            }
583        } else {
584            if processed == n {
585                return Ok(input.next_slice(offset));
586            }
587            final_count = processed + 1;
588        }
589    }
590    if PARTIAL && input.is_partial() {
591        if final_count == n {
592            Ok(input.finish())
593        } else {
594            let needed = if m > input.eof_offset() {
595                m - input.eof_offset()
596            } else {
597                1
598            };
599            Err(ParserError::incomplete(input, Needed::new(needed)))
600        }
601    } else {
602        if m <= final_count {
603            Ok(input.finish())
604        } else {
605            Err(ParserError::from_input(input))
606        }
607    }
608}
609
610/// Recognize the longest input slice (if any) till a member of a [set of tokens][ContainsToken] is found.
611///
612/// It doesn't consume the terminating token from the set.
613///
614/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
615/// end of input or if there was not match.
616///
617/// See also
618/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
619/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
620///
621/// # Effective Signature
622///
623/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
624/// ```rust
625/// # use std::ops::RangeFrom;
626/// # use winnow::prelude::*;
627/// # use winnow::stream::ContainsToken;
628/// # use winnow::error::ContextError;
629/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
630/// # {
631/// #     winnow::token::take_till(occurrences, set)
632/// # }
633/// ```
634///
635/// # Example
636///
637/// ```rust
638/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
639/// # use winnow::prelude::*;
640/// use winnow::token::take_till;
641///
642/// fn till_colon<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
643///   take_till(0.., |c| c == ':').parse_next(s)
644/// }
645///
646/// assert_eq!(till_colon.parse_peek("latin:123"), Ok((":123", "latin")));
647/// assert_eq!(till_colon.parse_peek(":empty matched"), Ok((":empty matched", ""))); //allowed
648/// assert_eq!(till_colon.parse_peek("12345"), Ok(("", "12345")));
649/// assert_eq!(till_colon.parse_peek(""), Ok(("", "")));
650/// ```
651///
652/// ```rust
653/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
654/// # use winnow::prelude::*;
655/// # use winnow::Partial;
656/// use winnow::token::take_till;
657///
658/// fn till_colon<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
659///   take_till(0.., |c| c == ':').parse_next(s)
660/// }
661///
662/// assert_eq!(till_colon.parse_peek(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
663/// assert_eq!(till_colon.parse_peek(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
664/// assert_eq!(till_colon.parse_peek(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
665/// assert_eq!(till_colon.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
666/// ```
667#[inline(always)]
668#[doc(alias = "is_not")]
669pub fn take_till<Set, Input, Error>(
670    occurrences: impl Into<Range>,
671    set: Set,
672) -> impl Parser<Input, <Input as Stream>::Slice, Error>
673where
674    Input: StreamIsPartial + Stream,
675    Set: ContainsToken<<Input as Stream>::Token>,
676    Error: ParserError<Input>,
677{
678    let Range {
679        start_inclusive,
680        end_inclusive,
681    } = occurrences.into();
682    trace("take_till", move |i: &mut Input| {
683        match (start_inclusive, end_inclusive) {
684            (0, None) => {
685                if <Input as StreamIsPartial>::is_partial_supported() {
686                    take_till0::<_, _, _, true>(i, |c| set.contains_token(c))
687                } else {
688                    take_till0::<_, _, _, false>(i, |c| set.contains_token(c))
689                }
690            }
691            (1, None) => {
692                if <Input as StreamIsPartial>::is_partial_supported() {
693                    take_till1::<_, _, _, true>(i, |c| set.contains_token(c))
694                } else {
695                    take_till1::<_, _, _, false>(i, |c| set.contains_token(c))
696                }
697            }
698            (start, end) => {
699                let end = end.unwrap_or(usize::MAX);
700                if <Input as StreamIsPartial>::is_partial_supported() {
701                    take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
702                } else {
703                    take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
704                }
705            }
706        }
707    })
708}
709
710/// Recognize an input slice containing the first N input elements (I[..N]).
711///
712/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))` if the input is shorter than the argument.
713///
714/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
715/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
716/// additional bytes the parser would need to succeed.
717/// It is well defined for `&[u8]` as the number of elements is the byte size,
718/// but for types like `&str`, we cannot know how many bytes correspond for
719/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
720///
721/// # Effective Signature
722///
723/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
724/// ```rust
725/// # use std::ops::RangeFrom;
726/// # use winnow::prelude::*;
727/// # use winnow::stream::ContainsToken;
728/// # use winnow::error::ContextError;
729/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
730/// # {
731/// #     winnow::token::take(token_count)
732/// # }
733/// ```
734///
735/// # Example
736///
737/// ```rust
738/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
739/// # use winnow::prelude::*;
740/// use winnow::token::take;
741///
742/// fn take6<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
743///   take(6usize).parse_next(s)
744/// }
745///
746/// assert_eq!(take6.parse_peek("1234567"), Ok(("7", "123456")));
747/// assert_eq!(take6.parse_peek("things"), Ok(("", "things")));
748/// assert!(take6.parse_peek("short").is_err());
749/// assert!(take6.parse_peek("").is_err());
750/// ```
751///
752/// The units that are taken will depend on the input type. For example, for a
753/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
754/// take that many `u8`'s:
755///
756/// ```rust
757/// # use winnow::prelude::*;
758/// use winnow::error::ContextError;
759/// use winnow::token::take;
760///
761/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙"), Ok(("", "💙")));
762/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
763/// ```
764///
765/// ```rust
766/// # use winnow::prelude::*;
767/// # use winnow::error::{ErrMode, ContextError, Needed};
768/// # use winnow::Partial;
769/// use winnow::token::take;
770///
771/// fn take6<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
772///   take(6usize).parse_next(s)
773/// }
774///
775/// assert_eq!(take6.parse_peek(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
776/// assert_eq!(take6.parse_peek(Partial::new("things")), Ok((Partial::new(""), "things")));
777/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
778/// assert_eq!(take6.parse_peek(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
779/// ```
780#[inline(always)]
781pub fn take<UsizeLike, Input, Error>(
782    token_count: UsizeLike,
783) -> impl Parser<Input, <Input as Stream>::Slice, Error>
784where
785    Input: StreamIsPartial + Stream,
786    UsizeLike: ToUsize,
787    Error: ParserError<Input>,
788{
789    let c = token_count.to_usize();
790    trace("take", move |i: &mut Input| {
791        if <Input as StreamIsPartial>::is_partial_supported() {
792            take_::<_, _, true>(i, c)
793        } else {
794            take_::<_, _, false>(i, c)
795        }
796    })
797}
798
799fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
800    i: &mut I,
801    c: usize,
802) -> Result<<I as Stream>::Slice, Error>
803where
804    I: StreamIsPartial,
805    I: Stream,
806{
807    match i.offset_at(c) {
808        Ok(offset) => Ok(i.next_slice(offset)),
809        Err(e) if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, e)),
810        Err(_needed) => Err(ParserError::from_input(i)),
811    }
812}
813
814/// Recognize the input slice up to the first occurrence of a [literal].
815///
816/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
817///
818/// It doesn't consume the literal.
819///
820/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))`
821/// if the literal wasn't met.
822///
823/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
824/// contain the literal or if the input is smaller than the literal.
825///
826/// See also
827/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
828/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
829///
830/// # Effective Signature
831///
832/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
833/// ```rust
834/// # use std::ops::RangeFrom;
835/// # use winnow::prelude::*;;
836/// # use winnow::error::ContextError;
837/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
838/// # {
839/// #     winnow::token::take_until(occurrences, literal)
840/// # }
841/// ```
842///
843/// # Example
844///
845/// ```rust
846/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
847/// # use winnow::prelude::*;
848/// use winnow::token::take_until;
849///
850/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
851///   take_until(0.., "eof").parse_next(s)
852/// }
853///
854/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
855/// assert!(until_eof.parse_peek("hello, world").is_err());
856/// assert!(until_eof.parse_peek("").is_err());
857/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
858/// ```
859///
860/// ```rust
861/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
862/// # use winnow::prelude::*;
863/// # use winnow::Partial;
864/// use winnow::token::take_until;
865///
866/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
867///   take_until(0.., "eof").parse_next(s)
868/// }
869///
870/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
871/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
872/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
873/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
874/// ```
875///
876/// ```rust
877/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
878/// # use winnow::prelude::*;
879/// use winnow::token::take_until;
880///
881/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
882///   take_until(1.., "eof").parse_next(s)
883/// }
884///
885/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
886/// assert!(until_eof.parse_peek("hello, world").is_err());
887/// assert!(until_eof.parse_peek("").is_err());
888/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
889/// assert!(until_eof.parse_peek("eof").is_err());
890/// ```
891///
892/// ```rust
893/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
894/// # use winnow::prelude::*;
895/// # use winnow::Partial;
896/// use winnow::token::take_until;
897///
898/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
899///   take_until(1.., "eof").parse_next(s)
900/// }
901///
902/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
903/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
904/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
905/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
906/// assert!(until_eof.parse_peek(Partial::new("eof")).is_err());
907/// ```
908#[inline(always)]
909pub fn take_until<Literal, Input, Error>(
910    occurrences: impl Into<Range>,
911    literal: Literal,
912) -> impl Parser<Input, <Input as Stream>::Slice, Error>
913where
914    Input: StreamIsPartial + Stream + FindSlice<Literal>,
915    Literal: Clone,
916    Error: ParserError<Input>,
917{
918    let Range {
919        start_inclusive,
920        end_inclusive,
921    } = occurrences.into();
922    trace("take_until", move |i: &mut Input| {
923        match (start_inclusive, end_inclusive) {
924            (0, None) => {
925                if <Input as StreamIsPartial>::is_partial_supported() {
926                    take_until0_::<_, _, _, true>(i, literal.clone())
927                } else {
928                    take_until0_::<_, _, _, false>(i, literal.clone())
929                }
930            }
931            (1, None) => {
932                if <Input as StreamIsPartial>::is_partial_supported() {
933                    take_until1_::<_, _, _, true>(i, literal.clone())
934                } else {
935                    take_until1_::<_, _, _, false>(i, literal.clone())
936                }
937            }
938            (start, end) => {
939                let end = end.unwrap_or(usize::MAX);
940                if <Input as StreamIsPartial>::is_partial_supported() {
941                    take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
942                } else {
943                    take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
944                }
945            }
946        }
947    })
948}
949
950fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
951    i: &mut I,
952    t: T,
953) -> Result<<I as Stream>::Slice, Error>
954where
955    I: StreamIsPartial,
956    I: Stream + FindSlice<T>,
957{
958    match i.find_slice(t) {
959        Some(range) => Ok(i.next_slice(range.start)),
960        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
961        None => Err(ParserError::from_input(i)),
962    }
963}
964
965fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
966    i: &mut I,
967    t: T,
968) -> Result<<I as Stream>::Slice, Error>
969where
970    I: StreamIsPartial,
971    I: Stream + FindSlice<T>,
972{
973    match i.find_slice(t) {
974        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
975        None => Err(ParserError::from_input(i)),
976        Some(range) => {
977            if range.start == 0 {
978                Err(ParserError::from_input(i))
979            } else {
980                Ok(i.next_slice(range.start))
981            }
982        }
983    }
984}
985
986fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
987    i: &mut I,
988    start: usize,
989    end: usize,
990    t: T,
991) -> Result<<I as Stream>::Slice, Error>
992where
993    I: StreamIsPartial,
994    I: Stream + FindSlice<T>,
995{
996    if end < start {
997        return Err(ParserError::assert(
998            i,
999            "`occurrences` should be ascending, rather than descending",
1000        ));
1001    }
1002
1003    match i.find_slice(t) {
1004        Some(range) => {
1005            let start_offset = i.offset_at(start);
1006            let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1007            if start_offset.map(|s| range.start < s).unwrap_or(true) {
1008                if PARTIAL && i.is_partial() {
1009                    return Err(ParserError::incomplete(i, Needed::Unknown));
1010                } else {
1011                    return Err(ParserError::from_input(i));
1012                }
1013            }
1014            if end_offset < range.start {
1015                return Err(ParserError::from_input(i));
1016            }
1017            Ok(i.next_slice(range.start))
1018        }
1019        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
1020        None => Err(ParserError::from_input(i)),
1021    }
1022}
1023
1024/// Return the remaining input.
1025///
1026/// # Effective Signature
1027///
1028/// Assuming you are parsing a `&str` [Stream]:
1029/// ```rust
1030/// # use winnow::prelude::*;;
1031/// pub fn rest<'i>(input: &mut &'i str) -> ModalResult<&'i str>
1032/// # {
1033/// #     winnow::token::rest.parse_next(input)
1034/// # }
1035/// ```
1036///
1037/// # Example
1038///
1039/// ```rust
1040/// # use winnow::prelude::*;
1041/// # use winnow::error::ContextError;
1042/// use winnow::token::rest;
1043/// assert_eq!(rest::<_,ContextError>.parse_peek("abc"), Ok(("", "abc")));
1044/// assert_eq!(rest::<_,ContextError>.parse_peek(""), Ok(("", "")));
1045/// ```
1046#[inline]
1047pub fn rest<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Slice, Error>
1048where
1049    Input: Stream,
1050    Error: ParserError<Input>,
1051{
1052    trace("rest", move |input: &mut Input| Ok(input.finish())).parse_next(input)
1053}
1054
1055/// Return the length of the remaining input.
1056///
1057/// <div class="warning">
1058///
1059/// Note: this does not advance the [`Stream`]
1060///
1061/// </div>
1062///
1063/// # Effective Signature
1064///
1065/// Assuming you are parsing a `&str` [Stream]:
1066/// ```rust
1067/// # use winnow::prelude::*;;
1068/// pub fn rest_len(input: &mut &str) -> ModalResult<usize>
1069/// # {
1070/// #     winnow::token::rest_len.parse_next(input)
1071/// # }
1072/// ```
1073///
1074/// # Example
1075///
1076/// ```rust
1077/// # use winnow::prelude::*;
1078/// # use winnow::error::ContextError;
1079/// use winnow::token::rest_len;
1080/// assert_eq!(rest_len::<_,ContextError>.parse_peek("abc"), Ok(("abc", 3)));
1081/// assert_eq!(rest_len::<_,ContextError>.parse_peek(""), Ok(("", 0)));
1082/// ```
1083#[inline]
1084pub fn rest_len<Input, Error>(input: &mut Input) -> Result<usize, Error>
1085where
1086    Input: Stream,
1087    Error: ParserError<Input>,
1088{
1089    trace("rest_len", move |input: &mut Input| {
1090        let len = input.eof_offset();
1091        Ok(len)
1092    })
1093    .parse_next(input)
1094}
winnow/token/mod.rs

winnow/token/
mod.rs
