//! The token iterator [TokenIter] allows you to get the //! general-purpose syntactic [Token]s represented by the source text. //! //! Both iterators are `peek`able: //! //! ``` //! let src = include_str!("../test.edn"); //! let mut view = tengri_dsl::TokenIter::new(src); //! assert_eq!(view.0.0, src); //! assert_eq!(view.peek(), view.0.peek()) //! ``` use crate::*; /// Provides a native [Iterator] API over the [ConstIntoIter] [SourceIter] /// [TokenIter::next] returns just the [Token] and mutates `self`, /// instead of returning an updated version of the struct as [SourceIter::next] does. #[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct TokenIter<'a>(pub SourceIter<'a>); impl<'a> TokenIter<'a> { pub const fn new (source: &'a str) -> Self { Self(SourceIter::new(source)) } pub const fn peek (&self) -> Option> { self.0.peek() } } impl<'a> Iterator for TokenIter<'a> { type Item = Token<'a>; fn next (&mut self) -> Option> { self.0.next().map(|(item, rest)|{self.0 = rest; item}) } } /// Owns a reference to the source text. /// [SourceIter::next] emits subsequent pairs of: /// * a [Token] and /// * the source text remaining /// * [ ] TODO: maybe [SourceIter::next] should wrap the remaining source in `Self` ? #[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct SourceIter<'a>(pub &'a str); const_iter!(<'a>|self: SourceIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result)); impl<'a> From<&'a str> for SourceIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}} impl<'a> SourceIter<'a> { pub const fn new (source: &'a str) -> Self { Self(source) } pub const fn chomp (&self, index: usize) -> Self { Self(split_at(self.0, index).1) } pub const fn next (mut self) -> Option<(Token<'a>, Self)> { Self::next_mut(&mut self) } pub const fn peek (&self) -> Option> { peek_src(self.0) } pub const fn next_mut (&mut self) -> Option<(Token<'a>, Self)> { match self.peek() { Some(token) => Some((token, self.chomp(token.end()))), None => None } } } pub const fn peek_src <'a> (source: &'a str) -> Option> { let mut token: Token<'a> = Token::new(source, 0, 0, Nil); iterate!(char_indices(source) => (start, c) => token = match token.value() { Err(_) => return Some(token), Nil => match c { ' '|'\n'|'\r'|'\t' => token.grow(), '(' => Token::new(source, start, 1, Exp(1, TokenIter::new(str_range(source, start, start + 1)))), ':'|'@' => Token::new(source, start, 1, Sym(str_range(source, start, start + 1))), '/'|'a'..='z' => Token::new(source, start, 1, Key(str_range(source, start, start + 1))), '0'..='9' => Token::new(source, start, 1, match to_digit(c) { Ok(c) => Value::Num(c), Result::Err(e) => Value::Err(e) }), _ => token.error(Unexpected(c)) }, Num(n) => match c { '0'..='9' => token.grow_num(n, c), ' '|'\n'|'\r'|'\t'|')' => return Some(token), _ => token.error(Unexpected(c)) }, Sym(_) => match c { 'a'..='z'|'A'..='Z'|'0'..='9'|'-' => token.grow_sym(), ' '|'\n'|'\r'|'\t'|')' => return Some(token), _ => token.error(Unexpected(c)) }, Key(_) => match c { 'a'..='z'|'0'..='9'|'-'|'/' => token.grow_key(), ' '|'\n'|'\r'|'\t'|')' => return Some(token), _ => token.error(Unexpected(c)) }, Exp(depth, _) => match depth { 0 => return Some(token.grow_exp()), _ => match c { ')' => token.grow_out(), '(' => token.grow_in(), _ => token.grow_exp(), } }, }); match token.value() { Nil => None, _ => Some(token), } } pub const fn to_number (digits: &str) -> Result { let mut value = 0; iterate!(char_indices(digits) => (_, c) => match to_digit(c) { Ok(digit) => value = 10 * value + digit, Result::Err(e) => return Result::Err(e) }); Ok(value) } pub const fn to_digit (c: char) -> Result { Ok(match c { '0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4, '5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9, _ => return Result::Err(Unexpected(c)) }) } #[cfg(test)] mod test_token_iter { use super::*; //use proptest::prelude::*; #[test] fn test_iters () { let mut iter = crate::SourceIter::new(&":foo :bar"); let _ = iter.next(); let mut iter = crate::TokenIter::new(&":foo :bar"); let _ = iter.next(); } #[test] const fn test_const_iters () { let mut iter = crate::SourceIter::new(&":foo :bar"); let _ = iter.next(); } #[test] fn test_num () { let digit = to_digit('0'); let digit = to_digit('x'); let number = to_number(&"123"); let number = to_number(&"12asdf3"); } //proptest! { //#[test] fn proptest_source_iter ( //source in "\\PC*" //) { //let mut iter = crate::SourceIter::new(&source); ////let _ = iter.next(); //} //#[test] fn proptest_token_iter ( //source in "\\PC*" //) { //let mut iter = crate::TokenIter::new(&source); ////let _ = iter.next(); //} //} }