edn -> dsl

dsl/src/iter.rs

@@ -0,0 +1,144 @@
+//! The token iterator [TokenIter] allows you to get the
+//! general-purpose syntactic [Token]s represented by the source text.
+//!
+//! Both iterators are `peek`able:
+//!
+//! ```
+//! let src = include_str!("../test.edn");
+//! let mut view = tengri_dsl::TokenIter::new(src);
+//! assert_eq!(view.0.0, src);
+//! assert_eq!(view.peek(), view.0.peek())
+//! ```
+use crate::*;
+/// Provides a native [Iterator] API over the [ConstIntoIter] [SourceIter]
+/// [TokenIter::next] returns just the [Token] and mutates `self`,
+/// instead of returning an updated version of the struct as [SourceIter::next] does.
+#[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct TokenIter<'a>(pub SourceIter<'a>);
+impl<'a> TokenIter<'a> {
+    pub const fn new (source: &'a str) -> Self { Self(SourceIter::new(source)) }
+    pub const fn peek (&self) -> Option<Token<'a>> { self.0.peek() }
+}
+impl<'a> Iterator for TokenIter<'a> {
+    type Item = Token<'a>;
+    fn next (&mut self) -> Option<Token<'a>> {
+        self.0.next().map(|(item, rest)|{self.0 = rest; item})
+    }
+}
+/// Owns a reference to the source text.
+/// [SourceIter::next] emits subsequent pairs of:
+///   * a [Token] and
+///   * the source text remaining
+///     * [ ] TODO: maybe [SourceIter::next] should wrap the remaining source in `Self` ?
+#[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct SourceIter<'a>(pub &'a str);
+const_iter!(<'a>|self: SourceIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result));
+impl<'a> From<&'a str> for SourceIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}}
+impl<'a> SourceIter<'a> {
+    pub const fn new (source: &'a str) -> Self { Self(source) }
+    pub const fn chomp (&self, index: usize) -> Self { Self(split_at(self.0, index).1) }
+    pub const fn next (mut self) -> Option<(Token<'a>, Self)> { Self::next_mut(&mut self) }
+    pub const fn peek (&self) -> Option<Token<'a>> { peek_src(self.0) }
+    pub const fn next_mut (&mut self) -> Option<(Token<'a>, Self)> {
+        match self.peek() {
+            Some(token) => Some((token, self.chomp(token.end()))),
+            None => None
+        }
+    }
+}
+pub const fn peek_src <'a> (source: &'a str) -> Option<Token<'a>> {
+    let mut token: Token<'a> = Token::new(source, 0, 0, Nil);
+    iterate!(char_indices(source) => (start, c) => token = match token.value() {
+        Err(_) => return Some(token),
+        Nil => match c {
+            ' '|'\n'|'\r'|'\t' =>
+                token.grow(),
+            '(' =>
+                Token::new(source, start, 1, Exp(1, TokenIter::new(str_range(source, start, start + 1)))),
+            ':'|'@' =>
+                Token::new(source, start, 1, Sym(str_range(source, start, start + 1))),
+            '/'|'a'..='z' =>
+                Token::new(source, start, 1, Key(str_range(source, start, start + 1))),
+            '0'..='9' =>
+                Token::new(source, start, 1, match to_digit(c) {
+                    Ok(c) => Value::Num(c),
+                    Result::Err(e) => Value::Err(e)
+                }),
+            _ => token.error(Unexpected(c))
+        },
+        Num(n) => match c {
+            '0'..='9' => token.grow_num(n, c),
+            ' '|'\n'|'\r'|'\t'|')' => return Some(token),
+            _ => token.error(Unexpected(c))
+        },
+        Sym(_) => match c {
+            'a'..='z'|'A'..='Z'|'0'..='9'|'-' => token.grow_sym(),
+            ' '|'\n'|'\r'|'\t'|')' => return Some(token),
+            _ => token.error(Unexpected(c))
+        },
+        Key(_) => match c {
+            'a'..='z'|'0'..='9'|'-'|'/' => token.grow_key(),
+            ' '|'\n'|'\r'|'\t'|')' => return Some(token),
+            _ => token.error(Unexpected(c))
+        },
+        Exp(depth, _) => match depth {
+            0 => return Some(token.grow_exp()),
+            _ => match c {
+                ')' => token.grow_out(),
+                '(' => token.grow_in(),
+                 _  => token.grow_exp(),
+            }
+        },
+    });
+    match token.value() {
+        Nil => None,
+        _ => Some(token),
+    }
+}
+pub const fn to_number (digits: &str) -> Result<usize, ParseError> {
+    let mut value = 0;
+    iterate!(char_indices(digits) => (_, c) => match to_digit(c) {
+        Ok(digit) => value = 10 * value + digit,
+        Result::Err(e) => return Result::Err(e)
+    });
+    Ok(value)
+}
+pub const fn to_digit (c: char) -> Result<usize, ParseError> {
+    Ok(match c {
+        '0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4,
+        '5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9,
+        _ => return Result::Err(Unexpected(c))
+    })
+}
+#[cfg(test)] mod test_token_iter {
+    use super::*;
+    //use proptest::prelude::*;
+    #[test] fn test_iters () {
+        let mut iter = crate::SourceIter::new(&":foo :bar");
+        let _ = iter.next();
+        let mut iter = crate::TokenIter::new(&":foo :bar");
+        let _ = iter.next();
+    }
+    #[test] const fn test_const_iters () {
+        let mut iter = crate::SourceIter::new(&":foo :bar");
+        let _ = iter.next();
+    }
+    #[test] fn test_num () {
+        let digit  = to_digit('0');
+        let digit  = to_digit('x');
+        let number = to_number(&"123");
+        let number = to_number(&"12asdf3");
+    }
+    //proptest! {
+        //#[test] fn proptest_source_iter (
+            //source in "\\PC*"
+        //) {
+            //let mut iter = crate::SourceIter::new(&source);
+            ////let _ = iter.next();
+        //}
+        //#[test] fn proptest_token_iter (
+            //source in "\\PC*"
+        //) {
+            //let mut iter = crate::TokenIter::new(&source);
+            ////let _ = iter.next();
+        //}
+    //}
+}