edn -> dsl

dsl/src/token.rs

@@ -0,0 +1,169 @@
+//! [Token]s are parsed substrings with an associated [Value].
+//!
+//! * [ ] FIXME: Value may be [Err] which may shadow [Result::Err]
+//! * [Value::Exp] wraps an expression depth and a [SourceIter]
+//!   with the remaining part of the expression.
+//!   * expression depth other that 0 mean unclosed parenthesis.
+//!   * closing and unopened parenthesis panics during reading.
+//!   * [ ] TODO: signed depth might be interesting
+//! * [Value::Sym] and [Value::Key] are stringish literals
+//!   with slightly different parsing rules.
+//! * [Value::Num] is an unsigned integer literal.
+//!```
+//! use tengri_dsl::{*, Value::*};
+//! let source = include_str!("../test.edn");
+//! let mut view = TokenIter::new(source);
+//! assert_eq!(view.peek(), Some(Token {
+//!     source,
+//!     start: 0,
+//!     length: source.len(),
+//!     value: Exp(0, TokenIter::new(&source[1..]))
+//! }));
+//!```
+use crate::*;
+use self::Value::*;
+#[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> {
+    pub source: &'a str,
+    pub start:  usize, 
+    pub length: usize,
+    pub value:  Value<'a>,
+}
+#[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum Value<'a> {
+    #[default] Nil,
+    Err(ParseError),
+    Num(usize),
+    Sym(&'a str),
+    Key(&'a str),
+    Exp(usize, TokenIter<'a>),
+}
+impl<'a> Token<'a> {
+    pub const fn new (source: &'a str, start: usize, length: usize, value: Value<'a>) -> Self {
+        Self { source, start, length, value }
+    }
+    pub const fn end (&self) -> usize {
+        self.start.saturating_add(self.length)
+    }
+    pub const fn slice (&'a self) -> &'a str {
+        self.slice_source(self.source)
+        //str_range(self.source, self.start, self.end())
+    }
+    pub const fn slice_source <'b> (&'a self, source: &'b str) -> &'b str {
+        str_range(source, self.start, self.end())
+    }
+    pub const fn slice_source_exp <'b> (&'a self, source: &'b str) -> &'b str {
+        str_range(source, self.start.saturating_add(1), self.end())
+    }
+    pub const fn value (&self) -> Value {
+        self.value
+    }
+    pub const fn error (self, error: ParseError) -> Self {
+        Self { value: Value::Err(error), ..self }
+    }
+    pub const fn grow (self) -> Self {
+        Self { length: self.length.saturating_add(1), ..self }
+    }
+    pub const fn grow_num (self, m: usize, c: char) -> Self {
+        match to_digit(c) {
+            Ok(n) => Self { value: Num(10*m+n), ..self.grow() },
+            Result::Err(e) => Self { value: Err(e), ..self.grow() },
+        } 
+    }
+    pub const fn grow_key (self) -> Self {
+        let mut token = self.grow();
+        token.value = Key(token.slice_source(self.source));
+        token
+    }
+    pub const fn grow_sym (self) -> Self {
+        let mut token = self.grow();
+        token.value = Sym(token.slice_source(self.source));
+        token
+    }
+    pub const fn grow_exp (self) -> Self {
+        let mut token = self.grow();
+        if let Exp(depth, _) = token.value {
+            token.value = Exp(depth, TokenIter::new(token.slice_source_exp(self.source)));
+        } else {
+            unreachable!()
+        }
+        token
+    }
+    pub const fn grow_in (self) -> Self {
+        let mut token = self.grow_exp();
+        if let Value::Exp(depth, source) = token.value {
+            token.value = Value::Exp(depth.saturating_add(1), source)
+        } else {
+            unreachable!()
+        }
+        token
+    }
+    pub const fn grow_out (self) -> Self {
+        let mut token = self.grow_exp();
+        if let Value::Exp(depth, source) = token.value {
+            if depth > 0 {
+                token.value = Value::Exp(depth - 1, source)
+            } else {
+                return self.error(Unexpected(')'))
+            }
+        } else {
+            unreachable!()
+        }
+        token
+    }
+}
+#[cfg(test)] mod test_token_prop {
+    use proptest::prelude::*;
+    proptest! {
+        #[test] fn test_token_prop (
+            source in "\\PC*",
+            start  in usize::MIN..usize::MAX,
+            length in usize::MIN..usize::MAX,
+        ) {
+            let token = crate::Token {
+                source: &source,
+                start,
+                length,
+                value: crate::Value::Nil
+            };
+            let _ = token.slice();
+        }
+    }
+}
+#[cfg(test)] #[test] fn test_token () -> Result<(), Box<dyn std::error::Error>> {
+    let source = ":f00";
+    let mut token = Token { source, start: 0, length: 1, value: Sym(":") };
+    token = token.grow_sym();
+    assert_eq!(token, Token { source, start: 0, length: 2, value: Sym(":f") }); 
+    token = token.grow_sym();
+    assert_eq!(token, Token { source, start: 0, length: 3, value: Sym(":f0") }); 
+    token = token.grow_sym();
+    assert_eq!(token, Token { source, start: 0, length: 4, value: Sym(":f00") }); 
+
+    let src = "";
+    assert_eq!(None, SourceIter(src).next());
+
+    let src = "  \n    \r   \t  ";
+    assert_eq!(None, SourceIter(src).next());
+
+    let src = "7";
+    assert_eq!(Num(7), SourceIter(src).next().unwrap().0.value);
+
+    let src = "     100         ";
+    assert_eq!(Num(100), SourceIter(src).next().unwrap().0.value);
+
+    let src = "     9a         ";
+    assert_eq!(Err(Unexpected('a')), SourceIter(src).next().unwrap().0.value);
+
+    let src = "     :123foo      ";
+    assert_eq!(Sym(":123foo"), SourceIter(src).next().unwrap().0.value);
+
+    let src = " \r\r\r\n\n\n@bar456\t\t\t\t\t\t";
+    assert_eq!(Sym("@bar456"), SourceIter(src).next().unwrap().0.value);
+
+    let src = "foo123";
+    assert_eq!(Key("foo123"), SourceIter(src).next().unwrap().0.value);
+
+    let src = "foo/bar";
+    assert_eq!(Key("foo/bar"), SourceIter(src).next().unwrap().0.value);
+
+    Ok(())
+}