dsl: add sexpr syntaces; prefix modules

dsl/src/dsl_token.rs

@@ -0,0 +1,115 @@
+//! [Token]s are parsed substrings with an associated [Value].
+//!
+//! * [ ] FIXME: Value may be [Err] which may shadow [Result::Err]
+//! * [Value::Exp] wraps an expression depth and a [SourceIter]
+//!   with the remaining part of the expression.
+//!   * expression depth other that 0 mean unclosed parenthesis.
+//!   * closing and unopened parenthesis panics during reading.
+//!   * [ ] TODO: signed depth might be interesting
+//! * [Value::Sym] and [Value::Key] are stringish literals
+//!   with slightly different parsing rules.
+//! * [Value::Num] is an unsigned integer literal.
+//!```
+//! use tengri_dsl::{*, Value::*};
+//! let source = include_str!("../test.edn");
+//! let mut view = TokenIter::new(source);
+//! assert_eq!(view.peek(), Some(Token {
+//!     source,
+//!     start: 0,
+//!     length: source.len(),
+//!     value: Exp(0, TokenIter::new(&source[1..]))
+//! }));
+//!```
+use crate::*;
+use self::Value::*;
+
+#[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> {
+    pub source: &'a str,
+    pub start:  usize, 
+    pub length: usize,
+    pub value:  Value<'a>,
+}
+
+#[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum Value<'a> {
+    #[default] Nil,
+    Err(ParseError),
+    Num(usize),
+    Sym(&'a str),
+    Key(&'a str),
+    Exp(usize, TokenIter<'a>),
+}
+
+impl<'a> Token<'a> {
+    pub const fn new (source: &'a str, start: usize, length: usize, value: Value<'a>) -> Self {
+        Self { source, start, length, value }
+    }
+    pub const fn end (&self) -> usize {
+        self.start.saturating_add(self.length)
+    }
+    pub const fn slice (&'a self) -> &'a str {
+        self.slice_source(self.source)
+        //str_range(self.source, self.start, self.end())
+    }
+    pub const fn slice_source <'b> (&'a self, source: &'b str) -> &'b str {
+        str_range(source, self.start, self.end())
+    }
+    pub const fn slice_source_exp <'b> (&'a self, source: &'b str) -> &'b str {
+        str_range(source, self.start.saturating_add(1), self.end())
+    }
+    pub const fn value (&self) -> Value {
+        self.value
+    }
+    pub const fn error (self, error: ParseError) -> Self {
+        Self { value: Value::Err(error), ..self }
+    }
+    pub const fn grow (self) -> Self {
+        Self { length: self.length.saturating_add(1), ..self }
+    }
+    pub const fn grow_num (self, m: usize, c: char) -> Self {
+        match to_digit(c) {
+            Ok(n) => Self { value: Num(10*m+n), ..self.grow() },
+            Result::Err(e) => Self { value: Err(e), ..self.grow() },
+        } 
+    }
+    pub const fn grow_key (self) -> Self {
+        let mut token = self.grow();
+        token.value = Key(token.slice_source(self.source));
+        token
+    }
+    pub const fn grow_sym (self) -> Self {
+        let mut token = self.grow();
+        token.value = Sym(token.slice_source(self.source));
+        token
+    }
+    pub const fn grow_exp (self) -> Self {
+        let mut token = self.grow();
+        if let Exp(depth, _) = token.value {
+            token.value = Exp(depth, TokenIter::new(token.slice_source_exp(self.source)));
+        } else {
+            unreachable!()
+        }
+        token
+    }
+    pub const fn grow_in (self) -> Self {
+        let mut token = self.grow_exp();
+        if let Value::Exp(depth, source) = token.value {
+            token.value = Value::Exp(depth.saturating_add(1), source)
+        } else {
+            unreachable!()
+        }
+        token
+    }
+    pub const fn grow_out (self) -> Self {
+        let mut token = self.grow_exp();
+        if let Value::Exp(depth, source) = token.value {
+            if depth > 0 {
+                token.value = Value::Exp(depth - 1, source)
+            } else {
+                return self.error(Unexpected(')'))
+            }
+        } else {
+            unreachable!()
+        }
+        token
+    }
+}