//! [Token]s are parsed substrings with an associated [Value]. //! //! * [ ] FIXME: Value may be [Err] which may shadow [Result::Err] //! * [Value::Exp] wraps an expression depth and a [SourceIter] //! with the remaining part of the expression. //! * expression depth other that 0 mean unclosed parenthesis. //! * closing and unopened parenthesis panics during reading. //! * [ ] TODO: signed depth might be interesting //! * [Value::Sym] and [Value::Key] are stringish literals //! with slightly different parsing rules. //! * [Value::Num] is an unsigned integer literal. //!``` //! use tek_edn::{*, Value::*}; //! let source = include_str!("../../tek/src/view_arranger.edn"); //! let mut view = TokenIter::new(source); //! assert_eq!(view.peek(), Some(Token { //! source, //! start: 0, //! length: source.len(), //! value: Exp(0, TokenIter::new(&source[1..])) //! })); //!``` use crate::*; use self::Value::*; #[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> { pub source: &'a str, pub start: usize, pub length: usize, pub value: Value<'a>, } #[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum Value<'a> { #[default] Nil, Err(ParseError), Num(usize), Sym(&'a str), Key(&'a str), Exp(usize, TokenIter<'a>), } impl<'a> Token<'a> { pub const fn new (source: &'a str, start: usize, length: usize, value: Value<'a>) -> Self { Self { source, start, length, value } } pub const fn end (&self) -> usize { self.start + self.length } pub const fn slice (&'a self) -> &'a str { self.slice_source(self.source) //str_range(self.source, self.start, self.end()) } pub const fn slice_source <'b> (&'a self, source: &'b str) -> &'b str { str_range(source, self.start, self.end()) } pub const fn slice_source_exp <'b> (&'a self, source: &'b str) -> &'b str { str_range(source, self.start + 1, self.end()) } pub const fn value (&self) -> Value { self.value } pub const fn error (self, error: ParseError) -> Self { Self { value: Value::Err(error), ..self } } pub const fn grow (self) -> Self { Self { length: self.length + 1, ..self } } pub const fn grow_num (self, m: usize, c: char) -> Self { match to_digit(c) { Ok(n) => Self { value: Num(10*m+n), ..self.grow() }, Result::Err(e) => Self { value: Err(e), ..self.grow() }, } } pub const fn grow_key (self) -> Self { let mut token = self.grow(); token.value = Key(token.slice_source(self.source)); token } pub const fn grow_sym (self) -> Self { let mut token = self.grow(); token.value = Sym(token.slice_source(self.source)); token } pub const fn grow_exp (self) -> Self { let mut token = self.grow(); if let Exp(depth, _) = token.value { token.value = Exp(depth, TokenIter::new(token.slice_source_exp(self.source))); } else { unreachable!() } token } pub const fn grow_in (self) -> Self { let mut token = self.grow_exp(); if let Value::Exp(depth, source) = token.value { token.value = Value::Exp(depth + 1, source) } else { unreachable!() } token } pub const fn grow_out (self) -> Self { let mut token = self.grow_exp(); if let Value::Exp(depth, source) = token.value { if depth > 0 { token.value = Value::Exp(depth - 1, source) } else { return self.error(Unexpected(')')) } } else { unreachable!() } token } }