mirror of
https://codeberg.org/unspeaker/tek.git
synced 2025-12-06 11:46:41 +01:00
remove Atom. almost there
This commit is contained in:
parent
dc7b713108
commit
cf1fd5b45a
20 changed files with 539 additions and 739 deletions
|
|
@ -1,10 +0,0 @@
|
|||
use crate::*;
|
||||
#[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum AtomType {
|
||||
#[default] Nil, Num, Sym, Key, Exp
|
||||
}
|
||||
pub trait Atom: Sized + Send + Sync {
|
||||
fn kind (&self) -> TokenKind;
|
||||
fn text (&self) -> &str;
|
||||
fn num (&self) -> usize;
|
||||
fn exp (&self) -> impl Iterator<Item = Self>;
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
use crate::*;
|
||||
pub type ArcAtomResult = ParseResult<ArcAtom>;
|
||||
//const_iter!(|self: ArcAtomIter| => ArcAtom =>
|
||||
//Some(if let Some(token) = Iterator::next(&mut self.0) {
|
||||
//token.to_arc_atom(self.0.0).unwrap()
|
||||
//} else {
|
||||
//return None
|
||||
//}));
|
||||
#[derive(Clone, PartialEq)] pub enum ArcAtom {
|
||||
Num(usize),
|
||||
Sym(Arc<str>),
|
||||
Key(Arc<str>),
|
||||
Exp(Vec<ArcAtom>),
|
||||
}
|
||||
impl ArcAtom {
|
||||
pub fn read_all <'a> (source: &'a str) -> impl Iterator<Item = Result<ArcAtom, ParseError>> + 'a {
|
||||
TokenResultIterator::new(source).map(move |result: TokenResult<'a>|match result{
|
||||
Err(e) => Err(e),
|
||||
Ok(token) => match token.kind() {
|
||||
Nil => Err(ParseError::Empty),
|
||||
Num => match to_number(token.slice_source(source).into()) {
|
||||
Ok(n) => Ok(ArcAtom::Num(n)),
|
||||
Err(e) => return Err(e)
|
||||
},
|
||||
Sym => Ok(ArcAtom::Sym(token.slice_source(source).into())),
|
||||
Key => Ok(ArcAtom::Key(token.slice_source(source).into())),
|
||||
Exp => Ok(ArcAtom::Exp({
|
||||
let mut iter = Self::read_all(token.slice_source(source));
|
||||
let mut atoms = vec![];
|
||||
while let Some(atom) = iter.next() {
|
||||
atoms.push(atom?);
|
||||
}
|
||||
atoms
|
||||
}))
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Atom for ArcAtom {
|
||||
fn kind (&self) -> TokenKind {
|
||||
match self {
|
||||
Self::Num(_) => TokenKind::Num,
|
||||
Self::Sym(_) => TokenKind::Sym,
|
||||
Self::Key(_) => TokenKind::Key,
|
||||
Self::Exp(_) => TokenKind::Exp,
|
||||
}
|
||||
}
|
||||
fn text (&self) -> &str {
|
||||
match self {
|
||||
Self::Num(_)|Self::Exp(_) => "",
|
||||
Self::Sym(s) => s.as_ref(),
|
||||
Self::Key(k) => k.as_ref(),
|
||||
}
|
||||
}
|
||||
fn num (&self) -> usize {
|
||||
match self {
|
||||
Self::Num(n) => *n,
|
||||
_ => 0
|
||||
}
|
||||
}
|
||||
fn exp (&self) -> impl Iterator<Item = ArcAtom> {
|
||||
let cheap_clone = |t: &ArcAtom|t.clone();
|
||||
if let Self::Exp(e) = self {
|
||||
return e.iter().map(cheap_clone)
|
||||
} else {
|
||||
let empty: &[ArcAtom] = &[];
|
||||
empty.iter().map(cheap_clone)
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<'a> Debug for ArcAtom {
|
||||
fn fmt (&self, f: &mut Formatter<'_>) -> Result<(), FormatError> {
|
||||
match self {
|
||||
Self::Num(n) => write!(f, "(num {n})"),
|
||||
Self::Sym(t) => write!(f, "(sym {t:?})"),
|
||||
Self::Key(t) => write!(f, "(key {t:?})"),
|
||||
Self::Exp(u) => write!(f, "(exp {u:?})"),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<'a> Display for ArcAtom {
|
||||
fn fmt (&self, f: &mut Formatter<'_>) -> Result<(), FormatError> {
|
||||
match self {
|
||||
Self::Num(n) => write!(f, "{n}"),
|
||||
Self::Sym(t) => write!(f, "{t}"),
|
||||
Self::Key(t) => write!(f, "{t}"),
|
||||
Self::Exp(u) => write!(f, "({})", join(u.iter().map(|i|format!("{}", &i)), " "))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
use crate::*;
|
||||
pub type RefAtomResult<'a> = ParseResult<RefAtom<'a>>;
|
||||
#[derive(Copy, Clone, PartialEq)] pub struct RefAtomIter<'a>(pub TokenIter<'a>);
|
||||
const_iter!(<'a>|self: RefAtomIter<'a>| => RefAtom<'a> =>
|
||||
Some(if let Some(token) = Iterator::next(&mut self.0) {
|
||||
token.to_ref_atom(self.0.0).unwrap()
|
||||
} else {
|
||||
return None
|
||||
}));
|
||||
impl<'a> From<&'a str> for RefAtomIter<'a> {
|
||||
fn from (source: &'a str) -> Self { Self::from_source(source) }
|
||||
}
|
||||
impl<'a> From<TokenIter<'a>> for RefAtomIter<'a> {
|
||||
fn from (iter: TokenIter<'a>) -> Self { Self::from_tokens_iter(iter) }
|
||||
}
|
||||
impl<'a> RefAtomIter<'a> {
|
||||
pub const fn from_source (source: &'a str) -> Self { Self(TokenIter(source)) }
|
||||
pub const fn from_tokens_iter (iter: TokenIter<'a>) -> Self { Self(iter) }
|
||||
}
|
||||
#[derive(Copy, Clone, PartialEq)] pub struct RefAtomResultIterator<'a>(pub TokenResultIterator<'a>);
|
||||
const_iter!(<'a>|self: RefAtomResultIterator<'a>| => RefAtomResult<'a> =>
|
||||
Some(if let Some(result) = Iterator::next(&mut self.0) {
|
||||
match result { Ok(token) => token.to_ref_atom(self.0.0), Err(e) => Err(e), }
|
||||
} else {
|
||||
return None
|
||||
}));
|
||||
impl<'a> From<&'a str> for RefAtomResultIterator<'a> {
|
||||
fn from (source: &'a str) -> Self { Self::from_source(source) }
|
||||
}
|
||||
impl<'a> From<TokenResultIterator<'a>> for RefAtomResultIterator<'a> {
|
||||
fn from (iter: TokenResultIterator<'a>) -> Self { Self::from_tokens_iter(iter) }
|
||||
}
|
||||
impl<'a> RefAtomResultIterator<'a> {
|
||||
pub const fn from_source (source: &'a str) -> Self { Self(TokenResultIterator(source)) }
|
||||
pub const fn from_tokens_iter (iter: TokenResultIterator<'a>) -> Self { Self(iter) }
|
||||
}
|
||||
#[derive(Copy, Clone, PartialEq)] pub enum RefAtom<'a> {
|
||||
Num(usize), Sym(&'a str), Key(&'a str), Exp(RefAtomIter<'a>),
|
||||
}
|
||||
impl<'a> RefAtom<'a> {
|
||||
pub fn to_arc_atom (&self) -> ArcAtom { todo!() }
|
||||
pub fn read_all <'b: 'a> (source: &'b str) -> impl Iterator<Item = ParseResult<RefAtom<'b>>> + 'b {
|
||||
TokenResultIterator::new(source).map(move |result|match result {
|
||||
Ok(token) => token.to_ref_atom(source), Err(e) => Err(e),
|
||||
})
|
||||
}
|
||||
}
|
||||
impl<'a> Atom for RefAtom<'a> {
|
||||
fn kind (&self) -> TokenKind {
|
||||
match self {
|
||||
Self::Num(_) => TokenKind::Num,
|
||||
Self::Sym(_) => TokenKind::Sym,
|
||||
Self::Key(_) => TokenKind::Key,
|
||||
Self::Exp(_) => TokenKind::Exp,
|
||||
}
|
||||
}
|
||||
fn text (&self) -> &str {
|
||||
match self {
|
||||
Self::Num(_)|Self::Exp(_) => "",
|
||||
Self::Sym(s) => s,
|
||||
Self::Key(k) => k,
|
||||
}
|
||||
}
|
||||
fn num (&self) -> usize { match self { Self::Num(n) => *n, _ => 0 } }
|
||||
fn exp (&self) -> impl Iterator<Item = RefAtom<'a>> {
|
||||
match self { Self::Exp(e) => *e, _ => RefAtomIter::from("") }
|
||||
}
|
||||
}
|
||||
impl<'a> Debug for RefAtom<'a> {
|
||||
fn fmt (&self, f: &mut Formatter<'_>) -> Result<(), FormatError> {
|
||||
match self {
|
||||
RefAtom::Num(n) => write!(f, "(num {n})"),
|
||||
RefAtom::Sym(t) => write!(f, "(sym {t:?})"),
|
||||
RefAtom::Key(t) => write!(f, "(key {t:?})"),
|
||||
RefAtom::Exp(u) => write!(f, "(exp {})", join(u.clone().map(|t|format!("{t:?}")), ","))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,26 +1,38 @@
|
|||
use crate::*;
|
||||
/// Map EDN tokens to parameters of a given type for a given context
|
||||
pub trait Context<U>: Sized {
|
||||
fn get (&self, _atom: &impl Atom) -> Option<U> {
|
||||
pub trait TryFromAtom<T>: Sized {
|
||||
fn try_from_atom (state: &T, value: Value) -> Option<Self> {
|
||||
if let Value::Exp(0, iter) = value { return Self::try_from_expr(state, iter) }
|
||||
None
|
||||
}
|
||||
fn get_or_fail (&self, atom: &impl Atom) -> U {
|
||||
fn try_from_expr (_state: &T, _iter: TokenIter) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
}
|
||||
pub trait TryIntoAtom<T>: Sized {
|
||||
fn try_into_atom (&self) -> Option<Value>;
|
||||
}
|
||||
/// Map EDN tokens to parameters of a given type for a given context
|
||||
pub trait Context<U>: Sized {
|
||||
fn get (&self, _atom: &Value) -> Option<U> {
|
||||
None
|
||||
}
|
||||
fn get_or_fail (&self, atom: &Value) -> U {
|
||||
self.get(atom).expect("no value")
|
||||
}
|
||||
}
|
||||
impl<T: Context<U>, U> Context<U> for &T {
|
||||
fn get (&self, atom: &impl Atom) -> Option<U> {
|
||||
fn get (&self, atom: &Value) -> Option<U> {
|
||||
(*self).get(atom)
|
||||
}
|
||||
fn get_or_fail (&self, atom: &impl Atom) -> U {
|
||||
fn get_or_fail (&self, atom: &Value) -> U {
|
||||
(*self).get_or_fail(atom)
|
||||
}
|
||||
}
|
||||
impl<T: Context<U>, U> Context<U> for Option<T> {
|
||||
fn get (&self, atom: &impl Atom) -> Option<U> {
|
||||
fn get (&self, atom: &Value) -> Option<U> {
|
||||
self.as_ref().map(|s|s.get(atom)).flatten()
|
||||
}
|
||||
fn get_or_fail (&self, atom: &impl Atom) -> U {
|
||||
fn get_or_fail (&self, atom: &Value) -> U {
|
||||
self.as_ref().map(|s|s.get_or_fail(atom)).expect("no provider")
|
||||
}
|
||||
}
|
||||
|
|
@ -29,9 +41,10 @@ impl<T: Context<U>, U> Context<U> for Option<T> {
|
|||
// Provide a value to the EDN template
|
||||
($type:ty:|$self:ident:$State:ty|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl Context<$type> for $State {
|
||||
fn get (&$self, atom: &impl Atom) -> Option<$type> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$((TokenKind::Sym, $pat) => $expr,)*
|
||||
#[allow(unreachable_code)]
|
||||
fn get (&$self, atom: &Value) -> Option<$type> {
|
||||
Some(match atom {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
|
|
@ -40,9 +53,10 @@ impl<T: Context<U>, U> Context<U> for Option<T> {
|
|||
// Provide a value more generically
|
||||
($lt:lifetime: $type:ty:|$self:ident:$State:ty|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl<$lt> Context<$lt, $type> for $State {
|
||||
fn get (&$lt $self, atom: &impl Atom) -> Option<$type> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$((TokenKind::Sym, $pat) => $expr,)*
|
||||
#[allow(unreachable_code)]
|
||||
fn get (&$lt $self, atom: &Value) -> Option<$type> {
|
||||
Some(match atom {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
|
|
@ -56,10 +70,10 @@ impl<T: Context<U>, U> Context<U> for Option<T> {
|
|||
// Provide a value that may also be a numeric literal in the EDN, to a generic implementation.
|
||||
($type:ty:|$self:ident:<$T:ident:$Trait:path>|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl<$T: $Trait> Context<$type> for $T {
|
||||
fn get (&$self, atom: &impl Atom) -> Option<$type> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$((TokenKind::Sym, $pat) => $expr,)*
|
||||
(TokenKind::Num, _) => atom.num() as $type,
|
||||
fn get (&$self, atom: &Value) -> Option<$type> {
|
||||
Some(match atom {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
Value::Num(n) => *n as $type,
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
|
|
@ -68,10 +82,10 @@ impl<T: Context<U>, U> Context<U> for Option<T> {
|
|||
// Provide a value that may also be a numeric literal in the EDN, to a concrete implementation.
|
||||
($type:ty:|$self:ident:$State:ty|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl Context<$type> for $State {
|
||||
fn get (&$self, atom: &impl Atom) -> Option<$type> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$((TokenKind::Sym, $pat) => $expr,)*
|
||||
(TokenKind::Num, _) => atom.num() as $type,
|
||||
fn get (&$self, atom: &Value) -> Option<$type> {
|
||||
Some(match atom {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
Value::Num(n) => *n as $type,
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
|
|
@ -84,17 +98,21 @@ impl<T: Context<U>, U> Context<U> for Option<T> {
|
|||
#[macro_export] macro_rules! provide_content {
|
||||
(|$self:ident:$State:ty|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl<'a, E: Output> Context<'a, Box<dyn Render<E> + 'a>> for $State {
|
||||
fn get (&'a $self, atom: &'a impl Atom) -> Option<Box<dyn Render<E> + 'a>> {
|
||||
use RefAtom::*;
|
||||
Some(match atom.to_ref() { $(RefAtom::Sym($pat) => $expr),*, _ => return None })
|
||||
fn get (&'a $self, atom: &'a Value) -> Option<Box<dyn Render<E> + 'a>> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
($Output:ty: |$self:ident:$State:ty|{ $($pat:pat => $expr:expr),* $(,)? }) => {
|
||||
impl<'a> Context<'a, Box<dyn Render<$Output> + 'a>> for $State {
|
||||
fn get (&'a $self, atom: &'a impl Atom) -> Option<Box<dyn Render<$Output> + 'a>> {
|
||||
use RefAtom::*;
|
||||
Some(match atom.to_ref() { $(Sym($pat) => $expr),*, _ => return None })
|
||||
fn get (&'a $self, atom: &'a Value) -> Option<Box<dyn Render<$Output> + 'a>> {
|
||||
Some(match (atom.kind(), atom.text()) {
|
||||
$(Value::Sym($pat) => $expr,)*
|
||||
_ => return None
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
87
edn/src/iter.rs
Normal file
87
edn/src/iter.rs
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
use crate::*;
|
||||
#[derive(Copy, Clone, Debug, PartialEq)] pub struct TokenIter<'a>(pub &'a str);
|
||||
const_iter!(<'a>|self: TokenIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result));
|
||||
impl<'a> From<&'a str> for TokenIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}}
|
||||
impl<'a> TokenIter<'a> {
|
||||
pub const fn new (source: &'a str) -> Self { Self(source) }
|
||||
pub const fn chomp (&self, index: usize) -> Self { Self(split_at(self.0, index).1) }
|
||||
pub const fn next (mut self) -> Option<(Token<'a>, Self)> { Self::next_mut(&mut self) }
|
||||
pub const fn peek (&self) -> Option<Token<'a>> { peek_src(self.0) }
|
||||
pub const fn next_mut (&mut self) -> Option<(Token<'a>, Self)> {
|
||||
match self.peek() {
|
||||
Some(token) => Some((token, self.chomp(token.end()))),
|
||||
None => None
|
||||
}
|
||||
}
|
||||
}
|
||||
pub const fn peek_src <'a> (source: &'a str) -> Option<Token<'a>> {
|
||||
let mut token: Token<'a> = Token::new(source, Nil, 0, 0);
|
||||
iterate!(char_indices(source) => (start, c) => token = match token.value() {
|
||||
Err(_) => return Some(token),
|
||||
Nil => match c {
|
||||
' '|'\n'|'\r'|'\t' => token.grow(),
|
||||
'(' => Token {
|
||||
source, start, length: 1,
|
||||
value: Value::Exp(1, TokenIter(str_range(source, start, start + 1))),
|
||||
},
|
||||
'0'..='9' => Token {
|
||||
source, start, length: 1,
|
||||
value: match to_digit(c) {
|
||||
Ok(c) => Value::Num(c),
|
||||
Result::Err(e) => Value::Err(e)
|
||||
}
|
||||
},
|
||||
':'|'@' => Token {
|
||||
source, start, length: 1,
|
||||
value: Value::Sym(str_range(source, start, start + 1)),
|
||||
},
|
||||
'/'|'a'..='z' => Token {
|
||||
source, start, length: 1,
|
||||
value: Value::Key(str_range(source, start, start + 1)),
|
||||
},
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Num(_) => match c {
|
||||
'0'..='9' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Sym(_) => match c {
|
||||
'a'..='z'|'0'..='9'|'-' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Key(_) => match c {
|
||||
'a'..='z'|'0'..='9'|'-'|'/' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Exp(depth, _) => match depth {
|
||||
0 => return Some(token),
|
||||
_ => match c {
|
||||
')' => token.grow_out(),
|
||||
'(' => token.grow_in(),
|
||||
_ => token.grow(),
|
||||
}
|
||||
},
|
||||
});
|
||||
match token.value() {
|
||||
Nil => None,
|
||||
_ => Some(token),
|
||||
}
|
||||
}
|
||||
pub const fn to_number (digits: &str) -> Result<usize, ParseError> {
|
||||
let mut value = 0;
|
||||
iterate!(char_indices(digits) => (_, c) => match to_digit(c) {
|
||||
Ok(digit) => value = 10 * value + digit,
|
||||
Result::Err(e) => return Result::Err(e)
|
||||
});
|
||||
Ok(value)
|
||||
}
|
||||
pub const fn to_digit (c: char) -> Result<usize, ParseError> {
|
||||
Ok(match c {
|
||||
'0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4,
|
||||
'5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9,
|
||||
_ => return Result::Err(Unexpected(c))
|
||||
})
|
||||
}
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
#![feature(adt_const_params)]
|
||||
#![feature(type_alias_impl_trait)]
|
||||
#![feature(impl_trait_in_fn_trait_return)]
|
||||
mod error; pub use self::error::*;
|
||||
mod token; pub use self::token::*;
|
||||
mod atom; pub use self::atom::*;
|
||||
mod atom_ref; pub use self::atom_ref::*;
|
||||
mod atom_arc; pub use self::atom_arc::*;
|
||||
mod context; pub use self::context::*;
|
||||
mod error; pub use self::error::*;
|
||||
mod token; pub use self::token::*;
|
||||
mod iter; pub use self::iter::*;
|
||||
mod context; pub use self::context::*;
|
||||
//mod atom; pub use self::atom::*;
|
||||
//mod atom_ref; pub use self::atom_ref::*;
|
||||
//mod atom_arc; pub use self::atom_arc::*;
|
||||
pub(crate) use self::Value::*;
|
||||
pub(crate) use self::ParseError::*;
|
||||
//pub(crate) use self::TokenKind::*;
|
||||
pub(crate) use std::sync::Arc;
|
||||
|
|
@ -40,6 +42,17 @@ pub(crate) use std::fmt::{Debug, Display, Formatter, Result as FormatResult, Err
|
|||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)] #[test] fn test_token () -> Result<(), Box<dyn std::error::Error>> {
|
||||
use Token::*;
|
||||
assert_eq!(Nil, Token::chomp_one("")?);
|
||||
assert_eq!(Nil, Token::chomp_one(" \n \r \t ")?);
|
||||
assert_eq!(Num("8", 0, 1), Token::chomp_one("8")?);
|
||||
assert_eq!(Num(" 8 ", 3, 1), Token::chomp_one(" 8 ")?);
|
||||
assert_eq!(Sym(":foo", 0, 4), Token::chomp_one(":foo")?);
|
||||
assert_eq!(Sym("@bar", 0, 4), Token::chomp_one("@bar")?);
|
||||
assert_eq!(Key("foo/bar", 0, 7), Token::chomp_one("foo/bar")?);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(test)] #[test] fn test_lang () -> Result<(), ParseError> {
|
||||
use Atom::*;
|
||||
assert_eq!(Atom::read_all("")?,
|
||||
|
|
|
|||
146
edn/src/token.rs
146
edn/src/token.rs
|
|
@ -1,12 +1,12 @@
|
|||
use crate::*;
|
||||
use self::TokenKind::*;
|
||||
use self::Value::*;
|
||||
#[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> {
|
||||
source: &'a str,
|
||||
start: usize,
|
||||
length: usize,
|
||||
kind: TokenKind<'a>,
|
||||
pub source: &'a str,
|
||||
pub start: usize,
|
||||
pub length: usize,
|
||||
pub value: Value<'a>,
|
||||
}
|
||||
#[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum TokenKind<'a> {
|
||||
#[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum Value<'a> {
|
||||
#[default] Nil,
|
||||
Err(ParseError),
|
||||
Num(usize),
|
||||
|
|
@ -14,24 +14,9 @@ use self::TokenKind::*;
|
|||
Key(&'a str),
|
||||
Exp(usize, TokenIter<'a>),
|
||||
}
|
||||
#[derive(Copy, Clone, Debug, PartialEq)] pub struct TokenIter<'a>(pub &'a str);
|
||||
const_iter!(<'a>|self: TokenIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result));
|
||||
impl<'a> From<&'a str> for TokenIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}}
|
||||
impl<'a> TokenIter<'a> {
|
||||
pub const fn new (source: &'a str) -> Self { Self(source) }
|
||||
pub const fn chomp (&self, index: usize) -> Self { Self(split_at(self.0, index).1) }
|
||||
pub const fn next (mut self) -> Option<(Token<'a>, Self)> { Self::next_mut(&mut self) }
|
||||
pub const fn peek (&self) -> Option<Token<'a>> { peek_src(self.0) }
|
||||
pub const fn next_mut (&mut self) -> Option<(Token<'a>, Self)> {
|
||||
match self.peek() {
|
||||
Some(token) => Some((token, self.chomp(token.end()))),
|
||||
None => None
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<'a> Token<'a> {
|
||||
pub const fn new (source: &'a str, kind: TokenKind<'a>, start: usize, length: usize) -> Self {
|
||||
Self { source, kind, start, length }
|
||||
pub const fn new (source: &'a str, value: Value<'a>, start: usize, length: usize) -> Self {
|
||||
Self { source, value, start, length }
|
||||
}
|
||||
pub const fn end (&self) -> usize {
|
||||
self.start + self.length
|
||||
|
|
@ -43,131 +28,34 @@ impl<'a> Token<'a> {
|
|||
pub const fn slice_source <'b> (&'a self, source: &'b str) -> &'b str {
|
||||
str_range(source, self.start, self.end())
|
||||
}
|
||||
pub const fn kind (&self) -> TokenKind {
|
||||
self.kind
|
||||
pub const fn value (&self) -> Value {
|
||||
self.value
|
||||
}
|
||||
pub const fn error (self, error: ParseError) -> Self {
|
||||
Self { kind: TokenKind::Err(error), ..self }
|
||||
Self { value: Value::Err(error), ..self }
|
||||
}
|
||||
pub const fn grow (self) -> Self {
|
||||
Self { length: self.length + 1, ..self }
|
||||
}
|
||||
pub const fn grow_in (self) -> Self {
|
||||
//Self { length: self.length + 1, depth: self.depth + 1, ..self };
|
||||
match self.kind {
|
||||
TokenKind::Exp(depth, source) => Self {
|
||||
kind: TokenKind::Exp(depth + 1, TokenIter(self.grow().slice_source(source.0))),
|
||||
match self.value {
|
||||
Value::Exp(depth, source) => Self {
|
||||
value: Value::Exp(depth + 1, TokenIter(self.grow().slice_source(source.0))),
|
||||
..self.grow()
|
||||
},
|
||||
_ => self.grow()
|
||||
}
|
||||
}
|
||||
pub const fn grow_out (self) -> Self {
|
||||
match self.kind {
|
||||
TokenKind::Exp(depth, source) => match depth {
|
||||
match self.value {
|
||||
Value::Exp(depth, source) => match depth {
|
||||
0 => self.error(Unexpected(')')),
|
||||
d => Self {
|
||||
kind: TokenKind::Exp(d - 1, TokenIter(self.grow().slice_source(source.0))),
|
||||
value: Value::Exp(d - 1, TokenIter(self.grow().slice_source(source.0))),
|
||||
..self.grow()
|
||||
}
|
||||
},
|
||||
_ => self.grow()
|
||||
}
|
||||
}
|
||||
//pub const fn to_ref_atom (&self, source: &'a str) -> RefAtom<'a> {
|
||||
//match self.kind {
|
||||
//TokenKind::Nil => Err(ParseError::Empty),
|
||||
//TokenKind::Num => match to_number(self.slice_source(source)) {
|
||||
//Err(e) => Err(e),
|
||||
//Ok(n) => Ok(RefAtom::Num(n)),
|
||||
//},
|
||||
//TokenKind::Sym => Ok(RefAtom::Sym(self.slice_source(source))),
|
||||
//TokenKind::Key => Ok(RefAtom::Key(self.slice_source(source))),
|
||||
//TokenKind::Exp => Ok(
|
||||
//RefAtom::Exp(RefAtomIterator(TokenIter::new(self.slice_source(source))))
|
||||
//)
|
||||
//}
|
||||
//}
|
||||
}
|
||||
pub const fn peek_src <'a> (source: &'a str) -> Option<Token<'a>> {
|
||||
let mut token: Token<'a> = Token::new(source, Nil, 0, 0);
|
||||
iterate!(char_indices(source) => (start, c) => token = match token.kind() {
|
||||
Err(_) => return Some(token),
|
||||
Nil => match c {
|
||||
' '|'\n'|'\r'|'\t' => token.grow(),
|
||||
'(' => Token {
|
||||
source, start, length: 1,
|
||||
kind: TokenKind::Exp(1, TokenIter(str_range(source, start, start + 1))),
|
||||
},
|
||||
'0'..='9' => Token {
|
||||
source, start, length: 1,
|
||||
kind: match to_digit(c) {
|
||||
Ok(c) => TokenKind::Num(c),
|
||||
Result::Err(e) => TokenKind::Err(e)
|
||||
}
|
||||
},
|
||||
':'|'@' => Token {
|
||||
source, start, length: 1,
|
||||
kind: TokenKind::Sym(str_range(source, start, start + 1)),
|
||||
},
|
||||
'/'|'a'..='z' => Token {
|
||||
source, start, length: 1,
|
||||
kind: TokenKind::Key(str_range(source, start, start + 1)),
|
||||
},
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Num(_) => match c {
|
||||
'0'..='9' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Sym(_) => match c {
|
||||
'a'..='z'|'0'..='9'|'-' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Key(_) => match c {
|
||||
'a'..='z'|'0'..='9'|'-'|'/' => token.grow(),
|
||||
' '|'\n'|'\r'|'\t' => return Some(token),
|
||||
_ => token.error(Unexpected(c))
|
||||
},
|
||||
Exp(depth, _) => match depth {
|
||||
0 => return Some(token),
|
||||
_ => match c {
|
||||
')' => token.grow_out(),
|
||||
'(' => token.grow_in(),
|
||||
_ => token.grow(),
|
||||
}
|
||||
},
|
||||
});
|
||||
match token.kind() {
|
||||
Nil => None,
|
||||
_ => Some(token),
|
||||
}
|
||||
}
|
||||
pub const fn to_number (digits: &str) -> Result<usize, ParseError> {
|
||||
let mut value = 0;
|
||||
iterate!(char_indices(digits) => (_, c) => match to_digit(c) {
|
||||
Ok(digit) => value = 10 * value + digit,
|
||||
Result::Err(e) => return Result::Err(e)
|
||||
});
|
||||
Ok(value)
|
||||
}
|
||||
pub const fn to_digit (c: char) -> Result<usize, ParseError> {
|
||||
Ok(match c {
|
||||
'0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4,
|
||||
'5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9,
|
||||
_ => return Result::Err(Unexpected(c))
|
||||
})
|
||||
}
|
||||
#[cfg(test)] #[test] fn test_token () -> Result<(), Box<dyn std::error::Error>> {
|
||||
use Token::*;
|
||||
assert_eq!(Nil, Token::chomp_one("")?);
|
||||
assert_eq!(Nil, Token::chomp_one(" \n \r \t ")?);
|
||||
assert_eq!(Num("8", 0, 1), Token::chomp_one("8")?);
|
||||
assert_eq!(Num(" 8 ", 3, 1), Token::chomp_one(" 8 ")?);
|
||||
assert_eq!(Sym(":foo", 0, 4), Token::chomp_one(":foo")?);
|
||||
assert_eq!(Sym("@bar", 0, 4), Token::chomp_one("@bar")?);
|
||||
assert_eq!(Key("foo/bar", 0, 7), Token::chomp_one("foo/bar")?);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue