use crate::*; use self::TokenKind::*; #[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> { source: &'a str, start: usize, length: usize, kind: TokenKind<'a>, } #[derive(Debug, Copy, Clone, Default, PartialEq)] pub enum TokenKind<'a> { #[default] Nil, Err(ParseError), Num(usize), Sym(&'a str), Key(&'a str), Exp(usize, TokenIter<'a>), } #[derive(Copy, Clone, Debug, PartialEq)] pub struct TokenIter<'a>(pub &'a str); const_iter!(<'a>|self: TokenIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result)); impl<'a> From<&'a str> for TokenIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}} impl<'a> TokenIter<'a> { pub const fn new (source: &'a str) -> Self { Self(source) } pub const fn chomp (&self, index: usize) -> Self { Self(split_at(self.0, index).1) } pub const fn next (mut self) -> Option<(Token<'a>, Self)> { Self::next_mut(&mut self) } pub const fn peek (&self) -> Option> { peek_src(self.0) } pub const fn next_mut (&mut self) -> Option<(Token<'a>, Self)> { match self.peek() { Some(token) => Some((token, self.chomp(token.end()))), None => None } } } impl<'a> Token<'a> { pub const fn new (source: &'a str, kind: TokenKind<'a>, start: usize, length: usize) -> Self { Self { source, kind, start, length } } pub const fn end (&self) -> usize { self.start + self.length } pub const fn slice (&'a self) -> &'a str { self.slice_source(self.source) //str_range(self.source, self.start, self.end()) } pub const fn slice_source <'b> (&'a self, source: &'b str) -> &'b str { str_range(source, self.start, self.end()) } pub const fn kind (&self) -> TokenKind { self.kind } pub const fn error (self, error: ParseError) -> Self { Self { kind: TokenKind::Err(error), ..self } } pub const fn grow (self) -> Self { Self { length: self.length + 1, ..self } } pub const fn grow_in (self) -> Self { //Self { length: self.length + 1, depth: self.depth + 1, ..self }; match self.kind { TokenKind::Exp(depth, source) => Self { kind: TokenKind::Exp(depth + 1, TokenIter(self.grow().slice_source(source.0))), ..self.grow() }, _ => self.grow() } } pub const fn grow_out (self) -> Self { match self.kind { TokenKind::Exp(depth, source) => match depth { 0 => self.error(Unexpected(')')), d => Self { kind: TokenKind::Exp(d - 1, TokenIter(self.grow().slice_source(source.0))), ..self.grow() } }, _ => self.grow() } } //pub const fn to_ref_atom (&self, source: &'a str) -> RefAtom<'a> { //match self.kind { //TokenKind::Nil => Err(ParseError::Empty), //TokenKind::Num => match to_number(self.slice_source(source)) { //Err(e) => Err(e), //Ok(n) => Ok(RefAtom::Num(n)), //}, //TokenKind::Sym => Ok(RefAtom::Sym(self.slice_source(source))), //TokenKind::Key => Ok(RefAtom::Key(self.slice_source(source))), //TokenKind::Exp => Ok( //RefAtom::Exp(RefAtomIterator(TokenIter::new(self.slice_source(source)))) //) //} //} } pub const fn peek_src <'a> (source: &'a str) -> Option> { let mut token: Token<'a> = Token::new(source, Nil, 0, 0); iterate!(char_indices(source) => (start, c) => token = match token.kind() { Err(_) => return Some(token), Nil => match c { ' '|'\n'|'\r'|'\t' => token.grow(), '(' => Token { source, start, length: 1, kind: TokenKind::Exp(1, TokenIter(str_range(source, start, start + 1))), }, '0'..='9' => Token { source, start, length: 1, kind: match to_digit(c) { Ok(c) => TokenKind::Num(c), Result::Err(e) => TokenKind::Err(e) } }, ':'|'@' => Token { source, start, length: 1, kind: TokenKind::Sym(str_range(source, start, start + 1)), }, '/'|'a'..='z' => Token { source, start, length: 1, kind: TokenKind::Key(str_range(source, start, start + 1)), }, _ => token.error(Unexpected(c)) }, Num(_) => match c { '0'..='9' => token.grow(), ' '|'\n'|'\r'|'\t' => return Some(token), _ => token.error(Unexpected(c)) }, Sym(_) => match c { 'a'..='z'|'0'..='9'|'-' => token.grow(), ' '|'\n'|'\r'|'\t' => return Some(token), _ => token.error(Unexpected(c)) }, Key(_) => match c { 'a'..='z'|'0'..='9'|'-'|'/' => token.grow(), ' '|'\n'|'\r'|'\t' => return Some(token), _ => token.error(Unexpected(c)) }, Exp(depth, _) => match depth { 0 => return Some(token), _ => match c { ')' => token.grow_out(), '(' => token.grow_in(), _ => token.grow(), } }, }); match token.kind() { Nil => None, _ => Some(token), } } pub const fn to_number (digits: &str) -> Result { let mut value = 0; iterate!(char_indices(digits) => (_, c) => match to_digit(c) { Ok(digit) => value = 10 * value + digit, Result::Err(e) => return Result::Err(e) }); Ok(value) } pub const fn to_digit (c: char) -> Result { Ok(match c { '0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4, '5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9, _ => return Result::Err(Unexpected(c)) }) } #[cfg(test)] #[test] fn test_token () -> Result<(), Box> { use Token::*; assert_eq!(Nil, Token::chomp_one("")?); assert_eq!(Nil, Token::chomp_one(" \n \r \t ")?); assert_eq!(Num("8", 0, 1), Token::chomp_one("8")?); assert_eq!(Num(" 8 ", 3, 1), Token::chomp_one(" 8 ")?); assert_eq!(Sym(":foo", 0, 4), Token::chomp_one(":foo")?); assert_eq!(Sym("@bar", 0, 4), Token::chomp_one("@bar")?); assert_eq!(Key("foo/bar", 0, 7), Token::chomp_one("foo/bar")?); Ok(()) }