const tokenization

This commit is contained in:
🪞👃🪞 2025-01-08 19:46:12 +01:00
parent 113e7b0bad
commit 0d9a4d4830

View file

@ -10,11 +10,12 @@ pub enum Token<'a> {
} }
impl<'a> Token<'a> { impl<'a> Token<'a> {
pub fn chomp (source: &'a str) -> Result<(&'a str, Self), ParseError> { pub const fn chomp (source: &'a str) -> Result<(&'a str, Self), ParseError> {
use Token::*; use Token::*;
use konst::string::{split_at, char_indices}; use konst::string::{split_at, char_indices};
let mut state = Self::Nil; let mut state = Self::Nil;
for (index, c) in source.char_indices() { let mut chars = char_indices(source);
while let Some(((index, c), next)) = chars.next() {
state = match state { state = match state {
// must begin expression // must begin expression
Nil => match c { Nil => match c {
@ -64,19 +65,24 @@ impl<'a> Token<'a> {
'(' => Exp(source, index, length + 1, depth + 1), '(' => Exp(source, index, length + 1, depth + 1),
_ => Exp(source, index, length + 1, depth) _ => Exp(source, index, length + 1, depth)
}, },
} };
chars = next
} }
Ok(("", state)) Ok(("", state))
} }
pub fn number (digits: &str) -> usize { pub const fn number (digits: &str) -> usize {
let mut value = 0; let mut value = 0;
for c in digits.chars() { value = 10 * value + Self::digit(c); } let mut chars = konst::string::char_indices(digits);
while let Some(((_, c), next)) = chars.next() {
value = 10 * value + Self::digit(c);
chars = next
}
value value
} }
pub const fn digit (c: char) -> usize { pub const fn digit (c: char) -> usize {
match c { '0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4, match c { '0' => 0, '1' => 1, '2' => 2, '3' => 3, '4' => 4,
'5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9, _ => unreachable!() } '5' => 5, '6' => 6, '7' => 7, '8' => 8, '9' => 9, _ => panic!("not a digit") }
} }
} }