#![feature(adt_const_params)] #![feature(type_alias_impl_trait)] #![feature(impl_trait_in_fn_trait_return)] //! [Token]s are parsed substrings with an associated [Value]. //! //! * [ ] FIXME: Value may be [Err] which may shadow [Result::Err] //! * [Value::Exp] wraps an expression depth and a [SourceIter] //! with the remaining part of the expression. //! * expression depth other that 0 mean unclosed parenthesis. //! * closing and unopened parenthesis panics during reading. //! * [ ] TODO: signed depth might be interesting //! * [Value::Sym] and [Value::Key] are stringish literals //! with slightly different parsing rules. //! * [Value::Num] is an unsigned integer literal. //!``` //! use tengri_dsl::{*, Value::*}; //! let source = include_str!("../test.edn"); //! let mut view = TokenIter::new(source); //! assert_eq!(view.peek(), Some(Token { //! source, //! start: 0, //! length: source.len(), //! value: Exp(0, TokenIter::new(&source[1..])) //! })); //!``` //! The token iterator [TokenIter] allows you to get the //! general-purpose syntactic [Token]s represented by the source text. //! //! Both iterators are `peek`able: //! //! ``` //! let src = include_str!("../test.edn"); //! let mut view = tengri_dsl::TokenIter::new(src); //! assert_eq!(view.0.0, src); //! assert_eq!(view.peek(), view.0.peek()) //! ``` pub(crate) use ::tengri_core::*; pub(crate) use std::fmt::Debug; pub(crate) use konst::iter::{ConstIntoIter, IsIteratorKind}; pub(crate) use konst::string::{split_at, str_range, char_indices}; pub(crate) use thiserror::Error; pub(crate) use self::Value::*; pub(crate) use self::DslError::*; mod dsl_error; pub use self::dsl_error::*; mod dsl_parse; pub use self::dsl_parse::*; mod dsl_provide; pub use self::dsl_provide::*; #[cfg(test)] mod test_token_iter { use crate::*; //use proptest::prelude::*; #[test] fn test_iters () { let mut iter = crate::SourceIter::new(&":foo :bar"); let _ = iter.next(); let mut iter = crate::TokenIter::new(&":foo :bar"); let _ = iter.next(); } #[test] const fn test_const_iters () { let mut iter = crate::SourceIter::new(&":foo :bar"); let _ = iter.next(); } #[test] fn test_num () { let digit = to_digit('0'); let digit = to_digit('x'); let number = to_number(&"123"); let number = to_number(&"12asdf3"); } //proptest! { //#[test] fn proptest_source_iter ( //source in "\\PC*" //) { //let mut iter = crate::SourceIter::new(&source); ////let _ = iter.next(); //} //#[test] fn proptest_token_iter ( //source in "\\PC*" //) { //let mut iter = crate::TokenIter::new(&source); ////let _ = iter.next(); //} //} } #[cfg(test)] mod test_token_prop { use proptest::prelude::*; proptest! { #[test] fn test_token_prop ( source in "\\PC*", start in usize::MIN..usize::MAX, length in usize::MIN..usize::MAX, ) { let token = crate::Token { source: &source, start, length, value: crate::Value::Nil }; let _ = token.slice(); } } } #[cfg(test)] #[test] fn test_token () -> Result<(), Box> { let source = ":f00"; let mut token = Token { source, start: 0, length: 1, value: Sym(":") }; token = token.grow_sym(); assert_eq!(token, Token { source, start: 0, length: 2, value: Sym(":f") }); token = token.grow_sym(); assert_eq!(token, Token { source, start: 0, length: 3, value: Sym(":f0") }); token = token.grow_sym(); assert_eq!(token, Token { source, start: 0, length: 4, value: Sym(":f00") }); let src = ""; assert_eq!(None, SourceIter(src).next()); let src = " \n \r \t "; assert_eq!(None, SourceIter(src).next()); let src = "7"; assert_eq!(Num(7), SourceIter(src).next().unwrap().0.value); let src = " 100 "; assert_eq!(Num(100), SourceIter(src).next().unwrap().0.value); let src = " 9a "; assert_eq!(Err(Unexpected('a')), SourceIter(src).next().unwrap().0.value); let src = " :123foo "; assert_eq!(Sym(":123foo"), SourceIter(src).next().unwrap().0.value); let src = " \r\r\r\n\n\n@bar456\t\t\t\t\t\t"; assert_eq!(Sym("@bar456"), SourceIter(src).next().unwrap().0.value); let src = "foo123"; assert_eq!(Key("foo123"), SourceIter(src).next().unwrap().0.value); let src = "foo/bar"; assert_eq!(Key("foo/bar"), SourceIter(src).next().unwrap().0.value); let src = "\"foo/bar\""; assert_eq!(Str("foo/bar"), SourceIter(src).next().unwrap().0.value); let src = " \"foo/bar\" "; assert_eq!(Str("foo/bar"), SourceIter(src).next().unwrap().0.value); Ok(()) } //#[cfg(test)] #[test] fn test_examples () -> Result<(), DslError> { //// Let's pretend to render some view. //let source = include_str!("../../tek/src/view_arranger.edn"); //// The token iterator allows you to get the tokens represented by the source text. //let mut view = TokenIter(source); //// The token iterator wraps a const token+source iterator. //assert_eq!(view.0.0, source); //let mut expr = view.peek(); //assert_eq!(view.0.0, source); //assert_eq!(expr, Some(Token { //source, start: 0, length: source.len() - 1, value: Exp(0, SourceIter(&source[1..])) //})); ////panic!("{view:?}"); ////panic!("{:#?}", expr); ////for example in [ ////include_str!("../../tui/examples/edn01.edn"), ////include_str!("../../tui/examples/edn02.edn"), ////] { //////let items = Dsl::read_all(example)?; //////panic!("{layout:?}"); //////let content = >::from(&layout); ////} //Ok(()) //}