document stuff; Thunk suffix -> prefix

edn/src/iter.rs

@@ -1,4 +1,18 @@
+//! The token iterator [TokenIter] allows you to get the
+//! general-purpose syntactic [Token]s represented by the source text.
+//!
+//! Both iterators are `peek`able:
+//!
+//! ```
+//! let src = include_str!("../../tek/src/view_arranger.edn")
+//! let mut view = TokenIter(src);
+//! assert_eq!(view.0.0, src);
+//! assert_eq!(src.peek(), src.0.peek()))
+//! ```
 use crate::*;
+/// Provides a native [Iterator] API over the [ConstIntoIter] [SourceIter]
+/// [TokenIter::next] returns just the [Token] and mutates `self`,
+/// instead of returning an updated version of the struct as [SourceIter::next] does.
 #[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct TokenIter<'a>(pub SourceIter<'a>);
 impl<'a> TokenIter<'a> {
     pub const fn new (source: &'a str) -> Self { Self(SourceIter::new(source)) }
@@ -10,6 +24,11 @@ impl<'a> Iterator for TokenIter<'a> {
         self.0.next().map(|(item, rest)|{self.0 = rest; item})
     }
 }
+/// Owns a reference to the source text.
+/// [SourceIter::next] emits subsequent pairs of:
+///   * a [Token] and
+///   * the source text remaining
+///     * [ ] TODO: maybe [SourceIter::next] should wrap the remaining source in `Self` ?
 #[derive(Copy, Clone, Debug, Default, PartialEq)] pub struct SourceIter<'a>(pub &'a str);
 const_iter!(<'a>|self: SourceIter<'a>| => Token<'a> => self.next_mut().map(|(result, _)|result));
 impl<'a> From<&'a str> for SourceIter<'a> {fn from (source: &'a str) -> Self{Self::new(source)}}

edn/src/lib.rs

@@ -5,15 +5,8 @@ mod error;   pub use self::error::*;
 mod token;   pub use self::token::*;
 mod iter;    pub use self::iter::*;
 mod context; pub use self::context::*;
-//mod atom;     pub use self::atom::*;
-//mod atom_ref; pub use self::atom_ref::*;
-//mod atom_arc; pub use self::atom_arc::*;
 pub(crate) use self::Value::*;
 pub(crate) use self::ParseError::*;
-//pub(crate) use self::TokenKind::*;
-pub(crate) use std::sync::Arc;
-//pub(crate) use std::marker::ConstParamTy;
-pub(crate) use itertools::join;
 pub(crate) use konst::iter::{ConstIntoIter, IsIteratorKind};
 pub(crate) use konst::string::{split_at, str_range, char_indices};
 pub(crate) use std::error::Error;
@@ -81,27 +74,27 @@ pub(crate) use std::fmt::{Debug, Display, Formatter, Result as FormatResult, Err
 
     Ok(())
 }
-#[cfg(test)] #[test] fn test_examples () -> Result<(), ParseError> {    
-    let src = include_str!("../../tek/src/view_arranger.edn");
-    let mut view = SourceIter(src);
-    assert_eq!(view.0, src);
-    let mut expr = view.peek();
-    assert_eq!(view.0, src);
-    assert_eq!(expr, Some(Token {
-        source: src,
-        start:  0,
-        length: src.len(),
-        value:  Exp(0, SourceIter(&src[1..]))
-    }));
-    //panic!("{view:?}");
-    //panic!("{:#?}", expr);
-    //for example in [
-        //include_str!("../../tui/examples/edn01.edn"),
-        //include_str!("../../tui/examples/edn02.edn"),
-    //] {
-        ////let items = Atom::read_all(example)?;
-        ////panic!("{layout:?}");
-        ////let content = <dyn ViewContext<::tek_engine::tui::Tui>>::from(&layout);
-    //}
-    Ok(())
-}
+//#[cfg(test)] #[test] fn test_examples () -> Result<(), ParseError> {    
+    //// Let's pretend to render some view.
+    //let source = include_str!("../../tek/src/view_arranger.edn");
+    //// The token iterator allows you to get the tokens represented by the source text.
+    //let mut view = TokenIter(source);
+    //// The token iterator wraps a const token+source iterator.
+    //assert_eq!(view.0.0, source);
+    //let mut expr = view.peek();
+    //assert_eq!(view.0.0, source);
+    //assert_eq!(expr, Some(Token {
+        //source, start: 0, length: source.len() - 1, value: Exp(0, SourceIter(&source[1..]))
+    //}));
+    ////panic!("{view:?}");
+    ////panic!("{:#?}", expr);
+    ////for example in [
+        ////include_str!("../../tui/examples/edn01.edn"),
+        ////include_str!("../../tui/examples/edn02.edn"),
+    ////] {
+        //////let items = Atom::read_all(example)?;
+        //////panic!("{layout:?}");
+        //////let content = <dyn ViewContext<::tek_engine::tui::Tui>>::from(&layout);
+    ////}
+    //Ok(())
+//}

edn/src/token.rs

@@ -1,3 +1,24 @@
+//! [Token]s are parsed substrings with an associated [Value].
+//!
+//! * [ ] FIXME: Value may be [Err] which may shadow [Result::Err]
+//! * [Value::Exp] wraps an expression depth and a [SourceIter]
+//!   with the remaining part of the expression.
+//!   * expression depth other that 0 mean unclosed parenthesis.
+//!   * closing and unopened parenthesis panics during reading.
+//!   * [ ] TODO: signed depth might be interesting
+//! * [Value::Sym] and [Value::Key] are stringish literals
+//!   with slightly different parsing rules.
+//! * [Value::Num] is an unsigned integer literal.
+//!```
+//! let src = include_str!("../../tek/src/view_arranger.edn")
+//! let mut view = TokenIter(src);
+//! assert_eq!(source.peek(), Some(Token {
+//!     source,
+//!     start: 0,
+//!     length: source.len() - 1,
+//!     value: Exp(0, SourceIter(&source[1..]))
+//! })))
+//!```
 use crate::*;
 use self::Value::*;
 #[derive(Debug, Copy, Clone, Default, PartialEq)] pub struct Token<'a> {