update with new user function lex_formula

Author: alexhallam

CHANGELOG.md

@@ -1,3 +1,9 @@
+# Changelog
+
+## [0.2.2] - 2025-09-05
+
+added `lex_formula` for users to inspect raw lexer output
+
 ## [0.2.1] - 2025-09-04
 
 just to trigger a new release

README.md

@@ -247,6 +247,28 @@ let metadata = parse_formula(formula).unwrap();
 println!("{}", serde_json::to_string_pretty(&metadata).unwrap());
 ```
 
+### Inspect Tokens: `lex_formula`
+
+If you want to inspect how the lexer tokenizes a formula (useful when debugging parse errors
+or understanding how functions and interactions are split), use `lex_formula` which returns a
+JSON array of token objects with `token` and `lexeme` fields.
+
+```rust
+use fiasto::lex_formula;
+
+let input = "mpg ~ cyl + wt*hp + poly(disp, 4) - 1";
+let tokens = lex_formula(input).unwrap();
+println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
+```
+
+This prints objects like:
+
+```json
+{ "token": "ColumnName", "lexeme": "mpg" }
+{ "token": "Tilde", "lexeme": "~" }
+{ "token": "Plus", "lexeme": "+" }
+```
+
 ### Basic Formula
 ```rust
 use fiasto::parse_formula;

examples/lex_formula.rs

@@ -0,0 +1,8 @@
+use fiasto::lex_formula;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let input = "mpg ~ cyl + wt*hp + poly(disp, 4) - 1";
+    let tokens = lex_formula(input)?;
+    println!("{}", serde_json::to_string_pretty(&tokens)?);
+    Ok(())
+}

src/lib.rs

@@ -299,3 +299,46 @@ pub fn parse_formula(formula: &str) -> Result<Value, Box<dyn std::error::Error>>
 
     Ok(serde_json::to_value(meta)?)
 }
+
+/// Lex a formula and return JSON describing each token.
+///
+/// The output is an array of objects with fields:
+/// - `token`: token name (enum debug)
+/// - `lexeme`: the original slice from the input
+///
+/// # Example
+///
+/// ```rust
+/// use fiasto::lex_formula;
+///
+/// let formula = "mpg ~ cyl + wt*hp + poly(disp, 4) - 1";
+/// let tokens = lex_formula(formula).unwrap();
+/// // tokens is a serde_json::Value::Array of objects like:
+/// // { "token": "ColumnName", "lexeme": "mpg" }
+/// // { "token": "Tilde", "lexeme": "~" }
+/// // { "token": "Plus", "lexeme": "+" }
+/// println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
+/// ```
+pub fn lex_formula(formula: &str) -> Result<Value, Box<dyn std::error::Error>> {
+    use logos::Logos;
+    use crate::internal::lexer::Token;
+
+    let mut lex = Token::lexer(formula);
+    let mut tokens = Vec::new();
+    while let Some(item) = lex.next() {
+        match item {
+            Ok(tok) => {
+                let slice = lex.slice();
+                let obj = serde_json::json!({
+                    "token": format!("{:?}", tok),
+                    "lexeme": slice,
+                });
+                tokens.push(obj);
+            }
+            Err(()) => {
+                return Err(Box::new(crate::internal::errors::ParseError::Lex(lex.slice().to_string())));
+            }
+        }
+    }
+    Ok(serde_json::Value::Array(tokens))
+}