Add visualization of ASTs to CLI & REPL (#158)

* Add visualization of ASTs to CLI & REPL

- Add actual arg parsing to CLI
- Make REPL a sub-mode initiated by a CLI command
- Add partial (lots of `todo!()`s) translation of AST into graphviz
- Add FFI to graphviz to layout & render
- Add `display` mode which prints image to console
- Add docs for above to README

Author: jpschorr

partiql-cli/Cargo.toml

@@ -33,22 +33,53 @@ partiql-source-map = { path = "../partiql-source-map" }
 partiql-ast = { path = "../partiql-ast" }
 
 
-rustyline = "9.1.2"
-syntect = "5.0"
-owo-colors = "3.4.0"
-supports-color = "1.3.0"
-supports-unicode = "1.0.2"
-supports-hyperlinks = "1.2.0"
-termbg = "0.4.1"
-shellexpand = "2.1.0"
-partiql-parser = { path = "../partiql-parser" }
-partiql-source-map = { path = "../partiql-source-map" }
-partiql-ast = { path = "../partiql-ast" }
+rustyline = "10.*"
+syntect = "5.*"
+owo-colors = "3.*"
+supports-color = "1.*"
+supports-unicode = "1.*"
+supports-hyperlinks = "1.*"
+termbg = "0.4.*"
+shellexpand = "2.*"
+
 
+thiserror = "1.*"
+miette = { version ="5.*", features = ["fancy"] }
+clap = { version = "3.*", features = ["derive"] }
 
-thiserror = "1.0.31"
-miette = { version ="4.7.1", features = ["fancy"] }
+# serde
+serde = { version ="1.*", features = ["derive"], optional = true }
+serde_json = { version ="1.*", optional = true }
 
+### Dependencies for the `render` feature
+viuer = { version ="0.6.*", features = ["sixel"], optional = true }
+image = { version ="0.24.*", optional = true }
+graphviz-sys = { version ="0.1.3", optional = true }
+resvg = { version ="0.23.*", optional = true }
+usvg = { version ="0.23.*", optional = true }
+tiny-skia = { version ="0.6.*", optional = true }
+strum = { version ="0.24.*", features = ["derive"], optional = true }
+dot-writer = { version = "0.1.*", optional = true }
 
-tui = "0.18.0"
-crossterm = "0.23.2"
+
+
+[features]
+default = []
+serde = [
+  "dep:serde",
+  "dep:serde_json",
+  "partiql-parser/serde",
+  "partiql-source-map/serde",
+  "partiql-ast/serde",
+]
+visualize = [
+  "serde",
+  "dep:viuer",
+  "dep:image",
+  "dep:graphviz-sys",
+  "dep:resvg",
+  "dep:usvg",
+  "dep:tiny-skia",
+  "dep:strum",
+  "dep:dot-writer",
+]

partiql-cli/README.md

@@ -1,23 +1,51 @@
-# PartiQL Rust REPL
+# PartiQL Rust CLI
+PoC for a CLI & REPL. It should be considered experimental, subject to change, etc.
 
-PoC for a REPL. It should be considered experimental, subject to change, etc.
-
-In its current state, it largely exists to test parser interface & types from the perspective of an external application. 
+In its current state, it largely exists to test parser interface & types from the perspective of an external application.
 Probably the the mietter::Diagnostic stuff should be refactored and moved to the main parser crate.
 
-The REPL currently accepts no commands, assuming any/all input is a PartiQL query, which it will attempt to parse. Parse errors are pretty printed to the output.
+## CLI Commands
+
+- **`help`** : print the CLI's help message and supported commands
+- **`repl`** : launches the [REPL](##REPL)
+- **`ast -T<format> "<query>"`**: outputs a rendered version of the parsed AST  ([see Visualization](##Visualizations)):
+  - **`<format>`**:
+    - **`json`** : pretty-print to stdout in a json dump
+    - **`dot`** : pretty-print to stdout in [Graphviz][Graphviz] [dot][GvDot] format
+    - **`svg`** : print to stdout a [Graphviz][Graphviz] rendered svg xml document
+    - **`png`** : print to stdout a [Graphviz][Graphviz] rendered png bitmap
+    - **`display`** : display a [Graphviz][Graphviz] rendered png bitmap directly in supported terminals
+  - **`query`** : the PartiQL query text
+
+## REPL
+
+The REPL currently assumes most of the input line is a PartiQL query, which it will attempt to parse.
+- For an invalid query, errors are pretty printed to the output.
+- For a valid query,
+  - with no prefix, `Parse OK!` is printed to the output
+  - if prefixed by `\ast`, a rendered AST tree image is printed to the output ([see Visualization](##Visualizations))
 
 Features:
 - Syntax highlighting of query input
 - User-friendly error reporting
 - Readling/editing
 - `CTRL-D`/`CTRL-C` to quit.
 
+# Visualizations
+
+In order to use any of the [Graphviz][Graphviz]-based visualizations, you will need the graphviz libraries
+installed on your machine (e.g. `brew install graphviz` or similar).
+
 # TODO
 
+See [REPL-tagged issues](https://github.com/partiql/partiql-lang-rust/issues?q=is%3Aissue+is%3Aopen+%5BREPL%5D)
+
 - Use central location for syntax files rather than embedded in this crate
-- Add github issue link for Internal Compiler Errors
 - Better interaction model
   - commands
   - more robust editing
-  - etc.
+  - etc.
+
+
+[Graphviz]: https://graphviz.org/
+[GvDot]: https://graphviz.org/doc/info/lang.html

partiql-cli/src/args.rs

@@ -0,0 +1,38 @@
+use clap::{ArgEnum, Parser, Subcommand};
+
+#[derive(Parser)]
+#[clap(author, version, about, long_about = None)]
+pub struct Args {
+    #[clap(subcommand)]
+    pub command: Commands,
+}
+
+#[derive(Subcommand)]
+pub enum Commands {
+    #[cfg(feature = "visualize")]
+    /// Dump the AST for a query
+    Ast {
+        #[clap(short = 'T', long = "format", value_enum)]
+        format: Format,
+
+        /// Query to parse
+        #[clap(value_parser)]
+        query: String,
+    },
+    /// interactive REPL (Read Eval Print Loop) shell
+    Repl,
+}
+
+#[derive(ArgEnum, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Format {
+    /// JSON
+    Json,
+    /// Graphviz dot
+    Dot,
+    /// Graphviz svg output
+    Svg,
+    /// Graphviz svg rendered to png
+    Png,
+    /// Display rendered output
+    Display,
+}

partiql-cli/src/error.rs

@@ -0,0 +1,103 @@
+use miette::{Diagnostic, LabeledSpan, SourceCode};
+use partiql_parser::{ParseError, ParserError};
+use partiql_source_map::location::{BytePosition, Location};
+
+use thiserror::Error;
+
+#[derive(Debug, Error, Diagnostic)]
+#[error("Error for query `{query}`")]
+pub struct CLIErrors {
+    query: String,
+    #[related]
+    related: Vec<CLIError>,
+}
+
+impl CLIErrors {
+    pub fn from_parser_error(err: ParserError) -> Self {
+        let query = err.text.to_string();
+
+        let related = err
+            .errors
+            .into_iter()
+            .map(|e| CLIError::from_parse_error(e, &query))
+            .collect();
+        CLIErrors { query, related }
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum CLIError {
+    #[error("PartiQL syntax error:")]
+    SyntaxError {
+        src: String,
+        msg: String,
+        loc: Location<BytePosition>,
+    },
+    #[error("Internal Compiler Error - please report this (https://github.com/partiql/partiql-lang-rust/issues).")]
+    InternalCompilerError { src: String },
+}
+
+impl Diagnostic for CLIError {
+    fn source_code(&self) -> Option<&dyn SourceCode> {
+        match self {
+            CLIError::SyntaxError { src, .. } => Some(src),
+            CLIError::InternalCompilerError { src, .. } => Some(src),
+        }
+    }
+
+    fn labels(&self) -> Option<Box<dyn Iterator<Item = LabeledSpan> + '_>> {
+        match self {
+            CLIError::SyntaxError { msg, loc, .. } => {
+                Some(Box::new(std::iter::once(LabeledSpan::new(
+                    Some(msg.to_string()),
+                    loc.start.0 .0 as usize,
+                    loc.end.0 .0 as usize - loc.start.0 .0 as usize,
+                ))))
+            }
+            CLIError::InternalCompilerError { .. } => None,
+        }
+    }
+}
+
+impl CLIError {
+    pub fn from_parse_error(err: ParseError, source: &str) -> Self {
+        match err {
+            ParseError::SyntaxError(partiql_source_map::location::Located { inner, location }) => {
+                CLIError::SyntaxError {
+                    src: source.to_string(),
+                    msg: format!("Syntax error `{}`", inner),
+                    loc: location,
+                }
+            }
+            ParseError::UnexpectedToken(partiql_source_map::location::Located {
+                inner,
+                location,
+            }) => CLIError::SyntaxError {
+                src: source.to_string(),
+                msg: format!("Unexpected token `{}`", inner.token),
+                loc: location,
+            },
+            ParseError::LexicalError(partiql_source_map::location::Located { inner, location }) => {
+                CLIError::SyntaxError {
+                    src: source.to_string(),
+                    msg: format!("Lexical error `{}`", inner),
+                    loc: location,
+                }
+            }
+            ParseError::Unknown(location) => CLIError::SyntaxError {
+                src: source.to_string(),
+                msg: "Unknown parser error".to_string(),
+                loc: Location {
+                    start: location,
+                    end: location,
+                },
+            },
+            ParseError::IllegalState(_location) => CLIError::InternalCompilerError {
+                src: source.to_string(),
+            },
+            _ => {
+                todo!("Not yet handled {:?}", err);
+            }
+        }
+    }
+}

partiql-cli/src/bin/ion.sublime-syntax renamed to partiql-cli/src/ion.sublime-syntax

@@ -0,0 +1,7 @@
+pub mod args;
+
+pub mod error;
+pub mod repl;
+
+#[cfg(feature = "visualize")]
+pub mod visualize;

partiql-cli/src/lib.rs

@@ -0,0 +1,44 @@
+#![deny(rustdoc::broken_intra_doc_links)]
+
+use clap::Parser;
+use partiql_cli::error::CLIErrors;
+use partiql_cli::{args, repl};
+
+use partiql_parser::Parsed;
+
+#[allow(dead_code)]
+fn parse(query: &str) -> Result<Parsed, CLIErrors> {
+    partiql_parser::Parser::default()
+        .parse(query)
+        .map_err(CLIErrors::from_parser_error)
+}
+
+fn main() -> miette::Result<()> {
+    let args = args::Args::parse();
+
+    match &args.command {
+        args::Commands::Repl => repl::repl(),
+
+        #[cfg(feature = "visualize")]
+        args::Commands::Ast { format, query } => {
+            use partiql_cli::args::Format;
+            use partiql_cli::visualize::render::{display, to_dot, to_json, to_png, to_svg};
+            use std::io::Write;
+
+            let parsed = parse(&query)?;
+            match format {
+                Format::Json => println!("{}", to_json(&parsed.ast)),
+                Format::Dot => println!("{}", to_dot(&parsed.ast)),
+                Format::Svg => println!("{}", to_svg(&parsed.ast)),
+                Format::Png => {
+                    std::io::stdout()
+                        .write(&to_png(&parsed.ast))
+                        .expect("png write");
+                }
+                Format::Display => display(&parsed.ast),
+            }
+
+            Ok(())
+        }
+    }
+}