Aliases

Cargo.toml

@@ -22,21 +22,21 @@ default = []
 self-update = ["self_update"]
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-tikv-jemallocator = "0.5.0"
+tikv-jemallocator = "0.6"
 
 [dependencies]
 serde_json = "1.0.33"
-itertools = "0.10.5"
+itertools = "0.14"
 nom = "7.1.1"
 nom_locate = "4.0.0"
 nom-supreme = "0.8.0"
-strsim = "0.10.0"
+strsim = "0.11"
 regex = "1.5.5"
-terminal_size = "0.2.1"
+terminal_size = "0.4"
 quantiles = "0.7.1"
 crossbeam-channel = "0.5.15"
-ordered-float = "3.3.0"
-thiserror = "1.0.37"
+ordered-float = "5"
+thiserror = "2"
 anyhow = "1"
 human-panic = "2"
 self_update = { version = "0.32.0", features = ["rustls"], default-features = false, optional = true }
@@ -46,19 +46,19 @@ im = "15.1.0"
 logfmt = "0.0.2"
 strfmt = "0.2.2"
 include_dir = "0.7.3"
-toml = "0.5.9"
+toml = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 chrono = "0.4"
-dtparse = "1.1"
+dtparse = "2"
 clap = { version = "4.0.18", features = ["derive"] }
 
 [dev-dependencies]
 assert_cmd = "2.0.5"
 cool_asserts = "2.0.3"
 expect-test = "1.1.0"
 predicates = "2.1.1"
-pulldown-cmark = "0.9.2"
-criterion = "0.4.0"
+pulldown-cmark = "0.13.0"
+criterion = "0.5"
 maplit = "1.0.1"
 test-generator = "0.3.0"
 [dev-dependencies.cargo-husky]

README.md

@@ -93,9 +93,25 @@ Sub-expressions _must_ be grouped in parenthesis. Only lines that match all filt
 ![filter.gif](/screen_shots/filter.gif)
 
 ### Aliases
-Starting with v0.12.0, angle grinder supports aliases, pre-built pipelines do simplify common tasks or formats. The only alias currently defined is `apache`, which parses apache logs. Adding more `aliases` is one of the easiest ways to [contribute](#contributing)!
+Starting with v0.12.0, angle grinder supports aliases, pre-built pipelines do simplify common tasks or formats.
+
+By default, angle-grinder will look in the `.agrind-aliases` directory in your current working directory and all parent directories.
+
+Alias files look like this:
+
+```toml
+keyword = "apache"
+template = """
+parse "* - * [*] \\"* * *\\" * *" as ip, name, timestamp, method, url, protocol, status, contentlength
+"""
+```
+
+Your operators are parsed, then expanded into the resulting pipeline. When invalid aliases are present, a warning will be displayed when running angle-grinder.
+
+Note that aliases are currently considered an experimental feature and precise behavior may change in the future.
 
 *Examples*:
+
 ```agrind
 * | apache | count by status
 ```

benches/e2e.rs

@@ -1,3 +1,4 @@
+use ag::alias::AliasCollection;
 use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer};
 use annotate_snippets::snippet::Snippet;
 use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
@@ -49,7 +50,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         },
     ];
     tests.into_iter().for_each(|test| {
-        let query_container = QueryContainer::new(test.query, Box::new(NopErrorReporter {}));
+        let query_container = QueryContainer::new_with_aliases(
+            test.query,
+            Box::new(NopErrorReporter {}),
+            AliasCollection::default(),
+        );
         let mut group = c.benchmark_group("e2e_query");
         let num_elems = BufReader::new(File::open(&test.file).unwrap())
             .lines()

src/alias.rs

@@ -5,6 +5,9 @@
 //! 2. Create a new test config inside `tests/structured_tests/aliases`.
 //! 3. Add the test config to the `test_aliases()` test.
 
+use std::borrow::Cow;
+use std::path::{Path, PathBuf};
+
 use lazy_static::lazy_static;
 
 use crate::errors::{QueryContainer, TermErrorReporter};
@@ -18,40 +21,165 @@ lazy_static! {
     pub static ref LOADED_ALIASES: Vec<AliasPipeline> = ALIASES_DIR
         .files()
         .map(|file| {
-            let config: AliasConfig =
-                toml::from_str(file.contents_utf8().expect("load string")).expect("toml valid");
-            let reporter = Box::new(TermErrorReporter {});
-            let qc = QueryContainer::new(config.template, reporter);
-            let pipeline = pipeline_template(&qc).expect("valid alias");
-
-            AliasPipeline {
-                keyword: config.keyword,
-                pipeline,
-            }
+            parse_alias(
+                file.contents_utf8().expect("invalid utf-8"),
+                file.path(),
+                &[],
+            )
+            .expect("invalid toml")
         })
         .collect();
     pub static ref LOADED_KEYWORDS: Vec<&'static str> =
         LOADED_ALIASES.iter().map(|a| a.keyword.as_str()).collect();
 }
 
+#[derive(Debug)]
+pub struct InvalidAliasError {
+    pub path: PathBuf,
+    pub cause: anyhow::Error,
+    pub keyword: Option<String>,
+    pub contents: Option<String>,
+}
+
+fn parse_alias(
+    contents: &str,
+    path: &Path,
+    aliases: &[AliasPipeline],
+) -> Result<AliasPipeline, InvalidAliasError> {
+    let config: AliasConfig = toml::from_str(contents).map_err(|err| InvalidAliasError {
+        path: path.to_owned(),
+        cause: err.into(),
+        keyword: None,
+        contents: Some(contents.to_string()),
+    })?;
+    let reporter = Box::new(TermErrorReporter {});
+    let aliases = AliasCollection {
+        aliases: Cow::Borrowed(aliases),
+    };
+    let qc = QueryContainer::new_with_aliases(config.template, reporter, aliases);
+    let keyword = config.keyword;
+    let pipeline = pipeline_template(&qc).map_err(|err| InvalidAliasError {
+        path: path.to_owned(),
+        cause: err.into(),
+        keyword: Some(keyword.clone()),
+        contents: Some(contents.to_string()),
+    })?;
+
+    Ok(AliasPipeline { keyword, pipeline })
+}
+
 #[derive(Debug, Deserialize, PartialEq, Eq)]
 pub struct AliasConfig {
     keyword: String,
     template: String,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct AliasPipeline {
     keyword: String,
     pipeline: Vec<Operator>,
 }
 
-impl AliasPipeline {
-    pub fn matching_string(s: &str) -> Result<&'static AliasPipeline, ()> {
-        LOADED_ALIASES
+#[derive(Default)]
+pub struct AliasCollection<'a> {
+    aliases: Cow<'a, [AliasPipeline]>,
+}
+
+#[derive(Default)]
+struct AliasAccum {
+    valid_aliases: Vec<AliasPipeline>,
+    invalid_aliases: Vec<InvalidAliasError>,
+}
+
+impl AliasCollection<'_> {
+    pub fn get_alias(&self, name: &str) -> Option<&AliasPipeline> {
+        self.aliases
             .iter()
-            .find(|alias| alias.keyword == s)
-            .ok_or(())
+            .find(|alias| alias.keyword == name)
+            .or_else(|| AliasPipeline::matching_string(name))
+    }
+
+    pub fn valid_aliases(&self) -> impl Iterator<Item = &str> {
+        self.aliases.iter().map(|a| a.keyword.as_str())
+    }
+}
+
+impl AliasCollection<'static> {
+    pub fn load_aliases_ancestors(
+        path: Option<PathBuf>,
+    ) -> anyhow::Result<(AliasCollection<'static>, Vec<InvalidAliasError>)> {
+        let path = match path {
+            Some(path) => path,
+            None => std::env::current_dir()?,
+        };
+        let (valid, invalid) = find_all_aliases(path)?;
+        Ok((
+            AliasCollection {
+                aliases: Cow::Owned(valid),
+            },
+            invalid,
+        ))
+    }
+
+    pub fn load_aliases_from_dir(
+        path: &Path,
+    ) -> anyhow::Result<(AliasCollection<'static>, Vec<InvalidAliasError>)> {
+        let mut aliases = AliasAccum::default();
+        aliases_from_dir(path, &mut aliases)?;
+        Ok((
+            AliasCollection {
+                aliases: Cow::Owned(aliases.valid_aliases),
+            },
+            aliases.invalid_aliases,
+        ))
+    }
+}
+
+fn find_local_aliases(dir: &Path, aliases: &mut AliasAccum) -> anyhow::Result<()> {
+    if let Some(alias_dir) = dir.read_dir()?.find_map(|file| match file {
+        Ok(entry) if entry.file_name() == ".agrind-aliases" => Some(entry),
+        _else => None,
+    }) {
+        aliases_from_dir(&alias_dir.path(), aliases)?;
+    }
+    Ok(())
+}
+
+fn aliases_from_dir(dir: &Path, pipelines: &mut AliasAccum) -> anyhow::Result<()> {
+    for entry in dir.read_dir()? {
+        let entry = entry?;
+        let path = entry.path();
+        let contents = match std::fs::read_to_string(&path) {
+            Ok(s) => s,
+            Err(e) => {
+                pipelines.invalid_aliases.push(InvalidAliasError {
+                    keyword: None,
+                    path,
+                    cause: e.into(),
+                    contents: None,
+                });
+                continue;
+            }
+        };
+        match parse_alias(&contents, &path, &pipelines.valid_aliases) {
+            Ok(alias) => pipelines.valid_aliases.push(alias),
+            Err(e) => pipelines.invalid_aliases.push(e),
+        }
+    }
+    Ok(())
+}
+
+fn find_all_aliases(path: PathBuf) -> anyhow::Result<(Vec<AliasPipeline>, Vec<InvalidAliasError>)> {
+    let mut accum = AliasAccum::default();
+    for path in path.ancestors() {
+        find_local_aliases(path, &mut accum)?;
+    }
+    Ok((accum.valid_aliases, accum.invalid_aliases))
+}
+
+impl AliasPipeline {
+    pub fn matching_string(s: &str) -> Option<&'static AliasPipeline> {
+        LOADED_ALIASES.iter().find(|alias| alias.keyword == s)
     }
 
     /// Render the alias as a string that should parse into a valid operator.

src/bin/agrind.rs

@@ -1,4 +1,7 @@
-use ag::pipeline::{OutputMode, Pipeline, QueryContainer, TermErrorReporter};
+use ag::alias::AliasCollection;
+use ag::pipeline::{ErrorReporter, OutputMode, Pipeline, QueryContainer, TermErrorReporter};
+use annotate_snippets::display_list::FormatOptions;
+use annotate_snippets::snippet::{Annotation, AnnotationType, Slice, Snippet};
 use human_panic::setup_panic;
 
 use clap::Parser;
@@ -7,6 +10,7 @@ use self_update;
 use std::fs::File;
 use std::io;
 use std::io::{stdout, BufReader};
+use std::path::PathBuf;
 use thiserror::Error;
 
 #[cfg(not(target_env = "msvc"))]
@@ -49,6 +53,16 @@ struct Cli {
                      - `legacy` The original output format, auto aligning [k=v]"
     )]
     output: Option<String>,
+
+    #[arg(
+        long = "alias-dir",
+        short = 'a',
+        long_help = "Specifies an alternative directory to use for aliases. Defaults to `.agrind-aliases` in all parent directories."
+    )]
+    alias_dir: Option<PathBuf>,
+
+    #[arg(long = "no-alias", long_help = "Disables aliases")]
+    no_alias: bool,
 }
 
 #[derive(Debug, Error)]
@@ -64,6 +78,9 @@ pub enum InvalidArgs {
 
     #[error("Can't supply a format string and an output mode")]
     CantSupplyBoth,
+
+    #[error("Can't disable aliases and also set a directory")]
+    CantDisableAndOverride,
 }
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -73,11 +90,36 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     if args.update {
         return update();
     }
-    let query = QueryContainer::new(
+    let (aliases, errors) = match (args.alias_dir, args.no_alias) {
+        (Some(dir), false) => AliasCollection::load_aliases_from_dir(&dir)?,
+        (None, false) => AliasCollection::load_aliases_ancestors(None)?,
+        (Some(_), true) => return Err(InvalidArgs::CantDisableAndOverride.into()),
+        (None, true) => (AliasCollection::default(), vec![]),
+    };
+    let error_reporter = Box::new(TermErrorReporter {});
+    for error in errors {
+        error_reporter.handle_error(Snippet {
+            title: Some(Annotation {
+                id: None,
+                label: Some(&format!("invalid alias: {}", error.cause)),
+                annotation_type: AnnotationType::Warning,
+            }),
+            footer: vec![],
+            slices: vec![Slice {
+                source: "",
+                line_start: 0,
+                origin: Some(error.path.to_str().unwrap()),
+                annotations: vec![],
+                fold: true,
+            }],
+            opt: FormatOptions::default(),
+        });
+    }
+    let query = QueryContainer::new_with_aliases(
         args.query.ok_or(InvalidArgs::MissingQuery)?,
-        Box::new(TermErrorReporter {}),
+        error_reporter,
+        aliases,
     );
-    //args.verbosity.setup_env_logger("agrind")?;
     let output_mode = match (args.output, args.format) {
         (Some(_output), Some(_format)) => Err(CantSupplyBoth),
         (Some(output), None) => parse_output(&output),

src/data.rs

@@ -269,17 +269,17 @@ impl Display for ValueDisplay<'_> {
             Value::Duration(ref d) => {
                 let mut remaining: Duration = *d;
                 let weeks = remaining.num_seconds() / Duration::weeks(1).num_seconds();
-                remaining = remaining - Duration::weeks(weeks);
+                remaining -= Duration::weeks(weeks);
                 let days = remaining.num_seconds() / Duration::days(1).num_seconds();
-                remaining = remaining - Duration::days(days);
+                remaining -= Duration::days(days);
                 let hours = remaining.num_seconds() / Duration::hours(1).num_seconds();
-                remaining = remaining - Duration::hours(hours);
+                remaining -= Duration::hours(hours);
                 let mins = remaining.num_seconds() / Duration::minutes(1).num_seconds();
-                remaining = remaining - Duration::minutes(mins);
+                remaining -= Duration::minutes(mins);
                 let secs = remaining.num_seconds() / Duration::seconds(1).num_seconds();
-                remaining = remaining - Duration::seconds(secs);
+                remaining -= Duration::seconds(secs);
                 let msecs = remaining.num_milliseconds();
-                remaining = remaining - Duration::milliseconds(msecs);
+                remaining -= Duration::milliseconds(msecs);
                 let usecs = remaining.num_microseconds().unwrap_or(0);
 
                 let pairs = &[