Write files for large test expectations; Move data to _test_data (#217)

Author: jpschorr

partiql-conformance-test-generator/src/generator.rs

@@ -47,7 +47,7 @@ pub enum TestTree {
 #[derive(Debug)]
 pub enum Node {
     Test(TestNode),
-    Env(EnvNode),
+    Value(TestValueNode),
 }
 
 #[derive(Debug)]
@@ -56,8 +56,8 @@ pub struct TestNode {
 }
 
 #[derive(Debug)]
-pub struct EnvNode {
-    pub env: String,
+pub struct TestValueNode {
+    pub value: String,
 }
 
 #[derive(Debug, Default)]
@@ -98,6 +98,10 @@ pub struct Generator {
     seen_fns: Vec<HashSet<String>>,
 }
 
+const TEST_DATA_DIR: &'static str = "_test_data";
+const ENV_INLINE_LOWER_BOUND_LINE_COUNT: usize = 10;
+const EXPECTED_INLINE_LOWER_BOUND_LINE_COUNT: usize = 25;
+
 impl Generator {
     pub fn new(config: GeneratorConfig) -> Generator {
         Self {
@@ -251,6 +255,29 @@ impl Generator {
     }
 
     fn gen_envs(&mut self, scope: &mut Scope, envs: &Environments) {
+        let envs = struct_to_string(&envs.envs);
+
+        if envs.lines().count() < ENV_INLINE_LOWER_BOUND_LINE_COUNT {
+            self.gen_envs_inline(scope, envs);
+        } else {
+            self.gen_envs_external(scope, envs);
+        }
+    }
+
+    fn gen_envs_inline(&mut self, scope: &mut Scope, envs: String) {
+        scope.raw(
+            quote! {
+                const ENV_ION_TEXT : &'static str = #envs;
+                fn environment() -> Option<TestValue<'static>> {
+                    Some(ENV_ION_TEXT.into())
+                }
+            }
+            .to_string()
+            .replace("\\n", "\n"),
+        );
+    }
+
+    fn gen_envs_external(&mut self, scope: &mut Scope, envs: String) {
         let env_file = self
             .curr_mod_path
             .iter()
@@ -259,9 +286,10 @@ impl Generator {
             .join("___")
             + ".env.ion";
 
+        let data_file = format!("{}/{}", TEST_DATA_DIR, env_file);
         scope.raw(
             quote! {
-                const ENV_ION_TEXT : &'static str = include_str!(#env_file);
+                const ENV_ION_TEXT : &'static str = include_str!(#data_file);
                 fn environment() -> Option<TestValue<'static>> {
                     Some(ENV_ION_TEXT.into())
                 }
@@ -270,15 +298,18 @@ impl Generator {
             .replace("\\n", "\n"),
         );
 
+        let td_dir = TEST_DATA_DIR.to_string();
         let env_path: Vec<_> = self
             .curr_path
             .iter()
+            .chain(std::iter::once(&td_dir))
             .chain(std::iter::once(&env_file))
             .collect();
-        let env = struct_to_string(&envs.envs);
 
-        self.result
-            .insert(env_path.as_slice(), Node::Env(EnvNode { env }));
+        self.result.insert(
+            env_path.as_slice(),
+            Node::Value(TestValueNode { value: envs }),
+        );
     }
 
     fn gen_equivs(&mut self, _scope: &mut Scope, equivs: &EquivalenceClass) {
@@ -378,6 +409,36 @@ impl Generator {
                     test_fn.line("\n//**** evaluation success test case(s) ****//");
 
                     let expected = elt_to_string(output);
+                    let expected =
+                        if expected.lines().count() > EXPECTED_INLINE_LOWER_BOUND_LINE_COUNT {
+                            let expected_file = self
+                                .curr_mod_path
+                                .iter()
+                                .map(|s| s.escape_path())
+                                .chain(std::iter::once(test_case.name.escape_path()))
+                                .collect::<Vec<_>>()
+                                .join("___")
+                                + ".expected.ion";
+
+                            let td_dir = TEST_DATA_DIR.to_string();
+                            let expected_path: Vec<_> = self
+                                .curr_path
+                                .iter()
+                                .chain(std::iter::once(&td_dir))
+                                .chain(std::iter::once(&expected_file))
+                                .collect();
+
+                            self.result.insert(
+                                expected_path.as_slice(),
+                                Node::Value(TestValueNode { value: expected }),
+                            );
+
+                            let data_file = format!("{}/{}", TEST_DATA_DIR, expected_file);
+                            quote! {include_str!(#data_file)}
+                        } else {
+                            quote! {#expected}
+                        };
+
                     let modes: Vec<_> = eval_mode
                         .into_iter()
                         .map(|mode| match mode {

partiql-conformance-test-generator/src/reader.rs

@@ -199,7 +199,6 @@ impl TryFrom<&Struct> for TestCase {
             }
         };
 
-        //let statement = expect_str!(value.get("statement"), "TestCase statement").into();
         let env = if let Some(v) = value.get("env") {
             Some(expect_struct!(Some(v), "TestCase envs").clone())
         } else {

partiql-conformance-test-generator/src/writer.rs

@@ -4,7 +4,7 @@ use crate::util::Escaper;
 use codegen::Scope;
 use miette::IntoDiagnostic;
 
-use std::fs::File;
+use std::fs::{DirBuilder, File};
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
@@ -14,6 +14,8 @@ const FILE_HEADER: &str = "\
 // ********************************************************************************************** //
 
 #[allow(unused_imports)]
+use super::*;
+
 ";
 
 #[derive(Debug)]
@@ -24,50 +26,55 @@ impl Writer {
         Self {}
     }
 
-    pub fn write(&self, path: impl AsRef<Path>, root: NamespaceNode) -> miette::Result<()> {
+    pub fn write(&self, path: impl AsRef<Path>, mut root: NamespaceNode) -> miette::Result<()> {
         let path: PathBuf = path
             .as_ref()
             .components()
             .map(|c| c.as_os_str().to_string_lossy().to_string().escape_path())
             .collect();
         std::fs::create_dir_all(&path).into_diagnostic()?;
 
-        write_module(&path, root)?;
+        write_module(&path, &mut root)?;
 
         Ok(())
     }
 }
 
-fn write_module(path: impl AsRef<Path>, module: NamespaceNode) -> miette::Result<()> {
-    let sub_mods = module.children.iter().filter_map(|(name, tree)| {
-        if let TestTree::Node(Node::Env(_)) = tree {
-            None
-        } else {
-            Some(name)
-        }
-    });
-    write_dir_mod(&path, sub_mods)?;
+fn write_module(path: impl AsRef<Path>, module: &mut NamespaceNode) -> miette::Result<bool> {
+    DirBuilder::new()
+        .recursive(true)
+        .create(&path)
+        .into_diagnostic()?;
+
+    let mut sub_mods = vec![];
 
-    for (name, child) in module.children {
+    for (name, child) in &mut module.children {
         let mut child_path: PathBuf = path.as_ref().into();
         child_path.push(&name);
-        match child {
-            TestTree::Node(Node::Test(mut s)) => write_scope(child_path, s.module.scope())?,
-            TestTree::Node(Node::Env(e)) => write_file(child_path, &e.env)?,
+        let is_sub_mod = match child {
+            TestTree::Node(Node::Test(s)) => write_scope(child_path, s.module.scope())?,
+            TestTree::Node(Node::Value(e)) => write_file(child_path, &e.value)?,
             TestTree::Namespace(m) => write_module(child_path, m)?,
+        };
+        if is_sub_mod {
+            sub_mods.push(name.clone());
         }
     }
-    Ok(())
+
+    if !sub_mods.is_empty() {
+        write_dir_mod(&path, sub_mods.iter())?;
+    }
+
+    Ok(!sub_mods.is_empty())
 }
 
 fn write_dir_mod<'a>(
     path: impl AsRef<Path>,
     sub_mods: impl Iterator<Item = &'a String>,
-) -> miette::Result<()> {
+) -> miette::Result<bool> {
     std::fs::create_dir_all(&path).into_diagnostic()?;
 
     let mut contents = FILE_HEADER.to_string();
-    contents.push_str("use super::*;");
     for sub_mod in sub_mods {
         contents.push_str(&format!("mod {};\n", sub_mod.replace(".rs", "")))
     }
@@ -76,16 +83,19 @@ fn write_dir_mod<'a>(
     File::create(file_path)
         .into_diagnostic()?
         .write_all(contents.as_bytes())
-        .into_diagnostic()
+        .into_diagnostic()?;
+    Ok(true)
 }
 
-fn write_scope(path: impl AsRef<Path>, scope: &Scope) -> miette::Result<()> {
+fn write_scope(path: impl AsRef<Path>, scope: &Scope) -> miette::Result<bool> {
     let contents = format!("{}{}", FILE_HEADER, scope.to_string());
-    write_file(path, &contents)
+    write_file(path, &contents)?;
+    Ok(true)
 }
 
-fn write_file(path: impl AsRef<Path>, contents: &str) -> miette::Result<()> {
+fn write_file(path: impl AsRef<Path>, contents: &str) -> miette::Result<bool> {
     let mut file = File::create(path).into_diagnostic()?;
     file.write_all(contents.as_bytes()).into_diagnostic()?;
-    file.sync_all().into_diagnostic()
+    file.sync_all().into_diagnostic()?;
+    Ok(false)
 }