Deletions

Author: langston-barrett

Cargo.lock

@@ -1,5 +1,8 @@
 use anyhow::Result;
 
 fn main() -> Result<()> {
-    tree_splicer::cli::main(tree_sitter_javascript::language())
+    tree_splicer::cli::main(
+        tree_sitter_javascript::language(),
+        tree_sitter_javascript::NODE_TYPES,
+    )
 }

crates/tree-splicer-javascript/src/main.rs

@@ -1,5 +1,5 @@
 use anyhow::Result;
 
 fn main() -> Result<()> {
-    tree_splicer::cli::main(tree_sitter_rust::language())
+    tree_splicer::cli::main(tree_sitter_rust::language(), tree_sitter_rust::NODE_TYPES)
 }

crates/tree-splicer-rust/src/main.rs

@@ -1,5 +1,8 @@
 use anyhow::Result;
 
 fn main() -> Result<()> {
-    tree_splicer::cli::main(tree_sitter_typescript::language_typescript())
+    tree_splicer::cli::main(
+        tree_sitter_typescript::language_typescript(),
+        tree_sitter_typescript::TYPESCRIPT_NODE_TYPES,
+    )
 }

crates/tree-splicer-typescript/src/main.rs

@@ -18,6 +18,8 @@ log = "0.4"
 nu-ansi-term = { version = "0.46", optional = true }
 num_cpus = { version = "1", optional = true }
 rand = "0.8"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["json"], optional = true }
 tree-sitter = "0.20"

crates/tree-splicer/Cargo.toml

@@ -65,6 +65,10 @@ pub struct Args {
     #[arg(short, long, default_value_t = 5)]
     pub chaos: u8,
 
+    /// Percent of deletion mutations - the rest are splices
+    #[arg(short, long, default_value_t = 5)]
+    pub deletions: u8,
+
     /// Behavior on parse errors
     #[arg(long, default_value_t = OnParseError::Warn, value_name = "CHOICE")]
     on_parse_error: OnParseError,
@@ -138,7 +142,7 @@ fn init_tracing(args: &Args) {
     builder.event_format(formatter::TerseFormatter).init();
 }
 
-pub fn main(language: tree_sitter::Language) -> Result<()> {
+pub fn main(language: tree_sitter::Language, node_types_json_str: &'static str) -> Result<()> {
     let args = Args::parse();
 
     init_tracing(&args);
@@ -160,15 +164,19 @@ pub fn main(language: tree_sitter::Language) -> Result<()> {
         }
     }
 
+    let node_types = crate::node_types::NodeTypes::new(node_types_json_str)?;
     let config = Config {
+        chaos: args.chaos,
+        deletions: args.deletions,
         language,
         // intra_splices: 10,
         inter_splices: args.mutations,
+        node_types,
         seed: args.seed,
         tests: args.tests,
     };
     std::fs::create_dir_all(&args.output).context("Couldn't create output directory")?;
-    for (i, out) in splice::splice(config, &files, args.chaos).enumerate() {
+    for (i, out) in splice::splice(config, &files).enumerate() {
         std::fs::write(args.output.join(i.to_string()), out)
             .context("Couldn't save generated test case")?;
     }

crates/tree-splicer/src/cli.rs

@@ -1,3 +1,4 @@
 #[cfg(feature = "cli")]
 pub mod cli;
+pub mod node_types;
 pub mod splice;

crates/tree-splicer/src/lib.rs

@@ -0,0 +1,156 @@
+//! Read tree-sitter's `node-types.json`.
+//
+// Copied in part from [treeedbgen] and treereduce.
+//
+// [treeedbgen]: https://github.com/langston-barrett/treeedb/blob/1a2fae3509c76cd5a8e1004f808ea800d49d1a19/treeedbgen/src/lib.rs
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+/// node-types.json
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Debug)]
+struct Node {
+    #[serde(rename(deserialize = "type", serialize = "type"))]
+    ty: String,
+    named: bool,
+    #[serde(default)] // empty
+    children: Children,
+    #[serde(default)] // empty
+    fields: HashMap<String, Field>,
+    #[serde(default)] // empty
+    subtypes: Vec<Subtype>,
+}
+
+#[derive(Default, Clone, Eq, PartialEq, Serialize, Deserialize, Debug)]
+struct Children {
+    multiple: bool,
+    required: bool,
+    types: Vec<Subtype>,
+}
+
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Debug)]
+struct Field {
+    multiple: bool,
+    required: bool,
+    types: Vec<Subtype>,
+}
+
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Debug)]
+struct Subtype {
+    #[serde(rename(deserialize = "type", serialize = "type"))]
+    ty: String,
+    named: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct FieldInfo {
+    parent_ty: String,
+    multiple: bool,
+    required: bool,
+}
+
+#[derive(Clone, Debug)]
+pub struct NodeTypes {
+    children: HashMap<String, Children>,
+    subtypes: HashMap<String, Vec<String>>,
+    reverse_fields: HashMap<String, Vec<FieldInfo>>,
+}
+
+fn subtypes(name: &str, nodes: &Vec<Node>) -> Vec<String> {
+    let mut r = vec![name.to_string()];
+    for n in nodes {
+        if n.ty == name {
+            for subty in &n.subtypes {
+                r.push(subty.ty.clone());
+                r.extend(subtypes(&subty.ty, nodes));
+            }
+        }
+    }
+    r
+}
+
+impl NodeTypes {
+    pub fn new(node_types_json_str: &str) -> Result<Self, serde_json::Error> {
+        let nodes: Vec<Node> = serde_json::from_str(node_types_json_str)?;
+        let subtypes: HashMap<_, _> = nodes
+            .iter()
+            .map(|n| (n.ty.clone(), subtypes(&n.ty, &nodes)))
+            .collect();
+        let mut reverse_fields = HashMap::new();
+
+        // For each type of node...
+        for node in &nodes {
+            // Loop through it's fields...
+            for (_field_name, field) in node.fields.iter() {
+                // And save the name of all types that the field could be.
+                for subtype in &field.types {
+                    for subsubty in subtypes.get(&subtype.ty).unwrap_or(&Vec::new()) {
+                        let entry = reverse_fields.entry(subsubty.clone());
+                        entry
+                            .and_modify(|v: &mut Vec<FieldInfo>| {
+                                v.push(FieldInfo {
+                                    parent_ty: node.ty.clone(),
+                                    multiple: field.multiple,
+                                    required: field.required,
+                                });
+                            })
+                            .or_insert_with(|| {
+                                vec![FieldInfo {
+                                    parent_ty: node.ty.clone(),
+                                    multiple: field.multiple,
+                                    required: field.required,
+                                }]
+                            });
+                    }
+                }
+            }
+        }
+        Ok(NodeTypes {
+            children: nodes
+                .iter()
+                .map(|n| (n.ty.clone(), n.children.clone()))
+                .collect(),
+            subtypes,
+            reverse_fields,
+        })
+    }
+
+    /// Defaults to `true` if the real answer can't be determined.
+    fn optional(&self, node_kind: &str, parent_kind: &str) -> bool {
+        if let Some(flds) = self.reverse_fields.get(node_kind) {
+            for fi in flds {
+                if parent_kind == fi.parent_ty && (!fi.multiple || fi.required) {
+                    return false;
+                }
+            }
+        }
+        true
+    }
+
+    /// Defaults to `true` if the real answer can't be determined.
+    pub fn optional_node(&self, node: &tree_sitter::Node) -> bool {
+        if let Some(p) = node.parent() {
+            self.optional(node.kind(), p.kind())
+        } else {
+            true
+        }
+    }
+
+    // TODO(#21): Also include fields, include multiple and not required
+    pub fn list_types(&self, node: &tree_sitter::Node) -> Vec<String> {
+        let mut kinds = Vec::new();
+        if let Some(children) = self.children.get(node.kind()) {
+            if children.multiple && !children.required {
+                for child in &children.types {
+                    kinds.push(child.ty.clone());
+                }
+            }
+        }
+        kinds
+    }
+
+    pub fn subtypes(&self, kind: &String) -> &[String] {
+        self.subtypes.get(kind).expect("Invalid node kind")
+    }
+}

crates/tree-splicer/src/node_types.rs

@@ -6,6 +6,8 @@ use tree_sitter::{Language, Node, Tree};
 
 use tree_sitter_edit::{Editor, NodeId};
 
+use crate::node_types::NodeTypes;
+
 #[derive(Debug, Default)]
 pub struct Edits<'a>(HashMap<usize, &'a [u8]>);
 
@@ -71,9 +73,12 @@ fn parse(language: Language, code: &str) -> tree_sitter::Tree {
 
 #[derive(Debug)]
 pub struct Config {
+    pub chaos: u8,
+    pub deletions: u8,
     pub language: Language,
     // pub intra_splices: usize,
     pub inter_splices: usize,
+    pub node_types: NodeTypes,
     pub seed: u64,
     pub tests: usize,
 }
@@ -82,9 +87,11 @@ struct Splicer<'a> {
     language: Language,
     branches: Branches<'a>,
     chaos: u8,
+    deletions: u8,
     kinds: Vec<&'static str>,
     // intra_splices: usize,
     inter_splices: usize,
+    node_types: NodeTypes,
     trees: Vec<(&'a [u8], &'a Tree)>,
     remaining: usize,
     rng: StdRng,
@@ -99,19 +106,16 @@ impl<'a> Splicer<'a> {
         self.pick_usize(v.len())
     }
 
-    fn pick_node<'b>(&mut self, tree: &'b Tree) -> Node<'b> {
-        let mut all_nodes = Vec::with_capacity(16); // min
+    fn all_nodes<'b>(&self, tree: &'b Tree) -> Vec<Node<'b>> {
+        let mut all = Vec::with_capacity(16); // min
         let root = tree.root_node();
         let mut cursor = tree.walk();
         let mut nodes: HashSet<_> = root.children(&mut cursor).collect();
-        if nodes.is_empty() {
-            return root;
-        }
         while !nodes.is_empty() {
             let mut next = HashSet::new();
             for node in nodes {
                 debug_assert!(!next.contains(&node));
-                all_nodes.push(node);
+                all.push(node);
                 let mut child_cursor = tree.walk();
                 for child in node.children(&mut child_cursor) {
                     debug_assert!(child.id() != node.id());
@@ -121,7 +125,31 @@ impl<'a> Splicer<'a> {
             }
             nodes = next;
         }
-        *all_nodes.get(self.pick_idx(&all_nodes)).unwrap()
+        all
+    }
+
+    fn pick_node<'b>(&mut self, tree: &'b Tree) -> Node<'b> {
+        let nodes = self.all_nodes(tree);
+        if nodes.is_empty() {
+            return tree.root_node();
+        }
+        *nodes.get(self.pick_idx(&nodes)).unwrap()
+    }
+
+    fn delete_node(&mut self, _text: &[u8], tree: &Tree) -> (usize, Vec<u8>) {
+        let chaotic = self.rng.gen_range(0..100) < self.chaos;
+        if chaotic {
+            return (self.pick_node(tree).id(), Vec::new());
+        }
+        let nodes = self.all_nodes(tree);
+        if nodes.iter().all(|n| !self.node_types.optional_node(n)) {
+            return (self.pick_node(tree).id(), Vec::new());
+        }
+        let mut node = nodes.get(self.pick_idx(&nodes)).unwrap();
+        while !self.node_types.optional_node(node) {
+            node = nodes.get(self.pick_idx(&nodes)).unwrap();
+        }
+        (node.id(), Vec::new())
     }
 
     fn splice_node(&mut self, text: &[u8], tree: &Tree) -> (usize, Vec<u8>) {
@@ -163,7 +191,11 @@ impl<'a> Splicer<'a> {
         let splices = self.rng.gen_range(0..self.inter_splices);
         let mut text = Vec::from(text0);
         for _ in 0..splices {
-            let (id, bytes) = self.splice_node(text.as_slice(), &tree);
+            let (id, bytes) = if self.rng.gen_range(0..100) < self.deletions {
+                self.delete_node(text.as_slice(), &tree)
+            } else {
+                self.splice_node(text.as_slice(), &tree)
+            };
             let id = NodeId { id };
             let bytes = bytes.to_vec();
             let mut result = Vec::with_capacity(text.len() / 4); // low guesstimate
@@ -200,7 +232,6 @@ impl<'a> Iterator for Splicer<'a> {
 pub fn splice<'a>(
     config: Config,
     files: &'a HashMap<String, (Vec<u8>, Tree)>,
-    chaos: u8,
 ) -> impl Iterator<Item = Vec<u8>> + 'a {
     let trees: Vec<_> = files
         .iter()
@@ -219,12 +250,14 @@ pub fn splice<'a>(
     let rng = rand::rngs::StdRng::seed_from_u64(config.seed);
     let kinds = branches.0.keys().copied().collect();
     Splicer {
+        chaos: config.chaos,
+        deletions: config.deletions,
         language: config.language,
         branches,
-        chaos,
         kinds,
         // intra_splices: config.intra_splices,
         inter_splices: config.inter_splices,
+        node_types: config.node_types,
         remaining: std::cmp::min(config.tests, possible),
         rng,
         trees,