[CHIA-2843] add function to check if a serialization uses canonical encoding of atoms (#580)

* add function to check if a CLVM serialization uses canonical encoding of atoms

* address clippy warning

Author: arvidn

fuzz/Cargo.toml

@@ -110,3 +110,15 @@ name = "tree-cache"
 path = "fuzz_targets/tree_cache.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "canonical-serialization"
+path = "fuzz_targets/canonical_serialization.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "canonical-serialization-br"
+path = "fuzz_targets/canonical_serialization_br.rs"
+test = false
+doc = false

fuzz/fuzz_targets/canonical_serialization.rs

@@ -0,0 +1,19 @@
+#![no_main]
+mod make_tree;
+
+use clvmr::serde::is_canonical_serialization;
+use clvmr::serde::node_to_bytes;
+use clvmr::Allocator;
+use libfuzzer_sys::fuzz_target;
+use make_tree::make_tree_limits;
+
+fuzz_target!(|data: &[u8]| {
+    let mut unstructured = arbitrary::Unstructured::new(data);
+    let mut a = Allocator::new();
+    let (tree, _) = make_tree_limits(&mut a, &mut unstructured, 1000, false);
+
+    let buffer = node_to_bytes(&a, tree).expect("internal error, failed to serialize");
+
+    // out serializer should always produce canonical serialization
+    assert!(is_canonical_serialization(&buffer));
+});

fuzz/fuzz_targets/canonical_serialization_br.rs

@@ -0,0 +1,19 @@
+#![no_main]
+mod make_tree;
+
+use clvmr::serde::is_canonical_serialization;
+use clvmr::serde::node_to_bytes_backrefs;
+use clvmr::Allocator;
+use libfuzzer_sys::fuzz_target;
+use make_tree::make_tree_limits;
+
+fuzz_target!(|data: &[u8]| {
+    let mut unstructured = arbitrary::Unstructured::new(data);
+    let mut a = Allocator::new();
+    let (tree, _) = make_tree_limits(&mut a, &mut unstructured, 1000, true);
+
+    let buffer = node_to_bytes_backrefs(&a, tree).expect("internal error, failed to serialize");
+
+    // out serializer should always produce canonical serialization
+    assert!(is_canonical_serialization(&buffer));
+});

src/reduction.rs

@@ -21,6 +21,6 @@ impl std::error::Error for EvalErr {}
 
 impl From<EvalErr> for io::Error {
     fn from(v: EvalErr) -> Self {
-        Self::new(io::ErrorKind::Other, v.1)
+        Self::other(v.1)
     }
 }

src/serde/mod.rs

@@ -36,6 +36,7 @@ pub use ser::{node_to_bytes, node_to_bytes_limit};
 pub use ser_br::{node_to_bytes_backrefs, node_to_bytes_backrefs_limit};
 pub use serialized_length::{serialized_length_atom, serialized_length_small_number};
 pub use tools::{
-    serialized_length_from_bytes, serialized_length_from_bytes_trusted, tree_hash_from_stream,
+    is_canonical_serialization, serialized_length_from_bytes, serialized_length_from_bytes_trusted,
+    tree_hash_from_stream,
 };
 pub use tree_cache::{TreeCache, TreeCacheCheckpoint};