Fix an infinite loop in wasm-smith (#1834)

* Fix an infinite loop in wasm-smith

This fixes a possible infinite loop in wasm-smith when the gc proposal
is enabled. When the GC proposal is enabled and a minimum number of
types were specified it could get wasm-smith stuck in an infinite loop
where it would either always create an empty rec group or it would
always try to clone an empty rec group. The fix here is to thread a
boolean which toggles whether an empty rec group is allowed based on
whether we're generating the minimum number of types or any extra types.

This is a relatively old bug so I'm not sure why this hasn't been
discovered prior to this. Local fuzzing in Wasmtime found this quite
quickly, so I may have just gotten unlucky. We also tend to not look at
timeouts on OSS-Fuzz that closely as well, so perhaps this is a reminder
to actually do that.

* Review feedback

Author: alexcrichton

crates/wasm-smith/src/core.rs

@@ -174,6 +174,12 @@ pub(crate) enum DuplicateImportsBehavior {
     Disallowed,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum AllowEmptyRecGroup {
+    Yes,
+    No,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum MaxTypeLimit {
     ModuleTypes,
@@ -549,14 +555,14 @@ impl Module {
     fn arbitrary_types(&mut self, u: &mut Unstructured) -> Result<()> {
         assert!(self.config.min_types <= self.config.max_types);
         while self.types.len() < self.config.min_types {
-            self.arbitrary_rec_group(u)?;
+            self.arbitrary_rec_group(u, AllowEmptyRecGroup::No)?;
         }
         while self.types.len() < self.config.max_types {
             let keep_going = u.arbitrary().unwrap_or(false);
             if !keep_going {
                 break;
             }
-            self.arbitrary_rec_group(u)?;
+            self.arbitrary_rec_group(u, AllowEmptyRecGroup::Yes)?;
         }
         Ok(())
     }
@@ -586,20 +592,28 @@ impl Module {
         index
     }
 
-    fn arbitrary_rec_group(&mut self, u: &mut Unstructured) -> Result<()> {
+    fn arbitrary_rec_group(
+        &mut self,
+        u: &mut Unstructured,
+        kind: AllowEmptyRecGroup,
+    ) -> Result<()> {
         let rec_group_start = self.types.len();
 
         assert!(matches!(self.max_type_limit, MaxTypeLimit::ModuleTypes));
 
         if self.config.gc_enabled {
             // With small probability, clone an existing rec group.
-            if self.clonable_rec_groups().next().is_some() && u.ratio(1, u8::MAX)? {
-                return self.clone_rec_group(u);
+            if self.clonable_rec_groups(kind).next().is_some() && u.ratio(1, u8::MAX)? {
+                return self.clone_rec_group(u, kind);
             }
 
             // Otherwise, create a new rec group with multiple types inside.
             let max_rec_group_size = self.config.max_types - self.types.len();
-            let rec_group_size = u.int_in_range(0..=max_rec_group_size)?;
+            let min_rec_group_size = match kind {
+                AllowEmptyRecGroup::Yes => 0,
+                AllowEmptyRecGroup::No => 1,
+            };
+            let rec_group_size = u.int_in_range(min_rec_group_size..=max_rec_group_size)?;
             let type_ref_limit = u32::try_from(self.types.len() + rec_group_size).unwrap();
             self.max_type_limit = MaxTypeLimit::Num(type_ref_limit);
             for _ in 0..rec_group_size {
@@ -621,22 +635,35 @@ impl Module {
 
     /// Returns an iterator of rec groups that we could currently clone while
     /// still staying within the max types limit.
-    fn clonable_rec_groups(&self) -> impl Iterator<Item = Range<usize>> + '_ {
+    fn clonable_rec_groups(
+        &self,
+        kind: AllowEmptyRecGroup,
+    ) -> impl Iterator<Item = Range<usize>> + '_ {
         self.rec_groups
             .iter()
-            .filter(|r| r.end - r.start <= self.config.max_types.saturating_sub(self.types.len()))
+            .filter(move |r| {
+                match kind {
+                    AllowEmptyRecGroup::Yes => {}
+                    AllowEmptyRecGroup::No => {
+                        if r.is_empty() {
+                            return false;
+                        }
+                    }
+                }
+                r.end - r.start <= self.config.max_types.saturating_sub(self.types.len())
+            })
             .cloned()
     }
 
-    fn clone_rec_group(&mut self, u: &mut Unstructured) -> Result<()> {
+    fn clone_rec_group(&mut self, u: &mut Unstructured, kind: AllowEmptyRecGroup) -> Result<()> {
         // NB: this does *not* guarantee that the cloned rec group will
         // canonicalize the same as the original rec group and be
         // deduplicated. That would reqiure a second pass over the cloned types
         // to rewrite references within the original rec group to be references
         // into the new rec group. That might make sense to do one day, but for
         // now we don't do it. That also means that we can't mark the new types
         // as "subtypes" of the old types and vice versa.
-        let candidates: Vec<_> = self.clonable_rec_groups().collect();
+        let candidates: Vec<_> = self.clonable_rec_groups(kind).collect();
         let group = u.choose(&candidates)?.clone();
         let new_rec_group_start = self.types.len();
         for index in group {