[Custom Descriptors] Update Unsubtyping (#7664)

When we have types A and B with descriptors A.desc and B.desc, A <: B
implies A.desc <: B.desc. Update Unsubypting to preserve the implication
and avoid generating invalid types.

Author: tlively

scripts/test/fuzzing.py

@@ -119,6 +119,7 @@
     'ref.cast_desc.wast',
     'struct.new-desc.wast',
     'remove-unused-types-descriptors.wast',
+    'unsubtyping-desc.wast',
     # TODO: fix split_wast() on tricky escaping situations like a string ending
     #       in \\" (the " is not escaped - there is an escaped \ before it)
     'string-lifting-section.wast',

src/passes/Unsubtyping.cpp

@@ -290,6 +290,11 @@ struct Unsubtyping
           case HeapTypeKind::Basic:
             WASM_UNREACHABLE("unexpected kind");
         }
+        if (auto desc = type.getDescriptorType()) {
+          if (auto superDesc = super.getDescriptorType()) {
+            noteSubtype(*desc, *superDesc);
+          }
+        }
       }
 
       // Analyze all casts at once.

test/lit/passes/unsubtyping-desc.wast

@@ -0,0 +1,64 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py --all-items and should not be edited.
+;; RUN: foreach %s %t wasm-opt -all --closed-world --preserve-type-order \
+;; RUN:     --unsubtyping --remove-unused-types -all -S -o - | filecheck %s
+
+(module
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $A (sub (descriptor $A.desc (struct))))
+    (type $A (sub (descriptor $A.desc (struct))))
+    ;; CHECK:       (type $A.desc (sub (describes $A (struct))))
+    (type $A.desc (sub (describes $A (struct))))
+    ;; CHECK:       (type $B (sub (descriptor $B.desc (struct))))
+    (type $B (sub $A (descriptor $B.desc (struct))))
+    ;; CHECK:       (type $B.desc (sub (describes $B (struct))))
+    (type $B.desc (sub $A.desc (describes $B (struct))))
+  )
+
+  ;; There is nothing requiring the subtype relationship, so we should optimize.
+  ;; CHECK:      (global $A (ref null $A) (ref.null none))
+  (global $A (ref null $A) (ref.null none))
+  ;; CHECK:      (global $B (ref null $B) (ref.null none))
+  (global $B (ref null $B) (ref.null none))
+)
+
+(module
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $A (sub (descriptor $A.desc (struct))))
+    (type $A (sub (descriptor $A.desc (struct))))
+    ;; CHECK:       (type $A.desc (sub (describes $A (struct))))
+    (type $A.desc (sub (describes $A (struct))))
+    ;; CHECK:       (type $B (sub $A (descriptor $B.desc (struct))))
+    (type $B (sub $A (descriptor $B.desc (struct))))
+    ;; CHECK:       (type $B.desc (sub $A.desc (describes $B (struct))))
+    (type $B.desc (sub $A.desc (describes $B (struct))))
+  )
+
+  ;; Now we require B <: A, which implies B.desc <: A.desc.
+  ;; CHECK:      (global $B (ref null $B) (ref.null none))
+  (global $B (ref null $B) (ref.null none))
+  ;; CHECK:      (global $A (ref null $A) (global.get $B))
+  (global $A (ref null $A) (global.get $B))
+)
+
+(module
+  (rec
+    ;; CHECK:      (rec
+    ;; CHECK-NEXT:  (type $A (sub (descriptor $A.desc (struct))))
+    (type $A (sub (descriptor $A.desc (struct))))
+    ;; CHECK:       (type $A.desc (sub (describes $A (struct))))
+    (type $A.desc (sub (describes $A (struct))))
+    ;; CHECK:       (type $B (sub (descriptor $B.desc (struct))))
+    (type $B (sub $A (descriptor $B.desc (struct))))
+    ;; CHECK:       (type $B.desc (sub $A.desc (describes $B (struct))))
+    (type $B.desc (sub $A.desc (describes $B (struct))))
+  )
+
+  ;; Now we directly require B.desc <: A.desc. This does *not* imply B <: A, so
+  ;; we can optimize $B (but not $B.desc).
+  ;; CHECK:      (global $B.desc (ref null $B.desc) (ref.null none))
+  (global $B.desc (ref null $B.desc) (ref.null none))
+  ;; CHECK:      (global $A.desc (ref null $A.desc) (global.get $B.desc))
+  (global $A.desc (ref null $A.desc) (global.get $B.desc))
+)