Support recursive struct in llvm-remove-addrspaces (#38573)

Author: tkf

src/llvm-remove-addrspaces.cpp

@@ -55,20 +55,37 @@ class AddrspaceRemoveTypeRemapper : public ValueMapTypeRemapper {
                     remapType(Ty->getReturnType()), Params, Ty->isVarArg());
         }
         else if (auto Ty = dyn_cast<StructType>(SrcTy)) {
-            if (!Ty->isOpaque()) {
-                auto Els = Ty->getNumElements();
-                SmallVector<Type *, 4> NewElTys(Els);
-                for (unsigned i = 0; i < Els; ++i)
-                    NewElTys[i] = remapType(Ty->getElementType(i));
+            if (Ty->isLiteral()) {
+                // Since a literal type has to have the body when it is created,
+                // we need to remap the element types first. This is safe only
+                // for literal types (i.e., no self-reference) and thus treated
+                // separately.
+                assert(!Ty->hasName()); // literal type has no name.
+                SmallVector<Type *, 4> NewElTys;
+                NewElTys.reserve(Ty->getNumElements());
+                for (auto E: Ty->elements())
+                    NewElTys.push_back(remapType(E));
+                DstTy = StructType::get(Ty->getContext(), NewElTys, Ty->isPacked());
+            } else if (!Ty->isOpaque()) {
+                // If the struct type is not literal and not opaque, it can have
+                // self-referential fields (i.e., pointer type of itself as a
+                // field).
+                StructType *DstTy_ = StructType::create(Ty->getContext());
                 if (Ty->hasName()) {
                     auto Name = std::string(Ty->getName());
                     Ty->setName(Name + ".bad");
-                    DstTy = StructType::create(
-                            Ty->getContext(), NewElTys, Name, Ty->isPacked());
+                    DstTy_->setName(Name);
                 }
-                else
-                    DstTy = StructType::get(
-                            Ty->getContext(), NewElTys, Ty->isPacked());
+                // To avoid infinite recursion, shove the placeholder of the DstTy before
+                // recursing into the element types:
+                MappedTypes[SrcTy] = DstTy_;
+
+                auto Els = Ty->getNumElements();
+                SmallVector<Type *, 4> NewElTys(Els);
+                for (unsigned i = 0; i < Els; ++i)
+                    NewElTys[i] = remapType(Ty->getElementType(i));
+                DstTy_->setBody(NewElTys, Ty->isPacked());
+                DstTy = DstTy_;
             }
         }
         else if (auto Ty = dyn_cast<ArrayType>(SrcTy))

test/llvmpasses/remove-addrspaces.ll

@@ -41,3 +41,63 @@ top:
   store i64 %0, i64 addrspace(13)* %4, align 8
   ret {} addrspace(10)* %1
 }
+
+; COM: A type used for testing that remove-addrspaces can handle recursive types.
+%list = type { i64, %list* }
+
+; COM: There's nothing to remove in this function; but remove-addrspaces shouldn't crash.
+define i64 @sum.linked.list() #0 {
+; CHECK-LABEL: @sum.linked.list
+top:
+  %a = alloca %list
+  %b = alloca %list
+  %c = alloca %list
+  %a.car = getelementptr %list, %list* %a, i32 0, i32 0
+  %a.cdr = getelementptr %list, %list* %a, i32 0, i32 1
+  %b.car = getelementptr %list, %list* %b, i32 0, i32 0
+  %b.cdr = getelementptr %list, %list* %b, i32 0, i32 1
+  %c.car = getelementptr %list, %list* %c, i32 0, i32 0
+  %c.cdr = getelementptr %list, %list* %c, i32 0, i32 1
+; COM: Allow remove-addrspaces to rename the type but expect it to use the same prefix.
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %a
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %a
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %b
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %b
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %c
+; CHECK: getelementptr %list
+; CHECK-SAME: %list
+; CHECK-SAME: * %c
+  store i64 111, i64* %a.car
+  store i64 222, i64* %b.car
+  store i64 333, i64* %c.car
+  store %list* %b, %list** %a.cdr
+  store %list* %c, %list** %b.cdr
+  store %list* null, %list** %c.cdr
+  br label %loop
+
+loop:
+  %x = phi %list* [ %a, %top ], [ %x.cdr.value, %loop ]
+  %sum.prev = phi i64 [ 0, %top ], [ %sum, %loop ]
+  %x.car = getelementptr %list, %list* %x, i32 0, i32 0
+  %x.cdr = getelementptr %list, %list* %x, i32 0, i32 1
+  %x.car.value = load i64, i64* %x.car
+  %x.cdr.value = load %list*, %list** %x.cdr
+  %sum = add i64 %sum.prev, %x.car.value
+  %null.int = ptrtoint %list* null to i64
+  %x.cdr.value.int = ptrtoint %list* %x.cdr.value to i64
+  %cond = icmp eq i64 %x.cdr.value.int, %null.int
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  ret i64 %sum
+}