From 269757404be275cb3de8d5a69eb92700d9bfba78 Mon Sep 17 00:00:00 2001 From: Matthew Curtis Date: Wed, 9 Jul 2025 03:32:37 -0500 Subject: [PATCH 1/3] AMDGPU: Fix runtime unrolling when cascaded GEPs present --- .../AMDGPU/AMDGPUTargetTransformInfo.cpp | 9 ++-- .../LoopUnroll/AMDGPU/unroll-runtime.ll | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index f693580929518..24f4df2aff9d1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -216,10 +216,13 @@ void AMDGPUTTIImpl::getUnrollingPreferences( // a variable, most likely we will be unable to combine it. // Do not unroll too deep inner loops for local memory to give a chance // to unroll an outer loop for a more important reason. - if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 || - (!isa(GEP->getPointerOperand()) && - !isa(GEP->getPointerOperand()))) + if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2) continue; + + const Value *V = getUnderlyingObject(GEP->getPointerOperand()); + if (!isa(V) && !isa(V)) + continue; + LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n" << *L << " due to LDS use.\n"); UP.Runtime = UnrollRuntimeLocal; diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll new file mode 100644 index 0000000000000..9414a1e24e542 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll @@ -0,0 +1,44 @@ +; RUN: opt -mtriple=amdgcn-unknown-amdhsa -passes=loop-unroll -S %s | FileCheck %s + +%struct.wombat = type { %struct.zot, i32, [16 x i32], [16 x i32], i32, i32, [16 x i32], i32 } +%struct.zot = type { i32, i32, [1024 x i32] } + +@global = external addrspace(3) global %struct.wombat + +; Ensure that a cascaded GEP for local address space does not inhibit unrolling +; +; CHECK-LABEL: @unroll_when_cascaded_gep +; CHECK: bb: +; CHECK: br {{.*}}, label %bb2.unr-lcssa, label %bb.new +; CHECK: bb.new: +; CHECK: %unroll_iter = +; CHECK: br label %bb1 +; CHECK: bb1: +; CHECK: br {{.*}}, label %bb2.unr-lcssa.loopexit, label %bb1 +; CHECK: bb2.unr-lcssa.loopexit: +; CHECK: br label %bb2.unr-lcssa +; CHECK: bb2.unr-lcssa: +; CHECK: br {{.*}}, label %bb1.epil.preheader, label %bb2 +; CHECK: bb1.epil.preheader: +; CHECK: br label %bb1.epil +; CHECK: bb1.epil: +; CHECK: br {{.*}}, label %bb1.epil, label %bb2.epilog-lcssa +; CHECK: bb2.epilog-lcssa: +; CHECK: br label %bb2 +; CHECK: bb2: +; CHECK: ret void +define amdgpu_kernel void @unroll_when_cascaded_gep(i32 %arg) { +bb: + br label %bb1 + +bb1: ; preds = %bb1, %bb + %phi = phi i32 [ 0, %bb ], [ %add, %bb1 ] + %getelementptr = getelementptr [1024 x i32], ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @global, i32 8), i32 0, i32 0 + %add = add i32 %phi, 1 + %icmp = icmp eq i32 %phi, %arg + br i1 %icmp, label %bb2, label %bb1 + +bb2: ; preds = %bb1 + ret void +} + From e545559416f59540c6def4270251b8c126570f7e Mon Sep 17 00:00:00 2001 From: Matthew Curtis Date: Wed, 9 Jul 2025 13:24:12 -0500 Subject: [PATCH 2/3] fixup! AMDGPU: Fix runtime unrolling when cascaded GEPs present --- .../LoopUnroll/AMDGPU/unroll-runtime.ll | 62 ++++++++++++------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll index 9414a1e24e542..aa3bd7cfa17ba 100644 --- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll +++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll @@ -1,4 +1,5 @@ -; RUN: opt -mtriple=amdgcn-unknown-amdhsa -passes=loop-unroll -S %s | FileCheck %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=loop-unroll -S %s | FileCheck %s %struct.wombat = type { %struct.zot, i32, [16 x i32], [16 x i32], i32, i32, [16 x i32], i32 } %struct.zot = type { i32, i32, [1024 x i32] } @@ -7,27 +8,46 @@ ; Ensure that a cascaded GEP for local address space does not inhibit unrolling ; -; CHECK-LABEL: @unroll_when_cascaded_gep -; CHECK: bb: -; CHECK: br {{.*}}, label %bb2.unr-lcssa, label %bb.new -; CHECK: bb.new: -; CHECK: %unroll_iter = -; CHECK: br label %bb1 -; CHECK: bb1: -; CHECK: br {{.*}}, label %bb2.unr-lcssa.loopexit, label %bb1 -; CHECK: bb2.unr-lcssa.loopexit: -; CHECK: br label %bb2.unr-lcssa -; CHECK: bb2.unr-lcssa: -; CHECK: br {{.*}}, label %bb1.epil.preheader, label %bb2 -; CHECK: bb1.epil.preheader: -; CHECK: br label %bb1.epil -; CHECK: bb1.epil: -; CHECK: br {{.*}}, label %bb1.epil, label %bb2.epilog-lcssa -; CHECK: bb2.epilog-lcssa: -; CHECK: br label %bb2 -; CHECK: bb2: -; CHECK: ret void define amdgpu_kernel void @unroll_when_cascaded_gep(i32 %arg) { +; CHECK-LABEL: @unroll_when_cascaded_gep( +; CHECK-NEXT: bb: +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[ARG:%.*]], 1 +; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 7 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 [[ARG]], 7 +; CHECK-NEXT: br i1 [[TMP1]], label [[BB2_UNR_LCSSA:%.*]], label [[BB_NEW:%.*]] +; CHECK: bb.new: +; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]] +; CHECK-NEXT: br label [[BB1:%.*]] +; CHECK: bb1: +; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, [[BB_NEW]] ], [ [[ADD_7:%.*]], [[BB1]] ] +; CHECK-NEXT: [[NITER:%.*]] = phi i32 [ 0, [[BB_NEW]] ], [ [[NITER_NEXT_7:%.*]], [[BB1]] ] +; CHECK-NEXT: [[ADD_7]] = add i32 [[PHI]], 8 +; CHECK-NEXT: [[NITER_NEXT_7]] = add i32 [[NITER]], 8 +; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i32 [[NITER_NEXT_7]], [[UNROLL_ITER]] +; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label [[BB2_UNR_LCSSA_LOOPEXIT:%.*]], label [[BB1]] +; CHECK: bb2.unr-lcssa.loopexit: +; CHECK-NEXT: [[PHI_UNR_PH:%.*]] = phi i32 [ [[ADD_7]], [[BB1]] ] +; CHECK-NEXT: br label [[BB2_UNR_LCSSA]] +; CHECK: bb2.unr-lcssa: +; CHECK-NEXT: [[PHI_UNR:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[PHI_UNR_PH]], [[BB2_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[BB1_EPIL_PREHEADER:%.*]], label [[BB2:%.*]] +; CHECK: bb1.epil.preheader: +; CHECK-NEXT: br label [[BB1_EPIL:%.*]] +; CHECK: bb1.epil: +; CHECK-NEXT: [[PHI_EPIL:%.*]] = phi i32 [ [[PHI_UNR]], [[BB1_EPIL_PREHEADER]] ], [ [[ADD_EPIL:%.*]], [[BB1_EPIL]] ] +; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, [[BB1_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], [[BB1_EPIL]] ] +; CHECK-NEXT: [[GETELEMENTPTR_EPIL:%.*]] = getelementptr [1024 x i32], ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @global, i32 8), i32 0, i32 0 +; CHECK-NEXT: [[ADD_EPIL]] = add i32 [[PHI_EPIL]], 1 +; CHECK-NEXT: [[ICMP_EPIL:%.*]] = icmp eq i32 [[PHI_EPIL]], [[ARG]] +; CHECK-NEXT: [[EPIL_ITER_NEXT]] = add i32 [[EPIL_ITER]], 1 +; CHECK-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_NEXT]], [[XTRAITER]] +; CHECK-NEXT: br i1 [[EPIL_ITER_CMP]], label [[BB1_EPIL]], label [[BB2_EPILOG_LCSSA:%.*]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: bb2.epilog-lcssa: +; CHECK-NEXT: br label [[BB2]] +; CHECK: bb2: +; CHECK-NEXT: ret void +; bb: br label %bb1 From 9ca34fc518a674d6dd02735a471252cf288c1ca8 Mon Sep 17 00:00:00 2001 From: Matthew Curtis Date: Wed, 9 Jul 2025 13:27:47 -0500 Subject: [PATCH 3/3] fixup! AMDGPU: Fix runtime unrolling when cascaded GEPs present --- llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll index aa3bd7cfa17ba..2486b8039803e 100644 --- a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll +++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime.ll @@ -1,5 +1,5 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=loop-unroll -S %s | FileCheck %s +; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=loop-unroll -S %s -o - | FileCheck %s %struct.wombat = type { %struct.zot, i32, [16 x i32], [16 x i32], i32, i32, [16 x i32], i32 } %struct.zot = type { i32, i32, [1024 x i32] }