diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp index 69b92917399fd..c1517d26d7942 100644 --- a/llvm/lib/CodeGen/RegAllocBase.cpp +++ b/llvm/lib/CodeGen/RegAllocBase.cpp @@ -21,6 +21,7 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Spiller.h" +#include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/VirtRegMap.h" #include "llvm/IR/DiagnosticInfo.h" @@ -60,6 +61,7 @@ void RegAllocBase::init(VirtRegMap &vrm, LiveIntervals &lis, LiveRegMatrix &mat) { TRI = &vrm.getTargetRegInfo(); MRI = &vrm.getRegInfo(); + TII = vrm.getMachineFunction().getSubtarget().getInstrInfo(); VRM = &vrm; LIS = &lis; Matrix = &mat; @@ -167,9 +169,15 @@ void RegAllocBase::cleanupFailedVReg(Register FailedReg, MCRegister PhysReg, // We still should produce valid IR. Kill all the uses and reduce the live // ranges so that we don't think it's possible to introduce kill flags later // which will fail the verifier. + + SmallVector UndefCopies; + for (MachineOperand &MO : MRI->reg_operands(FailedReg)) { - if (MO.readsReg()) + if (MO.readsReg()) { MO.setIsUndef(true); + if (MO.getParent()->isCopy() && MO.isUse()) + UndefCopies.push_back(MO.getParent()); + } } if (!MRI->isReserved(PhysReg)) { @@ -180,12 +188,22 @@ void RegAllocBase::cleanupFailedVReg(Register FailedReg, MCRegister PhysReg, for (MachineOperand &MO : MRI->reg_operands(*Aliases)) { if (MO.readsReg()) { MO.setIsUndef(true); + if (MO.getParent()->isCopy() && MO.isUse()) + UndefCopies.push_back(MO.getParent()); LIS->removeAllRegUnitsForPhysReg(MO.getReg()); } } } } + // If we have produced an undef copy, convert to IMPLICIT_DEF. + for (MachineInstr *UndefCopy : UndefCopies) { + assert(UndefCopy->isCopy() && UndefCopy->getNumOperands() == 2); + const MCInstrDesc &Desc = TII->get(TargetOpcode::IMPLICIT_DEF); + UndefCopy->removeOperand(1); + UndefCopy->setDesc(Desc); + } + // Directly perform the rewrite, and do not leave it to VirtRegRewriter as // usual. This avoids trying to manage illegal overlapping assignments in // LiveRegMatrix. diff --git a/llvm/lib/CodeGen/RegAllocBase.h b/llvm/lib/CodeGen/RegAllocBase.h index f1b5af8cd4d74..58743283458a6 100644 --- a/llvm/lib/CodeGen/RegAllocBase.h +++ b/llvm/lib/CodeGen/RegAllocBase.h @@ -65,6 +65,7 @@ class RegAllocBase { protected: const TargetRegisterInfo *TRI = nullptr; + const TargetInstrInfo *TII = nullptr; MachineRegisterInfo *MRI = nullptr; VirtRegMap *VRM = nullptr; LiveIntervals *LIS = nullptr; diff --git a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir index c1e0d0716acae..d12f5d7ef2315 100644 --- a/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir +++ b/llvm/test/CodeGen/AMDGPU/inflated-reg-class-snippet-copy-use-after-free.mir @@ -29,8 +29,8 @@ # CHECK-NEXT: SI_SPILL_AV512_SAVE [[ORIG_REG]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) # CHECK-NEXT: [[RESTORE0:%[0-9]+]]:vreg_512_align2 = SI_SPILL_V512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) # CHECK-NEXT: early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_16X16X1F32_vgprcd_e64 undef %3:vgpr_32, undef %3:vgpr_32, [[RESTORE0]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec -# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 { -# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0 +# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = IMPLICIT_DEF { +# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = IMPLICIT_DEF # CHECK-NEXT: } # CHECK-NEXT: undef [[SPLIT1:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 { # CHECK-NEXT: internal [[SPLIT1]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0 @@ -120,8 +120,8 @@ body: | # CHECK-NEXT: SI_SPILL_AV512_SAVE [[ORIG_REG]], %stack.0, $sgpr32, 0, implicit $exec :: (store (s512) into %stack.0, align 4, addrspace 5) # CHECK-NEXT: [[RESTORE_0:%[0-9]+]]:av_512_align2 = SI_SPILL_AV512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s512) from %stack.0, align 4, addrspace 5) # CHECK-NEXT: S_NOP 0, implicit-def early-clobber $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit [[RESTORE_0]].sub0_sub1_sub2_sub3, implicit [[RESTORE_0]].sub4_sub5_sub6_sub7 -# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = COPY undef $vgpr2_vgpr3 { -# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = COPY undef $vgpr0 +# CHECK-NEXT: undef [[SPLIT0:%[0-9]+]].sub2_sub3:av_512_align2 = IMPLICIT_DEF { +# CHECK-NEXT: internal [[SPLIT0]].sub0:av_512_align2 = IMPLICIT_DEF # CHECK-NEXT: } # CHECK-NEXT: undef [[SPLIT1:%[0-9]+]].sub2_sub3:av_512_align2 = COPY [[SPLIT0]].sub2_sub3 { # CHECK-NEXT: internal [[SPLIT1]].sub0:av_512_align2 = COPY [[SPLIT0]].sub0 diff --git a/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir new file mode 100644 index 0000000000000..ce1bb1f6f4b16 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/regalloc-undef-copy-fold.mir @@ -0,0 +1,115 @@ +# RUN: not llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs -start-before=greedy,1 -stop-after=virtregrewriter,2 -o - -verify-regalloc %s 2> %t.err | FileCheck %s +# RUN: FileCheck -check-prefix=ERR %s < %t.err + +# Make sure there's no machine verifier error after failure. + +# ERR: error: inline assembly requires more registers than available +# ERR: error: inline assembly requires more registers than available +# ERR: error: inline assembly requires more registers than available +# ERR: error: inline assembly requires more registers than available + +# This testcase cannot be compiled with the enforced register +# budget. Previously, tryLastChanceRecoloring would assert here. It +# was attempting to recolor a superregister with an overlapping +# subregister over the same range. + +--- | + define void @dead_copy() #0 { + ret void + } + + define void @copy_kill() #0 { + ret void + } + + define void @copy_subreg() #0 { + ret void + } + + define void @copy_subreg2() #0 { + ret void + } + + attributes #0 = { "amdgpu-num-vgpr"="6" } + +... + +# CHECK-LABEL: name: dead_copy +# CHECK: renamable $agpr0_agpr1_agpr2 = IMPLICIT_DEF +# CHECK: dead renamable $vgpr0_vgpr1_vgpr2 = COPY renamable $agpr0_agpr1_agpr2 + +--- +name: dead_copy +tracksRegLiveness: true +machineFunctionInfo: + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' +body: | + bb.0: + + INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 11534346 /* regdef:VReg_512 */, def %0:vreg_512, 10158090 /* regdef:VReg_256 */, def %1:vreg_256, 4784138 /* regdef:VReg_128 */, def %2:vreg_128, 3670026 /* regdef:VReg_96 */, def %3:vreg_96, 3670026 /* regdef:VReg_96 */, def %4:vreg_96 + %5:vreg_96 = COPY %3 + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3670025 /* reguse:VReg_96 */, %3 + SI_RETURN + +... + + +# CHECK-LABEL: name: copy_kill +# CHECK: renamable $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF + +--- +name: copy_kill +tracksRegLiveness: true +machineFunctionInfo: + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' +body: | + bb.0: + + INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 11534346 /* regdef:VReg_512 */, def %0:vreg_512, 10158090 /* regdef:VReg_256 */, def %1:vreg_256, 4784138 /* regdef:VReg_128 */, def %2:vreg_128, 3670026 /* regdef:VReg_96 */, def %3:vreg_96, 3670026 /* regdef:VReg_96 */, def %4:vreg_96 + %5:vreg_96 = COPY %3 + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3670025 /* reguse:VReg_96 */, %5 + SI_RETURN +... + +# CHECK-LABEL: name: copy_subreg +# CHECK: renamable $vgpr1_vgpr2 = IMPLICIT_DEF +# CHECK: renamable $vgpr0 = COPY renamable $vgpr1 +--- +name: copy_subreg +tracksRegLiveness: true +machineFunctionInfo: + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' +body: | + bb.0: + + INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 11534346 /* regdef:VReg_512 */, def %0:vreg_512, 10158090 /* regdef:VReg_256 */, def %1:vreg_256, 4784138 /* regdef:VReg_128 */, def %2:vreg_128, 3670026 /* regdef:VReg_96 */, def %3:vreg_96, 3670026 /* regdef:VReg_96 */, def %4:vreg_96 + %3.sub0 = COPY %3.sub1 + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3670025 /* reguse:VReg_96 */, %3 + SI_RETURN +... + +# CHECK-LABEL: name: copy_subreg2 +# CHECK: renamable $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF + +--- +name: copy_subreg2 +tracksRegLiveness: true +machineFunctionInfo: + scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3' + frameOffsetReg: '$sgpr33' + stackPtrOffsetReg: '$sgpr32' +body: | + bb.0: + + INLINEASM &"; def $0 $1 $2 $3 $4", 1 /* sideeffect attdialect */, 11534346 /* regdef:VReg_512 */, def %0:vreg_512, 10158090 /* regdef:VReg_256 */, def %1:vreg_256, 4784138 /* regdef:VReg_128 */, def %2:vreg_128, 3670026 /* regdef:VReg_96 */, def %3:vreg_96, 3670026 /* regdef:VReg_96 */, def %4:vreg_96 + undef %5.sub0:vreg_96 = COPY %3.sub0 + %5.sub1_sub2:vreg_96 = COPY %3.sub1_sub2 + INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3670025 /* reguse:VReg_96 */, %5 + SI_RETURN +...