From 43f3996d8d3341a2dfb34104c749cd88b4e630b8 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Mon, 24 Mar 2025 17:25:39 -0700 Subject: [PATCH 01/18] AArch64: Relax x16/x17 constraint on AUT in certain cases. On most operating systems, the x16 and x17 registers are not special, so there is no benefit, and only a code size cost, to constraining AUT to only using them. Therefore, adjust the backend to only use the AUT pseudo (renamed AUTx16x17 for clarity) on Darwin platforms, or if traps are requested. All other platforms use the unconstrained variant of the instruction for selection. Pull Request: https://github.com/llvm/llvm-project/pull/132857 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 11 +- .../Target/AArch64/AArch64ISelDAGToDAG.cpp | 10 +- llvm/lib/Target/AArch64/AArch64InstrInfo.td | 25 +- llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 23 ++ llvm/lib/Target/AArch64/AArch64Subtarget.h | 8 + .../GISel/AArch64InstructionSelector.cpp | 2 +- llvm/test/CodeGen/AArch64/ptrauth-call.ll | 53 ++-- llvm/test/CodeGen/AArch64/ptrauth-fpac.ll | 81 +++--- ...trauth-intrinsic-auth-resign-with-blend.ll | 48 ++-- .../AArch64/ptrauth-intrinsic-auth-resign.ll | 256 ++++++++++-------- 10 files changed, 306 insertions(+), 211 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index e3ad085bc5a29..8f26e9b791dff 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -69,15 +69,6 @@ using namespace llvm; -enum PtrauthCheckMode { Default, Unchecked, Poison, Trap }; -static cl::opt PtrauthAuthChecks( - "aarch64-ptrauth-auth-checks", cl::Hidden, - cl::values(clEnumValN(Unchecked, "none", "don't test for failure"), - clEnumValN(Poison, "poison", "poison on failure"), - clEnumValN(Trap, "trap", "trap on failure")), - cl::desc("Check pointer authentication auth/resign failures"), - cl::init(Default)); - #define DEBUG_TYPE "asm-printer" namespace { @@ -2866,7 +2857,7 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } - case AArch64::AUT: + case AArch64::AUTx16x17: case AArch64::AUTPAC: emitPtrauthAuthResign(MI); return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 22083460b400a..2eafb8eae2082 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -361,7 +361,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool tryIndexedLoad(SDNode *N); - void SelectPtrauthAuth(SDNode *N); + void SelectPtrauthAuthX16X17(SDNode *N); void SelectPtrauthResign(SDNode *N); bool trySelectStackSlotTagP(SDNode *N); @@ -1521,7 +1521,7 @@ extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) { AddrDisc); } -void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { +void AArch64DAGToDAGISel::SelectPtrauthAuthX16X17(SDNode *N) { SDLoc DL(N); // IntrinsicID is operand #0 SDValue Val = N->getOperand(1); @@ -1539,7 +1539,7 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { AArch64::X16, Val, SDValue()); SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)}; - SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops); + SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTx16x17, DL, MVT::i64, Ops); ReplaceNode(N, AUT); } @@ -5613,7 +5613,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { return; case Intrinsic::ptrauth_auth: - SelectPtrauthAuth(Node); + if (!Subtarget->isX16X17Safer(CurDAG->getMachineFunction())) + break; + SelectPtrauthAuthX16X17(Node); return; case Intrinsic::ptrauth_resign: diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 6c61e3a613f6f..98a35b3840771 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -66,6 +66,14 @@ def HasLOR : Predicate<"Subtarget->hasLOR()">, def HasPAuth : Predicate<"Subtarget->hasPAuth()">, AssemblerPredicateWithAll<(all_of FeaturePAuth), "pauth">; +// On most operating systems, the x16 and x17 registers are not special, so +// there is no benefit, and only a code size cost, to constraining PAC +// instructions to only using them. This predicate may be used to guard patterns +// that allow PAC instructions to be used with any register. +let RecomputePerFunction = 1 in { + def X16X17NotSafer : Predicate<"!Subtarget->isX16X17Safer(*MF)">; +} + def HasPAuthLR : Predicate<"Subtarget->hasPAuthLR()">, AssemblerPredicateWithAll<(all_of FeaturePAuthLR), "pauth-lr">; @@ -1820,7 +1828,9 @@ let Predicates = [HasPAuth] in { } defm PAC : SignAuth<0b000, 0b010, "pac", int_ptrauth_sign>; - defm AUT : SignAuth<0b001, 0b011, "aut", null_frag>; + let Predicates = [HasPAuth, X16X17NotSafer] in { + defm AUT : SignAuth<0b001, 0b011, "aut", int_ptrauth_auth>; + } def XPACI : ClearAuth<0, "xpaci">; def : Pat<(int_ptrauth_strip GPR64:$Rd, 0), (XPACI GPR64:$Rd)>; @@ -1912,10 +1922,11 @@ let Predicates = [HasPAuth] in { defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>; // AUT pseudo. - // This directly manipulates x16/x17, which are the only registers the OS - // guarantees are safe to use for sensitive operations. - def AUT : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc, GPR64noip:$AddrDisc), - []>, Sched<[WriteI, ReadI]> { + // This directly manipulates x16/x17, which are the only registers that + // certain OSs guarantee are safe to use for sensitive operations. + def AUTx16x17 : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc, + GPR64noip:$AddrDisc), + []>, Sched<[WriteI, ReadI]> { let isCodeGenOnly = 1; let hasSideEffects = 1; let mayStore = 0; @@ -1926,8 +1937,8 @@ let Predicates = [HasPAuth] in { } // AUT and re-PAC a value, using different keys/data. - // This directly manipulates x16/x17, which are the only registers the OS - // guarantees are safe to use for sensitive operations. + // This directly manipulates x16/x17, which are the only registers that + // certain OSs guarantee are safe to use for sensitive operations. def AUTPAC : Pseudo<(outs), (ins i32imm:$AUTKey, i64imm:$AUTDisc, GPR64noip:$AUTAddrDisc, diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 7b4ded6322098..6044e84ab6640 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -102,6 +102,16 @@ static cl::opt UseScalarIncVL("sve-use-scalar-inc-vl", cl::init(false), cl::Hidden, cl::desc("Prefer add+cnt over addvl/inc/dec")); +cl::opt llvm::PtrauthAuthChecks( + "aarch64-ptrauth-auth-checks", cl::Hidden, + cl::values(clEnumValN(PtrauthCheckMode::Unchecked, "none", + "don't test for failure"), + clEnumValN(PtrauthCheckMode::Poison, "poison", + "poison on failure"), + clEnumValN(PtrauthCheckMode::Trap, "trap", "trap on failure")), + cl::desc("Check pointer authentication auth/resign failures"), + cl::init(PtrauthCheckMode::Default)); + unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const { if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0) return OverrideVectorInsertExtractBaseCost; @@ -663,6 +673,19 @@ AArch64Subtarget::getPtrAuthBlockAddressDiscriminatorIfEnabled( (Twine(ParentFn.getName()) + " blockaddress").str()); } +bool AArch64Subtarget::isX16X17Safer(const MachineFunction &MF) const { + // The Darwin kernel implements special protections for x16 and x17 so we + // should prefer to use those registers on that platform. + if (isTargetDarwin()) + return true; + // Traps are only implemented for the pseudo instructions, but are only + // necessary if FEAT_FPAC is not implemented. + if (hasFPAC()) + return false; + return MF.getFunction().hasFnAttribute("ptrauth-auth-traps") || + PtrauthAuthChecks == PtrauthCheckMode::Trap; +} + bool AArch64Subtarget::enableMachinePipeliner() const { return getSchedModel().hasInstrSchedModel(); } diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index f5ffc72cae537..4e8f5f85146bd 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -26,6 +26,7 @@ #include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" +#include "llvm/Support/CommandLine.h" #define GET_SUBTARGETINFO_HEADER #include "AArch64GenSubtargetInfo.inc" @@ -35,6 +36,9 @@ class GlobalValue; class StringRef; class Triple; +enum class PtrauthCheckMode { Default, Unchecked, Poison, Trap }; +extern cl::opt PtrauthAuthChecks; + class AArch64Subtarget final : public AArch64GenSubtargetInfo { public: enum ARMProcFamilyEnum : uint8_t { @@ -318,6 +322,10 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo { } } + /// Returns whether the operating system makes it safer to store sensitive + /// values in x16 and x17 as opposed to other registers. + bool isX16X17Safer(const MachineFunction &MF) const; + /// ParseSubtargetFeatures - Parses features string setting specified /// subtarget options. Definition of function is auto generated by tblgen. void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 67a08e39fe879..6164f6e1e7d12 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -6735,7 +6735,7 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I, MIB.buildCopy({AArch64::X16}, {ValReg}); MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {}); - MIB.buildInstr(AArch64::AUT) + MIB.buildInstr(AArch64::AUTx16x17) .addImm(AUTKey) .addImm(AUTConstDiscC) .addUse(AUTAddrDisc) diff --git a/llvm/test/CodeGen/AArch64/ptrauth-call.ll b/llvm/test/CodeGen/AArch64/ptrauth-call.ll index bf35cf8fecbdb..7eb4cfca40f09 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-call.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-call.ll @@ -169,13 +169,23 @@ define i32 @test_tailcall_ib_var(ptr %arg0, ptr %arg1) #0 { define void @test_tailcall_omit_mov_x16_x16(ptr %objptr) #0 { ; CHECK-LABEL: test_tailcall_omit_mov_x16_x16: -; CHECK-NEXT: ldr x16, [x0] -; CHECK-NEXT: mov x17, x0 -; CHECK-NEXT: movk x17, #6503, lsl #48 -; CHECK-NEXT: autda x16, x17 -; CHECK-NEXT: ldr x1, [x16] -; CHECK-NEXT: movk x16, #54167, lsl #48 -; CHECK-NEXT: braa x1, x16 +; DARWIN-NEXT: ldr x16, [x0] +; DARWIN-NEXT: mov x17, x0 +; DARWIN-NEXT: movk x17, #6503, lsl #48 +; DARWIN-NEXT: autda x16, x17 +; DARWIN-NEXT: ldr x1, [x16] +; DARWIN-NEXT: movk x16, #54167, lsl #48 +; DARWIN-NEXT: braa x1, x16 +; ELF-NEXT: ldr x1, [x0] +; ELF-NEXT: mov x8, x0 +; ELF-NEXT: movk x8, #6503, lsl #48 +; ELF-NEXT: autda x1, x8 +; ELF-NEXT: ldr x2, [x1] +; FIXME: Get rid of the x16/x17 constraint on non-Darwin so we can eliminate +; this mov. +; ELF-NEXT: mov x16, x1 +; ELF-NEXT: movk x16, #54167, lsl #48 +; ELF-NEXT: braa x2, x16 %vtable.signed = load ptr, ptr %objptr, align 8 %objptr.int = ptrtoint ptr %objptr to i64 %vtable.discr = tail call i64 @llvm.ptrauth.blend(i64 %objptr.int, i64 6503) @@ -191,16 +201,27 @@ define void @test_tailcall_omit_mov_x16_x16(ptr %objptr) #0 { define i32 @test_call_omit_extra_moves(ptr %objptr) #0 { ; CHECK-LABEL: test_call_omit_extra_moves: ; DARWIN-NEXT: stp x29, x30, [sp, #-16]! -; ELF-NEXT: str x30, [sp, #-16]! -; CHECK-NEXT: ldr x16, [x0] -; CHECK-NEXT: mov x17, x0 -; CHECK-NEXT: movk x17, #6503, lsl #48 -; CHECK-NEXT: autda x16, x17 -; CHECK-NEXT: ldr x8, [x16] -; CHECK-NEXT: movk x16, #34646, lsl #48 -; CHECK-NEXT: blraa x8, x16 -; CHECK-NEXT: mov w0, #42 +; DARWIN-NEXT: ldr x16, [x0] +; DARWIN-NEXT: mov x17, x0 +; DARWIN-NEXT: movk x17, #6503, lsl #48 +; DARWIN-NEXT: autda x16, x17 +; DARWIN-NEXT: ldr x8, [x16] +; DARWIN-NEXT: movk x16, #34646, lsl #48 +; DARWIN-NEXT: blraa x8, x16 +; DARWIN-NEXT: mov w0, #42 ; DARWIN-NEXT: ldp x29, x30, [sp], #16 +; ELF-NEXT: str x30, [sp, #-16]! +; ELF-NEXT: ldr x8, [x0] +; ELF-NEXT: mov x9, x0 +; ELF-NEXT: movk x9, #6503, lsl #48 +; ELF-NEXT: autda x8, x9 +; ELF-NEXT: ldr x9, [x8] +; FIXME: Get rid of the x16/x17 constraint on non-Darwin so we can eliminate +; this mov. +; ELF-NEXT: mov x17, x8 +; ELF-NEXT: movk x17, #34646, lsl #48 +; ELF-NEXT: blraa x9, x17 +; ELF-NEXT: mov w0, #42 ; ELF-NEXT: ldr x30, [sp], #16 ; CHECK-NEXT: ret %vtable.signed = load ptr, ptr %objptr diff --git a/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll b/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll index d5340dcebad57..41d3ffe32c31e 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll @@ -1,17 +1,18 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,NOFPAC -; RUN: llc < %s -mtriple arm64e-apple-darwin -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,FPAC -; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,NOFPAC -; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,FPAC +; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,DARWIN,NOFPAC +; RUN: llc < %s -mtriple arm64e-apple-darwin -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,DARWIN,FPAC,DARWIN-FPAC +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,ELF,NOFPAC +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,ELF,FPAC,ELF-FPAC target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_ia(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_ia: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autia x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autia x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autia x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 %arg1) ret i64 %tmp @@ -20,9 +21,10 @@ define i64 @test_auth_ia(i64 %arg, i64 %arg1) { define i64 @test_auth_ia_zero(i64 %arg) { ; ALL-LABEL: test_auth_ia_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autiza x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autiza x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autiza x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 0) ret i64 %tmp @@ -31,9 +33,10 @@ define i64 @test_auth_ia_zero(i64 %arg) { define i64 @test_auth_ib(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_ib: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autib x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autib x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autib x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 1, i64 %arg1) ret i64 %tmp @@ -42,9 +45,10 @@ define i64 @test_auth_ib(i64 %arg, i64 %arg1) { define i64 @test_auth_ib_zero(i64 %arg) { ; ALL-LABEL: test_auth_ib_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autizb x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autizb x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autizb x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 1, i64 0) ret i64 %tmp @@ -53,9 +57,10 @@ define i64 @test_auth_ib_zero(i64 %arg) { define i64 @test_auth_da(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_da: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autda x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autda x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autda x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 2, i64 %arg1) ret i64 %tmp @@ -64,9 +69,10 @@ define i64 @test_auth_da(i64 %arg, i64 %arg1) { define i64 @test_auth_da_zero(i64 %arg) { ; ALL-LABEL: test_auth_da_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdza x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdza x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdza x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 2, i64 0) ret i64 %tmp @@ -75,9 +81,10 @@ define i64 @test_auth_da_zero(i64 %arg) { define i64 @test_auth_db(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_db: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdb x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdb x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdb x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 3, i64 %arg1) ret i64 %tmp @@ -86,9 +93,10 @@ define i64 @test_auth_db(i64 %arg, i64 %arg1) { define i64 @test_auth_db_zero(i64 %arg) { ; ALL-LABEL: test_auth_db_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdzb x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdzb x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdzb x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 3, i64 0) ret i64 %tmp @@ -362,12 +370,17 @@ define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { ; NOFPAC-NEXT: mov x0, x16 ; NOFPAC-NEXT: ret ; -; FPAC-LABEL: test_auth_trap_attribute: -; FPAC: %bb.0: -; FPAC-NEXT: mov x16, x0 -; FPAC-NEXT: autia x16, x1 -; FPAC-NEXT: mov x0, x16 -; FPAC-NEXT: ret +; DARWIN-FPAC-LABEL: test_auth_trap_attribute: +; DARWIN-FPAC: %bb.0: +; DARWIN-FPAC-NEXT: mov x16, x0 +; DARWIN-FPAC-NEXT: autia x16, x1 +; DARWIN-FPAC-NEXT: mov x0, x16 +; DARWIN-FPAC-NEXT: ret +; +; ELF-FPAC-LABEL: test_auth_trap_attribute: +; ELF-FPAC: %bb.0: +; ELF-FPAC-NEXT: autia x0, x1 +; ELF-FPAC-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 %arg1) ret i64 %tmp } diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll index 74d2370c74c54..ab8ce04b4816d 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll @@ -33,21 +33,27 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_blend(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_blend: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: mov x17, x1 -; UNCHECKED-NEXT: movk x17, #65535, lsl #48 -; UNCHECKED-NEXT: autda x16, x17 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: mov x17, x1 +; UNCHECKED-DARWIN-NEXT: movk x17, #65535, lsl #48 +; UNCHECKED-DARWIN-NEXT: autda x16, x17 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: mov x8, x1 +; UNCHECKED-ELF-NEXT: movk x8, #65535, lsl #48 +; UNCHECKED-ELF-NEXT: autda x0, x8 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_blend: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: mov x17, x1 -; CHECKED-NEXT: movk x17, #65535, lsl #48 -; CHECKED-NEXT: autda x16, x17 -; CHECKED-NEXT: mov x0, x16 +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: mov x17, x1 +; CHECKED-DARWIN-NEXT: movk x17, #65535, lsl #48 +; CHECKED-DARWIN-NEXT: autda x16, x17 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: mov x8, x1 +; CHECKED-ELF-NEXT: movk x8, #65535, lsl #48 +; CHECKED-ELF-NEXT: autda x0, x8 ; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_blend: @@ -146,10 +152,10 @@ define i64 @test_resign_blend_and_const(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov x17, x16 ; CHECKED-NEXT: xpacd x17 ; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_1 +; CHECKED-NEXT: b.eq [[L]]auth_success_[[N2:[0-9]]] ; CHECKED-NEXT: mov x16, x17 ; CHECKED-NEXT: b [[L]]resign_end_1 -; CHECKED-NEXT: Lauth_success_1: +; CHECKED-NEXT: Lauth_success_[[N2]]: ; CHECKED-NEXT: mov x17, #56789 ; CHECKED-NEXT: pacdb x16, x17 ; CHECKED-NEXT: Lresign_end_1: @@ -232,10 +238,10 @@ define i64 @test_auth_too_large_discriminator(i64 %arg, i64 %arg1) { ; UNCHECKED-NEXT: mov w8, #65536 ; UNCHECKED-DARWIN-NEXT: bfi x1, x8, #48, #16 ; UNCHECKED-DARWIN-NEXT: mov x16, x0 -; UNCHECKED-ELF-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autda x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 ; UNCHECKED-ELF-NEXT: bfi x1, x8, #48, #16 -; UNCHECKED-NEXT: autda x16, x1 -; UNCHECKED-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autda x0, x1 ; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_too_large_discriminator: @@ -243,10 +249,10 @@ define i64 @test_auth_too_large_discriminator(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov w8, #65536 ; CHECKED-DARWIN-NEXT: bfi x1, x8, #48, #16 ; CHECKED-DARWIN-NEXT: mov x16, x0 -; CHECKED-ELF-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autda x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 ; CHECKED-ELF-NEXT: bfi x1, x8, #48, #16 -; CHECKED-NEXT: autda x16, x1 -; CHECKED-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autda x0, x1 ; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_too_large_discriminator: diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll index fdd5ae29f35ea..a034c586073b3 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefixes=UNCHECKED,UNCHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefixes=UNCHECKED,UNCHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL="L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL="L" --check-prefixes=CHECKED,CHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL="L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL="L" --check-prefixes=CHECKED,CHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL="L" --check-prefix=TRAP @@ -15,14 +15,14 @@ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL="L" --check-prefix=TRAP ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefixes=UNCHECKED,UNCHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefixes=UNCHECKED,UNCHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL=".L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL=".L" --check-prefixes=CHECKED,CHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL=".L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL=".L" --check-prefixes=CHECKED,CHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL=".L" --check-prefix=TRAP @@ -33,18 +33,20 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_ia(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_ia: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autia x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autia x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autia x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autia x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autia x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autia x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia: ; TRAP: %bb.0: @@ -64,18 +66,20 @@ define i64 @test_auth_ia(i64 %arg, i64 %arg1) { define i64 @test_auth_ia_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ia_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autiza x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autiza x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autiza x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autiza x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autiza x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autiza x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia_zero: ; TRAP: %bb.0: @@ -95,18 +99,20 @@ define i64 @test_auth_ia_zero(i64 %arg) { define i64 @test_auth_ib(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_ib: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autib x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autib x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autib x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ib: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autib x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autib x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autib x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ib: ; TRAP: %bb.0: @@ -126,18 +132,20 @@ define i64 @test_auth_ib(i64 %arg, i64 %arg1) { define i64 @test_auth_ib_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ib_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autizb x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autizb x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autizb x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ib_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autizb x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autizb x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autizb x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ib_zero: ; TRAP: %bb.0: @@ -157,18 +165,20 @@ define i64 @test_auth_ib_zero(i64 %arg) { define i64 @test_auth_da(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_da: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autda x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autda x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autda x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_da: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autda x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autda x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autda x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_da: ; TRAP: %bb.0: @@ -188,18 +198,20 @@ define i64 @test_auth_da(i64 %arg, i64 %arg1) { define i64 @test_auth_da_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_da_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdza x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdza x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdza x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_da_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdza x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdza x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdza x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_da_zero: ; TRAP: %bb.0: @@ -219,18 +231,20 @@ define i64 @test_auth_da_zero(i64 %arg) { define i64 @test_auth_db(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_db: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdb x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdb x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdb x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_db: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdb x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdb x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdb x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_db: ; TRAP: %bb.0: @@ -250,18 +264,20 @@ define i64 @test_auth_db(i64 %arg, i64 %arg1) { define i64 @test_auth_db_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_db_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdzb x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdzb x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdzb x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_db_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdzb x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdzb x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdzb x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_db_zero: ; TRAP: %bb.0: @@ -535,24 +551,24 @@ define i64 @test_resign_da_dzb(i64 %arg, i64 %arg1, i64 %arg2) { define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { ; UNCHECKED-LABEL: test_auth_trap_attribute: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autia x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-NEXT: mov x16, x0 +; UNCHECKED-NEXT: autia x16, x1 +; UNCHECKED-NEXT: mov x0, x16 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_trap_attribute: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autia x16, x1 -; CHECKED-NEXT: mov x17, x16 -; CHECKED-NEXT: xpaci x17 -; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_6 -; CHECKED-NEXT: brk #0xc470 -; CHECKED-NEXT: Lauth_success_6: -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-NEXT: mov x16, x0 +; CHECKED-NEXT: autia x16, x1 +; CHECKED-NEXT: mov x17, x16 +; CHECKED-NEXT: xpaci x17 +; CHECKED-NEXT: cmp x16, x17 +; CHECKED-NEXT: b.eq [[L]]auth_success_6 +; CHECKED-NEXT: brk #0xc470 +; CHECKED-NEXT: Lauth_success_6: +; CHECKED-NEXT: mov x0, x16 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_trap_attribute: ; TRAP: %bb.0: @@ -572,20 +588,24 @@ define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { define i64 @test_auth_ia_constdisc(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ia_constdisc: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: mov x17, #256 -; UNCHECKED-NEXT: autia x16, x17 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: mov x17, #256 +; UNCHECKED-DARWIN-NEXT: autia x16, x17 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: mov w8, #256 +; UNCHECKED-ELF-NEXT: autia x0, x8 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia_constdisc: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: mov x17, #256 -; CHECKED-NEXT: autia x16, x17 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: mov x17, #256 +; CHECKED-DARWIN-NEXT: autia x16, x17 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: mov w8, #256 +; CHECKED-ELF-NEXT: autia x0, x8 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia_constdisc: ; TRAP: %bb.0: @@ -621,10 +641,10 @@ define i64 @test_resign_da_constdisc(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov x17, x16 ; CHECKED-NEXT: xpacd x17 ; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_7 +; CHECKED-NEXT: b.eq [[L]]auth_success_[[N1:[0-9]]] ; CHECKED-NEXT: mov x16, x17 ; CHECKED-NEXT: b [[L]]resign_end_6 -; CHECKED-NEXT: Lauth_success_7: +; CHECKED-NEXT: Lauth_success_[[N1]]: ; CHECKED-NEXT: mov x17, #256 ; CHECKED-NEXT: pacda x16, x17 ; CHECKED-NEXT: Lresign_end_6: From 5226493644d1f00d57aad66ad72a34a4653c495f Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 20 Mar 2025 11:07:42 -0700 Subject: [PATCH 02/18] libcxx: In gdb test detect execute_mi with feature check instead of version check. The existing version check can lead to test failures on some distribution packages of gdb where not all components of the version number are integers, such as Fedora where gdb.VERSION can be something like "15.2-4.fc41". Fix it by replacing the version check with a feature check. Pull Request: https://github.com/llvm/llvm-project/pull/132291 --- libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py index 254a61a8c633e..630b90c9d77a6 100644 --- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py +++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py @@ -30,7 +30,7 @@ # we exit. has_run_tests = False -has_execute_mi = tuple(map(int, gdb.VERSION.split("."))) >= (14, 2) +has_execute_mi = 'execute_mi' in gdb.__dict__ class CheckResult(gdb.Command): def __init__(self): From 13cb97d74469f3a8b3bc6458ce0a9daaa3a008fd Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Wed, 2 Apr 2025 18:10:52 -0700 Subject: [PATCH 03/18] Extract SipHash implementation into a header. This is so that we'll be able to use it in compiler-rt as well. Dependencies on LLVM Support were removed from the header by restoring code from the original SipHash implementation. Pull Request: https://github.com/llvm/llvm-project/pull/134197 --- llvm/lib/Support/CMakeLists.txt | 5 + llvm/lib/Support/SipHash.cpp | 131 +------------- .../gn/secondary/llvm/lib/Support/BUILD.gn | 1 + third-party/siphash/include/siphash/SipHash.h | 161 ++++++++++++++++++ 4 files changed, 168 insertions(+), 130 deletions(-) create mode 100644 third-party/siphash/include/siphash/SipHash.h diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 2754c97fce6c1..1b18eb7c6346b 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -368,3 +368,8 @@ if(LLVM_WITH_Z3) ${Z3_INCLUDE_DIR} ) endif() + +target_include_directories(LLVMSupport SYSTEM + PRIVATE + ${LLVM_THIRD_PARTY_DIR}/siphash/include + ) diff --git a/llvm/lib/Support/SipHash.cpp b/llvm/lib/Support/SipHash.cpp index 68545913a4f59..682e9231c776f 100644 --- a/llvm/lib/Support/SipHash.cpp +++ b/llvm/lib/Support/SipHash.cpp @@ -15,9 +15,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" +#include "siphash/SipHash.h" #include using namespace llvm; @@ -25,135 +25,6 @@ using namespace support; #define DEBUG_TYPE "llvm-siphash" -// Lightly adapted from the SipHash reference C implementation: -// https://github.com/veorq/SipHash -// by Jean-Philippe Aumasson and Daniel J. Bernstein - -#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) - -#define SIPROUND \ - do { \ - v0 += v1; \ - v1 = ROTL(v1, 13); \ - v1 ^= v0; \ - v0 = ROTL(v0, 32); \ - v2 += v3; \ - v3 = ROTL(v3, 16); \ - v3 ^= v2; \ - v0 += v3; \ - v3 = ROTL(v3, 21); \ - v3 ^= v0; \ - v2 += v1; \ - v1 = ROTL(v1, 17); \ - v1 ^= v2; \ - v2 = ROTL(v2, 32); \ - } while (0) - -namespace { - -/// Computes a SipHash value -/// -/// \param in: pointer to input data (read-only) -/// \param inlen: input data length in bytes (any size_t value) -/// \param k: reference to the key data 16-byte array (read-only) -/// \returns output data, must be 8 or 16 bytes -/// -template -void siphash(const unsigned char *in, uint64_t inlen, - const unsigned char (&k)[16], unsigned char (&out)[outlen]) { - - const unsigned char *ni = (const unsigned char *)in; - const unsigned char *kk = (const unsigned char *)k; - - static_assert(outlen == 8 || outlen == 16, "result should be 8 or 16 bytes"); - - uint64_t v0 = UINT64_C(0x736f6d6570736575); - uint64_t v1 = UINT64_C(0x646f72616e646f6d); - uint64_t v2 = UINT64_C(0x6c7967656e657261); - uint64_t v3 = UINT64_C(0x7465646279746573); - uint64_t k0 = endian::read64le(kk); - uint64_t k1 = endian::read64le(kk + 8); - uint64_t m; - int i; - const unsigned char *end = ni + inlen - (inlen % sizeof(uint64_t)); - const int left = inlen & 7; - uint64_t b = ((uint64_t)inlen) << 56; - v3 ^= k1; - v2 ^= k0; - v1 ^= k1; - v0 ^= k0; - - if (outlen == 16) - v1 ^= 0xee; - - for (; ni != end; ni += 8) { - m = endian::read64le(ni); - v3 ^= m; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= m; - } - - switch (left) { - case 7: - b |= ((uint64_t)ni[6]) << 48; - LLVM_FALLTHROUGH; - case 6: - b |= ((uint64_t)ni[5]) << 40; - LLVM_FALLTHROUGH; - case 5: - b |= ((uint64_t)ni[4]) << 32; - LLVM_FALLTHROUGH; - case 4: - b |= ((uint64_t)ni[3]) << 24; - LLVM_FALLTHROUGH; - case 3: - b |= ((uint64_t)ni[2]) << 16; - LLVM_FALLTHROUGH; - case 2: - b |= ((uint64_t)ni[1]) << 8; - LLVM_FALLTHROUGH; - case 1: - b |= ((uint64_t)ni[0]); - break; - case 0: - break; - } - - v3 ^= b; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= b; - - if (outlen == 16) - v2 ^= 0xee; - else - v2 ^= 0xff; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - endian::write64le(out, b); - - if (outlen == 8) - return; - - v1 ^= 0xdd; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - endian::write64le(out + 8, b); -} - -} // end anonymous namespace - void llvm::getSipHash_2_4_64(ArrayRef In, const uint8_t (&K)[16], uint8_t (&Out)[8]) { siphash<2, 4>(In.data(), In.size(), K, Out); diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn index fe7ff6ef68f99..5461ed5246b85 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn @@ -31,6 +31,7 @@ static_library("Support") { include_dirs = [ "Unix", "Windows", + "//third-party/siphash/include", ] sources = [ "AArch64AttributeParser.cpp", diff --git a/third-party/siphash/include/siphash/SipHash.h b/third-party/siphash/include/siphash/SipHash.h new file mode 100644 index 0000000000000..9653e9428b123 --- /dev/null +++ b/third-party/siphash/include/siphash/SipHash.h @@ -0,0 +1,161 @@ +//===--- SipHash.h - An implementation of SipHash -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This is a header-only implementation of SipHash. It lacks library +// dependencies so it can be used from LLVM and compiler-rt. +// +//===----------------------------------------------------------------------===// + +#include +#include + +// Lightly adapted from the SipHash reference C implementation: +// https://github.com/veorq/SipHash +// by Jean-Philippe Aumasson and Daniel J. Bernstein + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v)); \ + (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); \ + (p)[3] = (uint8_t)((v) >> 24); + +#define U64TO8_LE(p, v) \ + U32TO8_LE((p), (uint32_t)((v))); \ + U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define U8TO64_LE(p) \ + (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) + +#define SIPROUND \ + do { \ + v0 += v1; \ + v1 = ROTL(v1, 13); \ + v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; \ + v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 17); \ + v1 ^= v2; \ + v2 = ROTL(v2, 32); \ + } while (0) + +namespace { + +/// Computes a SipHash value +/// +/// \param in: pointer to input data (read-only) +/// \param inlen: input data length in bytes (any size_t value) +/// \param k: reference to the key data 16-byte array (read-only) +/// \returns output data, must be 8 or 16 bytes +/// +template +void siphash(const unsigned char *in, uint64_t inlen, + const unsigned char (&k)[16], unsigned char (&out)[outlen]) { + + const unsigned char *ni = (const unsigned char *)in; + const unsigned char *kk = (const unsigned char *)k; + + static_assert(outlen == 8 || outlen == 16, "result should be 8 or 16 bytes"); + + uint64_t v0 = UINT64_C(0x736f6d6570736575); + uint64_t v1 = UINT64_C(0x646f72616e646f6d); + uint64_t v2 = UINT64_C(0x6c7967656e657261); + uint64_t v3 = UINT64_C(0x7465646279746573); + uint64_t k0 = U8TO64_LE(kk); + uint64_t k1 = U8TO64_LE(kk + 8); + uint64_t m; + int i; + const unsigned char *end = ni + inlen - (inlen % sizeof(uint64_t)); + const int left = inlen & 7; + uint64_t b = ((uint64_t)inlen) << 56; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; + + if (outlen == 16) + v1 ^= 0xee; + + for (; ni != end; ni += 8) { + m = U8TO64_LE(ni); + v3 ^= m; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } + + switch (left) { + case 7: + b |= ((uint64_t)ni[6]) << 48; + /* FALLTHRU */ + case 6: + b |= ((uint64_t)ni[5]) << 40; + /* FALLTHRU */ + case 5: + b |= ((uint64_t)ni[4]) << 32; + /* FALLTHRU */ + case 4: + b |= ((uint64_t)ni[3]) << 24; + /* FALLTHRU */ + case 3: + b |= ((uint64_t)ni[2]) << 16; + /* FALLTHRU */ + case 2: + b |= ((uint64_t)ni[1]) << 8; + /* FALLTHRU */ + case 1: + b |= ((uint64_t)ni[0]); + /* FALLTHRU */ + break; + case 0: + break; + } + + v3 ^= b; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + if (outlen == 16) + v2 ^= 0xee; + else + v2 ^= 0xff; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out, b); + + if (outlen == 8) + return; + + v1 ^= 0xdd; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out + 8, b); +} + +} // end anonymous namespace From d8f654be3f7b6e15695165bc6aab8c7082b1a628 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 27 Mar 2025 16:21:42 -0700 Subject: [PATCH 04/18] compiler-rt: Introduce runtime functions for emulated PAC. The emulated PAC runtime functions emulate the ARMv8.3a pointer authentication instructions and are intended for use in heterogeneous testing environments. For more information, see the associated RFC: https://discourse.llvm.org/t/rfc-emulated-pac/85557 TODO: - Add tests. Pull Request: https://github.com/llvm/llvm-project/pull/133530 --- compiler-rt/cmake/Modules/AddCompilerRT.cmake | 8 +- compiler-rt/cmake/builtin-config-ix.cmake | 1 + compiler-rt/lib/builtins/CMakeLists.txt | 9 +- compiler-rt/lib/builtins/aarch64/emupac.cpp | 132 ++++++++++++++++++ .../compiler-rt/lib/builtins/BUILD.gn | 3 + 5 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 compiler-rt/lib/builtins/aarch64/emupac.cpp diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake index c3e734f72392f..2f6fd4ba1e4c9 100644 --- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake +++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake @@ -172,7 +172,7 @@ function(add_compiler_rt_runtime name type) cmake_parse_arguments(LIB "" "PARENT_TARGET" - "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS;EXTENSIONS" + "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS;EXTENSIONS;C_STANDARD;CXX_STANDARD" ${ARGN}) set(libnames) # Until we support this some other way, build compiler-rt runtime without LTO @@ -360,6 +360,12 @@ function(add_compiler_rt_runtime name type) set_target_link_flags(${libname} ${extra_link_flags_${libname}}) set_property(TARGET ${libname} APPEND PROPERTY COMPILE_DEFINITIONS ${LIB_DEFS}) + if(LIB_C_STANDARD) + set_property(TARGET ${libname} PROPERTY C_STANDARD ${LIB_C_STANDARD}) + endif() + if(LIB_CXX_STANDARD) + set_property(TARGET ${libname} PROPERTY CXX_STANDARD ${LIB_CXX_STANDARD}) + endif() set_target_output_directories(${libname} ${output_dir_${libname}}) install(TARGETS ${libname} ARCHIVE DESTINATION ${install_dir_${libname}} diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake index 7bd3269bd999d..7bdd30ee67f3d 100644 --- a/compiler-rt/cmake/builtin-config-ix.cmake +++ b/compiler-rt/cmake/builtin-config-ix.cmake @@ -26,6 +26,7 @@ builtin_check_c_compiler_flag("-Xclang -mcode-object-version=none" COMPILER_RT_H builtin_check_c_compiler_flag(-Wbuiltin-declaration-mismatch COMPILER_RT_HAS_WBUILTIN_DECLARATION_MISMATCH_FLAG) builtin_check_c_compiler_flag(/Zl COMPILER_RT_HAS_ZL_FLAG) builtin_check_c_compiler_flag(-fcf-protection=full COMPILER_RT_HAS_FCF_PROTECTION_FLAG) +builtin_check_c_compiler_flag(-nostdinc++ COMPILER_RT_HAS_NOSTDINCXX_FLAG) builtin_check_c_compiler_source(COMPILER_RT_HAS_ATOMIC_KEYWORD " diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index 19316c52d12ce..ca5cc7d218223 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -6,7 +6,7 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) cmake_minimum_required(VERSION 3.20.0) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) - project(CompilerRTBuiltins C ASM) + project(CompilerRTBuiltins C CXX ASM) set(COMPILER_RT_STANDALONE_BUILD TRUE) set(COMPILER_RT_BUILTINS_STANDALONE_BUILD TRUE) @@ -64,6 +64,8 @@ include(CMakePushCheckState) option(COMPILER_RT_BUILTINS_HIDE_SYMBOLS "Do not export any symbols from the static library." ON) +include_directories(../../../third-party/siphash/include) + # TODO: Need to add a mechanism for logging errors when builtin source files are # added to a sub-directory and not this CMakeLists file. set(GENERIC_SOURCES @@ -570,6 +572,7 @@ set(aarch64_SOURCES ${GENERIC_TF_SOURCES} ${GENERIC_SOURCES} cpu_model/aarch64.c + aarch64/emupac.cpp aarch64/fp_mode.c ) @@ -802,7 +805,7 @@ else () append_list_if(COMPILER_RT_ENABLE_CET -fcf-protection=full BUILTIN_CFLAGS) endif() - append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 BUILTIN_CFLAGS) + append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ BUILTIN_CFLAGS) append_list_if(COMPILER_RT_HAS_WBUILTIN_DECLARATION_MISMATCH_FLAG -Werror=builtin-declaration-mismatch BUILTIN_CFLAGS) # Don't embed directives for picking any specific CRT @@ -917,6 +920,8 @@ else () SOURCES ${${arch}_SOURCES} DEFS ${BUILTIN_DEFS} CFLAGS ${BUILTIN_CFLAGS_${arch}} + C_STANDARD 11 + CXX_STANDARD 17 PARENT_TARGET builtins) cmake_pop_check_state() endif () diff --git a/compiler-rt/lib/builtins/aarch64/emupac.cpp b/compiler-rt/lib/builtins/aarch64/emupac.cpp new file mode 100644 index 0000000000000..95a21c969da3d --- /dev/null +++ b/compiler-rt/lib/builtins/aarch64/emupac.cpp @@ -0,0 +1,132 @@ +//===--- emupac.cpp - Emulated PAC implementation -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements Emulated PAC using SipHash_2_4 as the IMPDEF hashing +// scheme. +// +//===----------------------------------------------------------------------===// + +#include + +#include "siphash/SipHash.h" + +// EmuPAC implements runtime emulation of PAC instructions. If the current +// CPU supports PAC, EmuPAC uses real PAC instructions. Otherwise, it uses the +// emulation, which is effectively an implementation of PAC with an IMPDEF +// hashing scheme based on SipHash_2_4. +// +// The purpose of the emulation is to allow programs to be built to be portable +// to machines without PAC support, with some performance loss and increased +// probability of false positives (due to not being able to portably determine +// the VA size), while being functionally almost equivalent to running on a +// machine with PAC support. One example of a use case is if PAC is used in +// production as a security mitigation, but the testing environment is +// heterogeneous (i.e. some machines lack PAC support). In this case we would +// like the testing machines to be able to detect issues resulting +// from the use of PAC instructions that would affect production by running +// tests. This can be achieved by building test binaries with EmuPAC and +// production binaries with real PAC. +// +// The emulation assumes that the VA size is at most 48 bits. The architecture +// as of ARMv8.2, which was the last architecture version in which PAC was not +// mandatory, permitted VA size up to 52 bits via ARMv8.2-LVA, but we are +// unaware of an ARMv8.2 CPU that implemented ARMv8.2-LVA. + +const uint64_t kMaxVASize = 48; +const uint64_t kPACMask = ((1ULL << 55) - 1) & ~((1ULL << kMaxVASize) - 1); +const uint64_t kTTBR1Mask = 1ULL << 55; + +// Determine whether PAC is supported without accessing memory. This utilizes +// the XPACLRI instruction which will copy bit 55 of x30 into at least bit 54 if +// PAC is supported and acts as a NOP if PAC is not supported. +static bool pac_supported() { + register uintptr_t x30 __asm__("x30") = 1ULL << 55; + __asm__ __volatile__("xpaclri" : "+r"(x30)); + return x30 & (1ULL << 54); +} + +// This asm snippet is used to force the creation of a frame record when +// calling the EmuPAC functions. This is important because the EmuPAC functions +// may crash if an auth failure is detected and may be unwound past using a +// frame pointer based unwinder. +#ifdef __GCC_HAVE_DWARF2_CFI_ASM +#define frame_pointer_wrap(sym) \ + "stp x29, x30, [sp, #-16]!\n" \ + ".cfi_def_cfa_offset 16\n" \ + "mov x29, sp\n" \ + ".cfi_def_cfa w29, 16\n" \ + ".cfi_offset w30, -8\n" \ + ".cfi_offset w29, -16\n" \ + "bl " #sym "\n" \ + ".cfi_def_cfa wsp, 16\n" \ + "ldp x29, x30, [sp], #16\n" \ + ".cfi_def_cfa_offset 0\n" \ + ".cfi_restore w30\n" \ + ".cfi_restore w29\n" \ + "ret" +#else +#define frame_pointer_wrap(sym) \ + "stp x29, x30, [sp, #-16]!\n" \ + "mov x29, sp\n" \ + "bl " #sym "\n" \ + "ldp x29, x30, [sp], #16\n" \ + "ret" +#endif + +static const uint8_t K[16] = {0xb5, 0xd4, 0xc9, 0xeb, 0x79, 0x10, 0x4a, 0x79, + 0x6f, 0xec, 0x8b, 0x1b, 0x42, 0x87, 0x81, 0xd4}; + +__attribute__((flatten)) +extern "C" uint64_t __emupac_pacda_impl(uint64_t ptr, uint64_t disc) { + if (pac_supported()) { + __asm__ __volatile__(".arch_extension pauth\npacda %0, %1" + : "+r"(ptr) + : "r"(disc)); + return ptr; + } + if (ptr & kTTBR1Mask) { + if ((ptr & kPACMask) != kPACMask) { + return ptr | kPACMask; + } + } else { + if (ptr & kPACMask) { + return ptr & ~kPACMask; + } + } + uint64_t hash; + siphash<2, 4>(reinterpret_cast(&ptr), 8, K, + *reinterpret_cast(&hash)); + return (ptr & ~kPACMask) | (hash & kPACMask); +} + +extern "C" __attribute__((naked)) uint64_t __emupac_pacda(uint64_t ptr, uint64_t disc) { + __asm__(frame_pointer_wrap(__emupac_pacda_impl)); +} + +__attribute__((flatten)) +extern "C" uint64_t __emupac_autda_impl(uint64_t ptr, uint64_t disc) { + if (pac_supported()) { + __asm__ __volatile__(".arch_extension pauth\nautda %0, %1" + : "+r"(ptr) + : "r"(disc)); + return ptr; + } + uint64_t ptr_without_pac = + (ptr & kTTBR1Mask) ? (ptr | kPACMask) : (ptr & ~kPACMask); + uint64_t hash; + siphash<2, 4>(reinterpret_cast(&ptr_without_pac), 8, K, + *reinterpret_cast(&hash)); + if (((ptr & ~kPACMask) | (hash & kPACMask)) != ptr) { + __builtin_trap(); + } + return ptr_without_pac; +} + +extern "C" __attribute__((naked)) uint64_t __emupac_autda(uint64_t ptr, uint64_t disc) { + __asm__(frame_pointer_wrap(__emupac_autda_impl)); +} diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn index d1048259bcd44..445c1c269e37e 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn @@ -87,6 +87,7 @@ static_library("builtins") { cflags += [ "-fomit-frame-pointer" ] } cflags_c = [ "-std=c11" ] + cflags_cc = [ "-nostdinc++" ] } sources = [ @@ -500,6 +501,7 @@ static_library("builtins") { if (current_cpu == "arm64") { sources -= [ "fp_mode.c" ] sources += [ + "aarch64/emupac.cpp", "aarch64/fp_mode.c", "cpu_model/aarch64.c", ] @@ -601,6 +603,7 @@ static_library("builtins") { } deps = lse_targets + include_dirs = [ "//third-party/siphash/include" ] } # Currently unused but necessary to make sync_source_lists_from_cmake.py happy. From b4c74c0adbb968df496bff07eb40a26f81b1fd60 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Fri, 28 Feb 2025 22:11:25 -0800 Subject: [PATCH 05/18] ELF: Do not rewrite IFUNC relocations to point to PLT if no GOT/PLT needed. This enables the use of IFUNC to implement custom relocation types, such as the PAuth relocation types that will be introduced in a subsequent pull request. TODO: - Add tests. - Fix broken tests. Pull Request: https://github.com/llvm/llvm-project/pull/133531 --- lld/ELF/Relocations.cpp | 8 +++++--- lld/ELF/Symbols.h | 2 ++ lld/ELF/SyntheticSections.cpp | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 629702b45965b..46117a75ccea8 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -869,7 +869,8 @@ static void addRelativeReloc(Ctx &ctx, InputSectionBase &isec, // relrDyn sections don't support odd offsets. Also, relrDyn sections // don't store the addend values, so we must write it to the relocated // address. - if (part.relrDyn && isec.addralign >= 2 && offsetInSec % 2 == 0) { + if (part.relrDyn && isec.addralign >= 2 && offsetInSec % 2 == 0 && + !sym.isGnuIFunc()) { isec.addReloc({expr, type, offsetInSec, addend, &sym}); if (shard) part.relrDyn->relocsVec[parallel::getThreadIndex()].push_back( @@ -1107,8 +1108,6 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset, } } else if (needsPlt(expr)) { sym.setFlags(NEEDS_PLT); - } else if (LLVM_UNLIKELY(isIfunc)) { - sym.setFlags(HAS_DIRECT_RELOC); } // If the relocation is known to be a link-time constant, we know no dynamic @@ -1194,6 +1193,9 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset, } } + if (LLVM_UNLIKELY(isIfunc && !needsGot(expr) && !needsPlt(expr))) + sym.setFlags(HAS_DIRECT_RELOC); + // When producing an executable, we can perform copy relocations (for // STT_OBJECT) and canonical PLT (for STT_FUNC) if sym is defined by a DSO. // Copy relocations/canonical PLT entries are unsupported for diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h index 64f2f6eaa8d09..d4280f367d23b 100644 --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -42,6 +42,8 @@ void printTraceSymbol(const Symbol &sym, StringRef name); enum { NEEDS_GOT = 1 << 0, NEEDS_PLT = 1 << 1, + // True if this is an ifunc with a direct relocation that cannot be + // represented as a RELATIVE relocation. HAS_DIRECT_RELOC = 1 << 2, // True if this symbol needs a canonical PLT entry, or (during // postScanRelocations) a copy relocation. diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index b03c4282ab1aa..afa0482bae3ba 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1707,9 +1707,13 @@ void RelocationBaseSection::mergeRels() { } void RelocationBaseSection::partitionRels() { + const RelType relativeRel = ctx.target->relativeRel; + const RelType iRelativeRel = ctx.target->iRelativeRel; + for (auto &r : relocs) + if (r.type == relativeRel && r.sym->isGnuIFunc()) + r.type = iRelativeRel; if (!combreloc) return; - const RelType relativeRel = ctx.target->relativeRel; numRelativeRelocs = std::stable_partition(relocs.begin(), relocs.end(), [=](auto &r) { return r.type == relativeRel; }) - From 29c4e5e22f5ba60448739ec55047cab76bb2598a Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Tue, 4 Mar 2025 14:29:38 -0800 Subject: [PATCH 06/18] ELF: Add a -z glibc-228-compat flag for working around an old glibc bug. The -z glibc-228-compat flag is intended to be used for binaries utilizing IFUNCs which need to be compatible with glibc versions containing a bug that was fixed in commit b5c45e83753b27dc538dff2d55d4410c385cf3a4 which was released in version 2.29. The bug causes glibc to mprotect the .text section as RW while calling ifunc resolvers in binaries linked with -z notext, leading to a SIGSEGV at startup time. By setting the W flag on the executable section we work around the bug by avoiding the code path that does the mprotect. It is recommended that binaries linked with this flag contain startup code (e.g. in .init_array) that remaps the executable section as non-writable. TODO: - Add tests. - Possibly decide on another mechanism for enabling this besides the -z flag (e.g. examine symbol version data on libc.so.6). Pull Request: https://github.com/llvm/llvm-project/pull/133532 --- lld/ELF/Config.h | 1 + lld/ELF/Driver.cpp | 1 + lld/ELF/Writer.cpp | 13 +++++++++++++ 3 files changed, 15 insertions(+) diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index 03b3cd4771f49..e611e5059dc19 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -366,6 +366,7 @@ struct Config { bool zCopyreloc; bool zForceBti; bool zForceIbt; + bool zGlibc228Compat; bool zGlobal; bool zHazardplt; bool zIfuncNoplt; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 4555a85a4d216..5a173565e478f 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1559,6 +1559,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) { ctx.arg.zForceBti = hasZOption(args, "force-bti"); ctx.arg.zForceIbt = hasZOption(args, "force-ibt"); ctx.arg.zGcs = getZGcs(ctx, args); + ctx.arg.zGlibc228Compat = hasZOption(args, "glibc-228-compat"); ctx.arg.zGlobal = hasZOption(args, "global"); ctx.arg.zGnustack = getZGnuStack(args); ctx.arg.zHazardplt = hasZOption(args, "hazardplt"); diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 2cea6a44b391a..c266aff3a5153 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -2277,6 +2277,19 @@ static uint64_t computeFlags(Ctx &ctx, uint64_t flags) { return PF_R | PF_W | PF_X; if (ctx.arg.executeOnly && (flags & PF_X)) return flags & ~PF_R; + + // The -z glibc-228-compat flag is used for binaries utilizing IFUNCs which + // need to be compatible with glibc versions containing a bug that was fixed + // in commit b5c45e83753b27dc538dff2d55d4410c385cf3a4 which was released in + // version 2.29. The bug causes glibc to mprotect the .text section as RW + // while calling ifunc resolvers in binaries linked with -z notext, leading to + // a SIGSEGV at startup time. By setting the W flag on the executable section + // we work around the bug by avoiding the code path that does the mprotect. It + // is recommended that binaries linked with this flag contain startup code + // (e.g. in .init_array) that remaps the executable section as non-writable. + if (ctx.arg.zGlibc228Compat && !ctx.arg.zText && (flags & PF_X)) + return flags | PF_W; + return flags; } From 43fd6225677a586752033635c749dbc907a3fc7e Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 27 Mar 2025 17:15:39 -0700 Subject: [PATCH 07/18] CodeGen: Optionally emit PAuth relocations as IRELATIVE relocations. This supports the following use cases: - ConstantPtrAuth expressions that are unrepresentable using standard PAuth relocations such as expressions involving an integer operand or deactivation symbols. - libc implementations that do not support PAuth relocations. For more information see the RFC: https://discourse.llvm.org/t/rfc-structure-protection-a-family-of-uaf-mitigation-techniques/85555 TODO: - Add tests. Pull Request: https://github.com/llvm/llvm-project/pull/133533 --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 178 ++++++++++++++++-- 1 file changed, 163 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 8f26e9b791dff..cbff94f4dc227 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -54,6 +54,7 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCValue.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" @@ -84,6 +85,7 @@ class AArch64AsmPrinter : public AsmPrinter { bool EnableImportCallOptimization = false; DenseMap>> SectionToImportedFunctionCalls; + unsigned PAuthIFuncNextUniqueID = 1; public: AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) @@ -191,6 +193,10 @@ class AArch64AsmPrinter : public AsmPrinter { // authenticating) void LowerLOADgotAUTH(const MachineInstr &MI); + const MCExpr *emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal); + /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst); @@ -2218,6 +2224,145 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, BRInst); } +static void emitAddress(MCStreamer &Streamer, MCRegister Reg, + const MCExpr *Expr, bool DSOLocal, + const MCSubtargetInfo &STI) { + MCValue Val; + if (!Expr->evaluateAsRelocatable(Val, nullptr)) + report_fatal_error("emitAddress could not evaluate"); + if (DSOLocal) { + Streamer.emitInstruction( + MCInstBuilder(AArch64::ADRP) + .addReg(Reg) + .addExpr(AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, + Streamer.getContext())), + STI); + Streamer.emitInstruction( + MCInstBuilder(AArch64::ADDXri) + .addReg(Reg) + .addReg(Reg) + .addExpr(AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, + Streamer.getContext())) + .addImm(0), + STI); + } else { + Streamer.emitInstruction(MCInstBuilder(AArch64::ADRP) + .addReg(Reg) + .addExpr(AArch64MCExpr::create( + Val.getSymA(), AArch64MCExpr::VK_GOT_PAGE, + Streamer.getContext())), + STI); + Streamer.emitInstruction(MCInstBuilder(AArch64::LDRXui) + .addReg(Reg) + .addReg(Reg) + .addExpr(AArch64MCExpr::create( + Val.getSymA(), AArch64MCExpr::VK_GOT_LO12, + Streamer.getContext())), + STI); + if (Val.getConstant()) + Streamer.emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(Reg) + .addReg(Reg) + .addImm(Val.getConstant()) + .addImm(0), + STI); + } +} + +static bool targetSupportsPAuthRelocation(const Triple &TT, + const MCExpr *Target) { + // No released version of glibc supports PAuth relocations. + if (TT.isOSGlibc()) + return false; + + // We emit PAuth constants as IRELATIVE relocations in cases where the + // constant cannot be represented as a PAuth relocation: + // 1) The signed value is not a symbol. + return !isa(Target); +} + +static bool targetSupportsIRelativeRelocation(const Triple &TT) { + // IFUNCs are ELF-only. + if (!TT.isOSBinFormatELF()) + return false; + + // musl doesn't support IFUNCs. + if (TT.isMusl()) + return false; + + return true; +} + +const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal) { + const Triple &TT = TM.getTargetTriple(); + + // We only emit an IRELATIVE relocation if the target supports IRELATIVE and + // does not support the kind of PAuth relocation that we are trying to emit. + if (targetSupportsPAuthRelocation(TT, Target, DSExpr) || + !targetSupportsIRelativeRelocation(TT)) + return nullptr; + + // For now, only the DA key is supported. + if (KeyID != AArch64PACKey::DA) + return nullptr; + + std::unique_ptr STI( + TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); + assert(STI && "Unable to create subtarget info"); + + MCSymbol *Place = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(Place); + OutStreamer->pushSection(); + + OutStreamer->switchSection(OutStreamer->getContext().getELFSection( + ".text.startup", ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHF_EXECINSTR, + 0, "", true, PAuthIFuncNextUniqueID++, nullptr)); + + MCSymbol *IFuncSym = + OutStreamer->getContext().createLinkerPrivateSymbol("pauth_ifunc"); + OutStreamer->emitSymbolAttribute(IFuncSym, MCSA_ELF_TypeIndFunction); + OutStreamer->emitLabel(IFuncSym); + if (isa(Target)) { + OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi) + .addReg(AArch64::X0) + .addExpr(Target) + .addImm(0), + *STI); + } else { + emitAddress(*OutStreamer, AArch64::X0, Target, IsDSOLocal, *STI); + } + if (HasAddressDiversity) { + auto *PlacePlusDisc = MCBinaryExpr::createAdd( + MCSymbolRefExpr::create(Place, OutStreamer->getContext()), + MCConstantExpr::create(static_cast(Disc), + OutStreamer->getContext()), + OutStreamer->getContext()); + emitAddress(*OutStreamer, AArch64::X1, PlacePlusDisc, /*IsDSOLocal=*/true, + *STI); + } else { + emitMOVZ(AArch64::X1, Disc, 0); + } + + MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(PrePACInst); + + // We don't know the subtarget because this is being emitted for a global + // initializer. Because the performance of IFUNC resolvers is unimportant, we + // always call the EmuPAC runtime, which will end up using the PAC instruction + // if the target supports PAC. + MCSymbol *EmuPAC = + OutStreamer->getContext().getOrCreateSymbol("__emupac_pacda"); + const MCSymbolRefExpr *EmuPACRef = + MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext()); + OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef), + *STI); + OutStreamer->popSection(); + + return MCSymbolRefExpr::create(IFuncSym, OutStreamer->getContext()); +} + const MCExpr * AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { MCContext &Ctx = OutContext; @@ -2229,23 +2374,20 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { auto *BaseGVB = dyn_cast(BaseGV); - // If we can't understand the referenced ConstantExpr, there's nothing - // else we can do: emit an error. - if (!BaseGVB) { - BaseGV->getContext().emitError( - "cannot resolve target base/addend of ptrauth constant"); - return nullptr; + const MCExpr *Sym; + if (BaseGVB) { + // If there is an addend, turn that into the appropriate MCExpr. + Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx); + if (Offset.sgt(0)) + Sym = MCBinaryExpr::createAdd( + Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx); + else if (Offset.slt(0)) + Sym = MCBinaryExpr::createSub( + Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx); + } else { + Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx); } - // If there is an addend, turn that into the appropriate MCExpr. - const MCExpr *Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx); - if (Offset.sgt(0)) - Sym = MCBinaryExpr::createAdd( - Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx); - else if (Offset.slt(0)) - Sym = MCBinaryExpr::createSub( - Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx); - uint64_t KeyID = CPA.getKey()->getZExtValue(); // We later rely on valid KeyID value in AArch64PACKeyIDToString call from // AArch64AuthMCExpr::printImpl, so fail fast. @@ -2259,6 +2401,12 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { report_fatal_error("AArch64 PAC Discriminator '" + Twine(Disc) + "' out of range [0, 0xFFFF]"); + // Check if we need to represent this with an IRELATIVE and emit it if so. + if (auto *IFuncSym = emitPAuthRelocationAsIRelative( + Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), + BaseGVB && BaseGVB->isDSOLocal())) + return IFuncSym; + // Finally build the complete @AUTH expr. return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), Ctx); From 646aaefb847eb178a8331879bc2000d1900a841e Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Fri, 28 Mar 2025 14:16:17 -0700 Subject: [PATCH 08/18] ELF: Add support for R_AARCH64_INST32 relocation. The R_AARCH64_INST32 relocation type is to support deactivation symbols. For more information, see the RFC: https://discourse.llvm.org/t/rfc-deactivation-symbols/85556 TODO: - Agree on semantics and relocation type number. - Add tests. Pull Request: https://github.com/llvm/llvm-project/pull/133534 --- lld/ELF/Arch/AArch64.cpp | 8 ++++++++ llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def | 1 + 2 files changed, 9 insertions(+) diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp index 9538dd4a70bae..110d087230a9c 100644 --- a/lld/ELF/Arch/AArch64.cpp +++ b/lld/ELF/Arch/AArch64.cpp @@ -135,6 +135,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, switch (type) { case R_AARCH64_ABS16: case R_AARCH64_ABS32: + case R_AARCH64_INST32: case R_AARCH64_ABS64: case R_AARCH64_ADD_ABS_LO12_NC: case R_AARCH64_LDST128_ABS_LO12_NC: @@ -278,6 +279,7 @@ int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const { case R_AARCH64_PREL16: return SignExtend64<16>(read16(ctx, buf)); case R_AARCH64_ABS32: + case R_AARCH64_INST32: case R_AARCH64_PREL32: return SignExtend64<32>(read32(ctx, buf)); case R_AARCH64_ABS64: @@ -505,6 +507,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel, checkIntUInt(ctx, loc, val, 32, rel); write32(ctx, loc, val); break; + case R_AARCH64_INST32: + if (!rel.sym->isUndefined()) { + checkIntUInt(ctx, loc, val, 32, rel); + write32(ctx, loc, val); + } + break; case R_AARCH64_PLT32: case R_AARCH64_GOTPCREL32: checkInt(ctx, loc, val, 32, rel); diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def index 05b79eae573f7..c9f17ee4e0c7a 100644 --- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def @@ -61,6 +61,7 @@ ELF_RELOC(R_AARCH64_LD64_GOT_LO12_NC, 0x138) ELF_RELOC(R_AARCH64_LD64_GOTPAGE_LO15, 0x139) ELF_RELOC(R_AARCH64_PLT32, 0x13a) ELF_RELOC(R_AARCH64_GOTPCREL32, 0x13b) +ELF_RELOC(R_AARCH64_INST32, 0x13c) // General dynamic TLS relocations ELF_RELOC(R_AARCH64_TLSGD_ADR_PREL21, 0x200) ELF_RELOC(R_AARCH64_TLSGD_ADR_PAGE21, 0x201) From 85a1228835295257ba4a1dbcf261f525b22f7d97 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Fri, 31 Jan 2025 18:35:43 -0800 Subject: [PATCH 09/18] MachineInstrBuilder: Introduce copyMIMetadata() function. This reduces the amount of boilerplate required when adding a new field to MIMetadata and reduces the chance of bugs like the one I fixed in TargetInstrInfo::reassociateOps. Pull Request: https://github.com/llvm/llvm-project/pull/133535 --- .../llvm/CodeGen/MachineInstrBuilder.h | 84 ++++++++----------- llvm/lib/CodeGen/TargetInstrInfo.cpp | 2 +- 2 files changed, 38 insertions(+), 48 deletions(-) diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 49d9402d0c273..576dc7ed3a55c 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -68,6 +68,32 @@ enum { } // end namespace RegState +/// Set of metadata that should be preserved when using BuildMI(). This provides +/// a more convenient way of preserving DebugLoc, PCSections and MMRA. +class MIMetadata { +public: + MIMetadata() = default; + MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) + : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} + MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, + MDNode *MMRA = nullptr) + : DL(DI), PCSections(PCSections), MMRA(MMRA) {} + explicit MIMetadata(const Instruction &From) + : DL(From.getDebugLoc()), + PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} + explicit MIMetadata(const MachineInstr &From) + : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} + + const DebugLoc &getDL() const { return DL; } + MDNode *getPCSections() const { return PCSections; } + MDNode *getMMRAMetadata() const { return MMRA; } + +private: + DebugLoc DL; + MDNode *PCSections = nullptr; + MDNode *MMRA = nullptr; +}; + class MachineInstrBuilder { MachineFunction *MF = nullptr; MachineInstr *MI = nullptr; @@ -317,15 +343,11 @@ class MachineInstrBuilder { } } - const MachineInstrBuilder &setPCSections(MDNode *MD) const { - if (MD) - MI->setPCSections(*MF, MD); - return *this; - } - - const MachineInstrBuilder &setMMRAMetadata(MDNode *MMRA) const { - if (MMRA) - MI->setMMRAMetadata(*MF, MMRA); + const MachineInstrBuilder ©MIMetadata(const MIMetadata &MIMD) const { + if (MIMD.getPCSections()) + MI->setPCSections(*MF, MIMD.getPCSections()); + if (MIMD.getMMRAMetadata()) + MI->setMMRAMetadata(*MF, MIMD.getMMRAMetadata()); return *this; } @@ -343,38 +365,11 @@ class MachineInstrBuilder { } }; -/// Set of metadata that should be preserved when using BuildMI(). This provides -/// a more convenient way of preserving DebugLoc, PCSections and MMRA. -class MIMetadata { -public: - MIMetadata() = default; - MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) - : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} - MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, - MDNode *MMRA = nullptr) - : DL(DI), PCSections(PCSections), MMRA(MMRA) {} - explicit MIMetadata(const Instruction &From) - : DL(From.getDebugLoc()), - PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} - explicit MIMetadata(const MachineInstr &From) - : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} - - const DebugLoc &getDL() const { return DL; } - MDNode *getPCSections() const { return PCSections; } - MDNode *getMMRAMetadata() const { return MMRA; } - -private: - DebugLoc DL; - MDNode *PCSections = nullptr; - MDNode *MMRA = nullptr; -}; - /// Builder interface. Specify how to create the initial instruction itself. inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, MIMD.getDL())) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } /// This version of the builder sets up the first operand as a @@ -382,8 +377,7 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID, Register DestReg) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, MIMD.getDL())) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -398,8 +392,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -417,8 +410,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -450,8 +442,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, @@ -462,8 +453,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr &I, diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 7e0a1e2a8a06e..1ba47db017781 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1380,7 +1380,7 @@ void TargetInstrInfo::reassociateOps( const MCInstrDesc &MCID, Register DestReg) { return MachineInstrBuilder( MF, MF.CreateMachineInstr(MCID, MIMD.getDL(), /*NoImpl=*/true)) - .setPCSections(MIMD.getPCSections()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); }; From cad9057f3d21b725bb8072c2547ba6c606ca50ca Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Fri, 28 Mar 2025 14:16:29 -0700 Subject: [PATCH 10/18] Add IR and codegen support for deactivation symbols. Deactivation symbols are a mechanism for allowing object files to disable specific instructions in other object files at link time. The initial use case is for pointer field protection. For more information, see the RFC: https://discourse.llvm.org/t/rfc-deactivation-symbols/85556 TODO: - Add tests. Pull Request: https://github.com/llvm/llvm-project/pull/133536 --- .../llvm/CodeGen/GlobalISel/CallLowering.h | 2 ++ .../CodeGen/GlobalISel/MachineIRBuilder.h | 5 +++ llvm/include/llvm/CodeGen/ISDOpcodes.h | 2 ++ llvm/include/llvm/CodeGen/MachineInstr.h | 9 +++++ .../llvm/CodeGen/MachineInstrBuilder.h | 27 ++++++++++++--- llvm/include/llvm/CodeGen/SelectionDAG.h | 1 + llvm/include/llvm/CodeGen/SelectionDAGISel.h | 1 + llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 17 ++++++++++ llvm/include/llvm/CodeGen/TargetLowering.h | 6 ++++ llvm/include/llvm/IR/LLVMContext.h | 1 + llvm/include/llvm/Target/Target.td | 1 + llvm/lib/CodeGen/GlobalISel/CallLowering.cpp | 4 +++ llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 3 ++ .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 6 ++-- llvm/lib/CodeGen/MachineInstr.cpp | 6 +++- .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 33 +++++++++++++------ .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 15 +++++++++ .../SelectionDAG/SelectionDAGBuilder.cpp | 19 +++++++++-- .../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 14 +++++++- llvm/lib/IR/LLVMContext.cpp | 4 ++- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 17 ++++++++++ .../Target/AArch64/AArch64ISelLowering.cpp | 3 ++ .../AArch64/GISel/AArch64CallLowering.cpp | 1 + .../MCTargetDesc/AArch64ELFObjectWriter.cpp | 13 ++++++++ llvm/utils/TableGen/DAGISelMatcherEmitter.cpp | 11 +++++-- 25 files changed, 197 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 9c8226660e087..66f9090a967c2 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -161,6 +161,8 @@ class CallLowering { /// True if this call results in convergent operations. bool IsConvergent = true; + + GlobalValue *DeactivationSymbol = nullptr; }; /// Argument handling is mostly uniform between the four places that diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index f9dcbeb370bef..4fc69a6ef67d5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -55,6 +55,7 @@ struct MachineIRBuilderState { MDNode *PCSections = nullptr; /// MMRA Metadata to be set on any instruction we create. MDNode *MMRA = nullptr; + Value *DS = nullptr; /// \name Fields describing the insertion point. /// @{ @@ -368,6 +369,7 @@ class MachineIRBuilder { State.II = MI.getIterator(); setPCSections(MI.getPCSections()); setMMRAMetadata(MI.getMMRAMetadata()); + setDeactivationSymbol(MI.getDeactivationSymbol()); } /// @} @@ -404,6 +406,9 @@ class MachineIRBuilder { /// Set the PC sections metadata to \p MD for all the next build instructions. void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; } + Value *getDeactivationSymbol() { return State.DS; } + void setDeactivationSymbol(Value *DS) { State.DS = DS; } + /// Get the current instruction's MMRA metadata. MDNode *getMMRAMetadata() { return State.MMRA; } diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 59f31f8443947..17db9352de4d9 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1524,6 +1524,8 @@ enum NodeType { // Outputs: Output Chain CLEAR_CACHE, + DEACTIVATION_SYMBOL, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index 997d6a5554e06..2a8f6eaa6e8b3 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -878,6 +878,15 @@ class MachineInstr return nullptr; } + // FIXME: Move to Info. + Value *DeactivationSymbol = nullptr; + Value *getDeactivationSymbol() const { + return DeactivationSymbol; + } + void setDeactivationSymbol(MachineFunction &MF, Value *DeactivationSymbol) { + this->DeactivationSymbol = DeactivationSymbol; + } + /// Helper to extract a CFI type hash if one has been added. uint32_t getCFIType() const { if (!Info) diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 576dc7ed3a55c..2a475b97101f7 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -69,29 +69,44 @@ enum { } // end namespace RegState /// Set of metadata that should be preserved when using BuildMI(). This provides -/// a more convenient way of preserving DebugLoc, PCSections and MMRA. +/// a more convenient way of preserving certain data from the original +/// instruction. class MIMetadata { public: MIMetadata() = default; - MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) - : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} + MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr, + Value *DeactivationSymbol = nullptr) + : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA), + DeactivationSymbol(DeactivationSymbol) {} MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) : DL(DI), PCSections(PCSections), MMRA(MMRA) {} explicit MIMetadata(const Instruction &From) : DL(From.getDebugLoc()), - PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} + PCSections(From.getMetadata(LLVMContext::MD_pcsections)), + DeactivationSymbol(getDeactivationSymbol(&From)) {} explicit MIMetadata(const MachineInstr &From) - : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} + : DL(From.getDebugLoc()), PCSections(From.getPCSections()), + DeactivationSymbol(From.getDeactivationSymbol()) {} const DebugLoc &getDL() const { return DL; } MDNode *getPCSections() const { return PCSections; } MDNode *getMMRAMetadata() const { return MMRA; } + Value *getDeactivationSymbol() const { return DeactivationSymbol; } private: DebugLoc DL; MDNode *PCSections = nullptr; MDNode *MMRA = nullptr; + Value *DeactivationSymbol = nullptr; + + static inline Value *getDeactivationSymbol(const Instruction *I) { + if (auto *CB = dyn_cast(I)) + if (auto Bundle = + CB->getOperandBundle(llvm::LLVMContext::OB_deactivation_symbol)) + return Bundle->Inputs[0].get(); + return nullptr; + } }; class MachineInstrBuilder { @@ -348,6 +363,8 @@ class MachineInstrBuilder { MI->setPCSections(*MF, MIMD.getPCSections()); if (MIMD.getMMRAMetadata()) MI->setMMRAMetadata(*MF, MIMD.getMMRAMetadata()); + if (MIMD.getDeactivationSymbol()) + MI->setDeactivationSymbol(*MF, MIMD.getDeactivationSymbol()); return *this; } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 15a2370e5d8b8..3dba2c6d55e13 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -752,6 +752,7 @@ class SelectionDAG { int64_t offset = 0, unsigned TargetFlags = 0) { return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); } + SDValue getDeactivationSymbol(const GlobalValue *GV); SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); SDValue getTargetFrameIndex(int FI, EVT VT) { return getFrameIndex(FI, VT, true); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 55f8f19d437a0..34528fb09b7a0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -152,6 +152,7 @@ class SelectionDAGISel { OPC_RecordChild7, OPC_RecordMemRef, OPC_CaptureGlueInput, + OPC_CaptureDeactivationSymbol, OPC_MoveChild, OPC_MoveChild0, OPC_MoveChild1, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 2283f99202e2f..94a4262e9b986 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1928,6 +1928,23 @@ class GlobalAddressSDNode : public SDNode { } }; +class DeactivationSymbolSDNode : public SDNode { + friend class SelectionDAG; + + const GlobalValue *TheGlobal; + + DeactivationSymbolSDNode(const GlobalValue *GV, SDVTList VTs) + : SDNode(ISD::DEACTIVATION_SYMBOL, 0, DebugLoc(), VTs), + TheGlobal(GV) {} + +public: + const GlobalValue *getGlobal() const { return TheGlobal; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::DEACTIVATION_SYMBOL; + } +}; + class FrameIndexSDNode : public SDNode { friend class SelectionDAG; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 58ac87206b9a6..35cbeea57566b 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4612,6 +4612,7 @@ class TargetLowering : public TargetLoweringBase { SmallVector InVals; const ConstantInt *CFIType = nullptr; SDValue ConvergenceControlToken; + GlobalValue *DeactivationSymbol = nullptr; std::optional PAI; @@ -4757,6 +4758,11 @@ class TargetLowering : public TargetLoweringBase { return *this; } + CallLoweringInfo &setDeactivationSymbol(GlobalValue *Sym) { + DeactivationSymbol = Sym; + return *this; + } + ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index bbd125fd38cf1..6acc55ee55289 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -96,6 +96,7 @@ class LLVMContext { OB_ptrauth = 7, // "ptrauth" OB_kcfi = 8, // "kcfi" OB_convergencectrl = 9, // "convergencectrl" + OB_deactivation_symbol = 10, // "deactivation-symbol" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index e8b460aaf803b..3d8d20c6b2de7 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -682,6 +682,7 @@ class Instruction : InstructionEncoding { // If so, make sure to override // TargetInstrInfo::getInsertSubregLikeInputs. bit variadicOpsAreDefs = false; // Are variadic operands definitions? + bit supportsDeactivationSymbol = false; // Does the instruction have side effects that are not captured by any // operands of the instruction or other flags? diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 0af70f333f864..af86219c495dd 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -195,6 +195,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + Info.DeactivationSymbol = cast(Bundle->Inputs[0]); + } + Info.CB = &CB; Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); Info.CallConv = CallConv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index f8afb42bf5535..ba5ad836263c6 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2861,6 +2861,9 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { } } + if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_deactivation_symbol)) + MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get()); + return true; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 359677027f52f..5592430d1ea73 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -38,8 +38,10 @@ void MachineIRBuilder::setMF(MachineFunction &MF) { //------------------------------------------------------------------------------ MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) { - return BuildMI(getMF(), {getDL(), getPCSections(), getMMRAMetadata()}, - getTII().get(Opcode)); + return BuildMI( + getMF(), + {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()}, + getTII().get(Opcode)); } MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 2409e601ceb46..8eca107385b84 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -120,7 +120,7 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &TID, MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) : MCID(&MI.getDesc()), NumOperands(0), Flags(0), AsmPrinterFlags(0), Info(MI.Info), DbgLoc(MI.getDebugLoc()), DebugInstrNum(0), - Opcode(MI.getOpcode()) { + Opcode(MI.getOpcode()), DeactivationSymbol(MI.getDeactivationSymbol()) { assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor"); CapOperands = OperandCapacity::get(MI.getNumOperands()); @@ -728,6 +728,8 @@ bool MachineInstr::isIdenticalTo(const MachineInstr &Other, // Call instructions with different CFI types are not identical. if (isCall() && getCFIType() != Other.getCFIType()) return false; + if (getDeactivationSymbol() != Other.getDeactivationSymbol()) + return false; return true; } @@ -2009,6 +2011,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << ','; OS << " cfi-type " << CFIType; } + if (getDeactivationSymbol()) + OS << ", deactivation-symbol " << getDeactivationSymbol()->getName(); if (DebugInstrNum) { if (!FirstOp) diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 5182e4124f548..8528543211223 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -15,10 +15,12 @@ #include "InstrEmitter.h" #include "SDNodeDbgValue.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -61,6 +63,8 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses, unsigned N = Node->getNumOperands(); while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) --N; + if (N && Node->getOperand(N - 1).getOpcode() == ISD::DEACTIVATION_SYMBOL) + --N; // Ignore deactivation symbol if it exists. if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) --N; // Ignore chain if it exists. @@ -1219,15 +1223,23 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } - if (SDNode *GluedNode = Node->getGluedNode()) { - // FIXME: Possibly iterate over multiple glue nodes? - if (GluedNode->getOpcode() == - ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { - Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); - MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, - /*isImp=*/true); - MIB->addOperand(MO); - } + unsigned Op = Node->getNumOperands(); + if (Op != 0 && Node->getOperand(Op - 1)->getOpcode() == + ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { + Register VReg = getVR(Node->getOperand(Op - 1)->getOperand(0), VRBaseMap); + MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, + /*isImp=*/true); + MIB->addOperand(MO); + Op--; + } + + if (Op != 0 && + Node->getOperand(Op - 1)->getOpcode() == ISD::DEACTIVATION_SYMBOL) { + MI->setDeactivationSymbol( + *MF, const_cast( + cast(Node->getOperand(Op - 1)) + ->getGlobal())); + Op--; } // Run post-isel target hook to adjust this instruction if needed. @@ -1248,7 +1260,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, llvm_unreachable("This target-independent node should have been selected!"); case ISD::EntryToken: case ISD::MERGE_VALUES: - case ISD::TokenFactor: // fall thru + case ISD::TokenFactor: + case ISD::DEACTIVATION_SYMBOL: break; case ISD::CopyToReg: { Register DestReg = cast(Node->getOperand(1))->getReg(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 7ce4eebf685e1..4981272c69d71 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1913,6 +1913,21 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, return SDValue(N, 0); } +SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) { + SDVTList VTs = getVTList(MVT::Untyped); + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::DEACTIVATION_SYMBOL, VTs, {}); + ID.AddPointer(GV); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) + return SDValue(E, 0); + + auto *N = newSDNode(GV, VTs); + CSEMap.InsertNode(N, IP); + InsertNode(N); + return SDValue(N, 0); +} + SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; SDVTList VTs = getVTList(VT); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 6db2a5ffbfb84..727bec637bc79 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -45,6 +45,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" @@ -5280,6 +5281,13 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Create the node. SDValue Result; + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + auto *Sym = Bundle->Inputs[0].get(); + SDValue SDSym = getValue(Sym); + SDSym = DAG.getDeactivationSymbol(cast(Sym)); + Ops.push_back(SDSym); + } + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { auto *Token = Bundle->Inputs[0].get(); SDValue ConvControlToken = getValue(Token); @@ -8928,6 +8936,11 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, ConvControlToken = getValue(Token); } + GlobalValue *DeactivationSymbol = nullptr; + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + DeactivationSymbol = cast(Bundle->Inputs[0].get()); + } + TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -8937,7 +8950,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) .setCFIType(CFIType) - .setConvergenceControlToken(ConvControlToken); + .setConvergenceControlToken(ConvControlToken) + .setDeactivationSymbol(DeactivationSymbol); // Set the pointer authentication info if we have it. if (PAI) { @@ -9554,7 +9568,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && "Cannot lower calls with arbitrary operand bundles!"); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 63ee2d78cfa1b..45d9863835e10 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3258,6 +3258,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::LIFETIME_START: case ISD::LIFETIME_END: case ISD::PSEUDO_PROBE: + case ISD::DEACTIVATION_SYMBOL: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: @@ -3334,7 +3335,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // These are the current input chain and glue for use when generating nodes. // Various Emit operations change these. For example, emitting a copytoreg // uses and updates these. - SDValue InputChain, InputGlue; + SDValue InputChain, InputGlue, DeactivationSymbol; // ChainNodesMatched - If a pattern matches nodes that have input/output // chains, the OPC_EmitMergeInputChains operation is emitted which indicates @@ -3487,6 +3488,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, InputGlue = N->getOperand(N->getNumOperands()-1); continue; + case OPC_CaptureDeactivationSymbol: + // If the current node has a deactivation symbol, capture it in + // DeactivationSymbol. + if (N->getNumOperands() != 0 && + N->getOperand(N->getNumOperands() - 1).getOpcode() == + ISD::DEACTIVATION_SYMBOL) + DeactivationSymbol = N->getOperand(N->getNumOperands()-1); + continue; + case OPC_MoveChild: { unsigned ChildNo = MatcherTable[MatcherIndex++]; if (ChildNo >= N.getNumOperands()) @@ -4169,6 +4179,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // If this has chain/glue inputs, add them. if (EmitNodeInfo & OPFL_Chain) Ops.push_back(InputChain); + if (DeactivationSymbol.getNode() != nullptr) + Ops.push_back(DeactivationSymbol); if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) Ops.push_back(InputGlue); diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 447e5d92e0b99..9946fe6b9af13 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -53,6 +53,8 @@ static StringRef knownBundleName(unsigned BundleTagID) { return "kcfi"; case LLVMContext::OB_convergencectrl: return "convergencectrl"; + case LLVMContext::OB_deactivation_symbol: + return "deactivation-symbol"; default: llvm_unreachable("unknown bundle id"); } @@ -76,7 +78,7 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { } for (unsigned BundleTagID = LLVMContext::OB_deopt; - BundleTagID <= LLVMContext::OB_convergencectrl; ++BundleTagID) { + BundleTagID <= LLVMContext::OB_deactivation_symbol; ++BundleTagID) { [[maybe_unused]] const auto *Entry = pImpl->getOrInsertBundleTag(knownBundleName(BundleTagID)); assert(Entry->second == BundleTagID && "operand bundle id drifted!"); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index cbff94f4dc227..49bb04daef26f 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -2896,6 +2896,23 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { OutStreamer->emitLabel(LOHLabel); } + if (MI->getDeactivationSymbol()) { + if (isa(MI->getDeactivationSymbol())) { + // Just emit the nop directly. + EmitToStreamer(MCInstBuilder(AArch64::HINT).addImm(0)); + return; + } + MCSymbol *Dot = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Dot); + const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Dot, OutContext); + + MCSymbol *DS = + OutContext.getOrCreateSymbol(MI->getDeactivationSymbol()->getName()); + const MCExpr *DSExpr = MCSymbolRefExpr::create(DS, OutContext); + OutStreamer->emitRelocDirective(*DeactDotExpr, "R_AARCH64_INST32", DSExpr, + SMLoc(), *TM.getMCSubtargetInfo()); + } + AArch64TargetStreamer *TS = static_cast(OutStreamer->getTargetStreamer()); // Do any manual lowerings. diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 1c8e3afdfd718..793ab48b76903 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9614,6 +9614,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, if (InGlue.getNode()) Ops.push_back(InGlue); + if (CLI.DeactivationSymbol) + Ops.push_back(DAG.getDeactivationSymbol(CLI.DeactivationSymbol)); + // If we're doing a tall call, use a TC_RETURN here rather than an // actual call instruction. if (IsTailCall) { diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index e4719b26cab52..7c99eb002876f 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -1367,6 +1367,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, } else if (Info.CFIType) { MIB->setCFIType(MF, Info.CFIType->getZExtValue()); } + MIB->setDeactivationSymbol(MF, Info.DeactivationSymbol); MIB.add(Info.Callee); diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index a509edf160d32..e9dc50c6907d3 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -40,6 +40,8 @@ class AArch64ELFObjectWriter : public MCELFObjectTargetWriter { const MCFixup &Fixup, bool IsPCRel) const override; bool needsRelocateWithSymbol(const MCValue &Val, const MCSymbol &Sym, unsigned Type) const override; + void sortRelocs(const MCAssembler &Asm, + std::vector &Relocs) override; bool IsILP32; }; @@ -558,6 +560,17 @@ bool AArch64ELFObjectWriter::needsRelocateWithSymbol(const MCValue &Val, Val.getAccessVariant()); } +void AArch64ELFObjectWriter::sortRelocs( + const MCAssembler &Asm, std::vector &Relocs) { + // INST32 relocations should be applied last because they may overwrite the + // whole instruction and so should take precedence over other relocations that + // modify operands of the original instruction. + std::stable_partition(Relocs.begin(), Relocs.end(), + [](const ELFRelocationEntry &R) { + return R.Type != ELF::R_AARCH64_INST32; + }); +} + std::unique_ptr llvm::createAArch64ELFObjectWriter(uint8_t OSABI, bool IsILP32) { return std::make_unique(OSABI, IsILP32); diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 57997a6b0e4e0..9fc4022e55123 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -954,6 +954,13 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, } } const EmitNodeMatcherCommon *EN = cast(N); + bool SupportsDeactivationSymbol = + EN->getInstruction().TheDef->getValueAsBit( + "supportsDeactivationSymbol"); + if (SupportsDeactivationSymbol) { + OS << "OPC_CaptureDeactivationSymbol,\n"; + OS.indent(FullIndexWidth + Indent); + } bool IsEmitNode = isa(EN); OS << (IsEmitNode ? "OPC_EmitNode" : "OPC_MorphNodeTo"); bool CompressVTs = EN->getNumVTs() < 3; @@ -1045,8 +1052,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, } else OS << '\n'; - return 4 + !CompressVTs + !CompressNodeInfo + NumTypeBytes + - NumOperandBytes + NumCoveredBytes; + return 4 + SupportsDeactivationSymbol + !CompressVTs + !CompressNodeInfo + + NumTypeBytes + NumOperandBytes + NumCoveredBytes; } case Matcher::CompleteMatch: { const CompleteMatchMatcher *CM = cast(N); From 4a1a56b732481a3cf7fa5945688a108f1421f380 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 27 Mar 2025 17:09:14 -0700 Subject: [PATCH 11/18] Add deactivation symbol operand to ConstantPtrAuth. Deactivation symbol operands are supported in the code generator by building on the previously added support for IRELATIVE relocations. TODO: - Fix broken test. - Add bitcode and IR writer support. - Add tests. Pull Request: https://github.com/llvm/llvm-project/pull/133537 --- clang/lib/CodeGen/CGPointerAuth.cpp | 6 +-- llvm/include/llvm/Bitcode/LLVMBitCodes.h | 1 + llvm/include/llvm/IR/Constants.h | 13 +++++-- llvm/include/llvm/SandboxIR/Constant.h | 5 ++- llvm/lib/AsmParser/LLParser.cpp | 29 +++++++++++---- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 18 ++++++++- llvm/lib/IR/AsmWriter.cpp | 4 +- llvm/lib/IR/Constants.cpp | 12 ++++-- llvm/lib/IR/ConstantsContext.h | 3 +- llvm/lib/IR/Core.cpp | 4 +- llvm/lib/SandboxIR/Constant.cpp | 11 +++++- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 37 ++++++++++++++++--- .../InstCombine/InstCombineCalls.cpp | 4 +- llvm/lib/Transforms/Utils/ValueMapper.cpp | 5 ++- llvm/unittests/SandboxIR/SandboxIRTest.cpp | 2 +- .../Transforms/Utils/ValueMapperTest.cpp | 13 +++++-- 16 files changed, 126 insertions(+), 41 deletions(-) diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp index 4b032306ead72..2d72fef470af6 100644 --- a/clang/lib/CodeGen/CGPointerAuth.cpp +++ b/clang/lib/CodeGen/CGPointerAuth.cpp @@ -308,9 +308,9 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0); } - return llvm::ConstantPtrAuth::get(Pointer, - llvm::ConstantInt::get(Int32Ty, Key), - IntegerDiscriminator, AddressDiscriminator); + return llvm::ConstantPtrAuth::get( + Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator, + AddressDiscriminator, llvm::Constant::getNullValue(UnqualPtrTy)); } /// Does a given PointerAuthScheme require us to sign a value diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index ec2535ac85966..13521ba6cd00f 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -431,6 +431,7 @@ enum ConstantsCodes { CST_CODE_CE_GEP_WITH_INRANGE = 31, // [opty, flags, range, n x operands] CST_CODE_CE_GEP = 32, // [opty, flags, n x operands] CST_CODE_PTRAUTH = 33, // [ptr, key, disc, addrdisc] + CST_CODE_PTRAUTH2 = 34, // [ptr, key, disc, addrdisc, DeactivationSymbol] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index a50217078d0ed..45d5352bf06a6 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1022,10 +1022,10 @@ class ConstantPtrAuth final : public Constant { friend struct ConstantPtrAuthKeyType; friend class Constant; - constexpr static IntrusiveOperandsAllocMarker AllocMarker{4}; + constexpr static IntrusiveOperandsAllocMarker AllocMarker{5}; ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, - Constant *AddrDisc); + Constant *AddrDisc, Constant *DeactivationSymbol); void *operator new(size_t s) { return User::operator new(s, AllocMarker); } @@ -1035,7 +1035,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// Produce a new ptrauth expression signing the given value using /// the same schema as is stored in one. @@ -1067,6 +1068,10 @@ class ConstantPtrAuth final : public Constant { return !getAddrDiscriminator()->isNullValue(); } + Constant *getDeactivationSymbol() const { + return cast(Op<4>().get()); + } + /// A constant value for the address discriminator which has special /// significance to ctors/dtors lowering. Regular address discrimination can't /// be applied for them since uses of llvm.global_{c|d}tors are disallowed @@ -1094,7 +1099,7 @@ class ConstantPtrAuth final : public Constant { template <> struct OperandTraits - : public FixedNumOperandTraits {}; + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPtrAuth, Constant) diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h index 17f55e973cd76..5243a9476ac64 100644 --- a/llvm/include/llvm/SandboxIR/Constant.h +++ b/llvm/include/llvm/SandboxIR/Constant.h @@ -1096,7 +1096,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// The pointer that is signed in this ptrauth signed pointer. Constant *getPointer() const; @@ -1111,6 +1112,8 @@ class ConstantPtrAuth final : public Constant { /// the only global-initializer user of the ptrauth signed pointer. Constant *getAddrDiscriminator() const; + Constant *getDeactivationSymbol() const; + /// Whether there is any non-null address discriminator. bool hasAddressDiscriminator() const { return cast(Val)->hasAddressDiscriminator(); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 960119bab0933..dfa014aa0bd7d 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -4226,11 +4226,12 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { } case lltok::kw_ptrauth: { // ValID ::= 'ptrauth' '(' ptr @foo ',' i32 - // (',' i64 (',' ptr addrdisc)? )? ')' + // (',' i64 (',' ptr addrdisc (',' ptr ds)? )? )? ')' Lex.Lex(); Constant *Ptr, *Key; - Constant *Disc = nullptr, *AddrDisc = nullptr; + Constant *Disc = nullptr, *AddrDisc = nullptr, + *DeactivationSymbol = nullptr; if (parseToken(lltok::lparen, "expected '(' in constant ptrauth expression") || @@ -4239,11 +4240,14 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { "expected comma in constant ptrauth expression") || parseGlobalTypeAndValue(Key)) return true; - // If present, parse the optional disc/addrdisc. - if (EatIfPresent(lltok::comma)) - if (parseGlobalTypeAndValue(Disc) || - (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc))) - return true; + // If present, parse the optional disc/addrdisc/ds. + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(Disc)) + return true; + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc)) + return true; + if (EatIfPresent(lltok::comma) && + parseGlobalTypeAndValue(DeactivationSymbol)) + return true; if (parseToken(lltok::rparen, "expected ')' in constant ptrauth expression")) return true; @@ -4274,7 +4278,16 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { AddrDisc = ConstantPointerNull::get(PointerType::get(Context, 0)); } - ID.ConstantVal = ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc); + if (DeactivationSymbol) { + if (!DeactivationSymbol->getType()->isPointerTy()) + return error( + ID.Loc, "constant ptrauth deactivation symbol must be a pointer"); + } else { + DeactivationSymbol = ConstantPointerNull::get(PointerType::get(Context, 0)); + } + + ID.ConstantVal = + ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc, DeactivationSymbol); ID.Kind = ValID::t_Constant; return false; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 40e755902b724..c09c3b4f7d38c 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1611,7 +1611,13 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, if (!Disc) return error("ptrauth disc operand must be ConstantInt"); - C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3]); + auto *DeactivationSymbol = + ConstOps.size() > 4 ? ConstOps[4] + : ConstantPointerNull::get(cast( + ConstOps[3]->getType())); + + C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3], + DeactivationSymbol); break; } case BitcodeConstant::NoCFIOpcode: { @@ -3811,6 +3817,16 @@ Error BitcodeReader::parseConstants() { (unsigned)Record[2], (unsigned)Record[3]}); break; } + case bitc::CST_CODE_PTRAUTH2: { + if (Record.size() < 4) + return error("Invalid ptrauth record"); + // Ptr, Key, Disc, AddrDisc, DeactivationSymbol + V = BitcodeConstant::create( + Alloc, CurTy, BitcodeConstant::ConstantPtrAuthOpcode, + {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2], + (unsigned)Record[3], (unsigned)Record[4]}); + break; + } } assert(V->getType() == getTypeByID(CurTyID) && "Incorrect result type ID"); diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 79547b299a903..5efb321967008 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1630,12 +1630,14 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, if (const ConstantPtrAuth *CPA = dyn_cast(CV)) { Out << "ptrauth ("; - // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?) + // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?) unsigned NumOpsToWrite = 2; if (!CPA->getOperand(2)->isNullValue()) NumOpsToWrite = 3; if (!CPA->getOperand(3)->isNullValue()) NumOpsToWrite = 4; + if (!CPA->getOperand(4)->isNullValue()) + NumOpsToWrite = 5; ListSeparator LS; for (unsigned i = 0, e = NumOpsToWrite; i != e; ++i) { diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index fb659450bfeeb..007d36d19f373 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -2072,19 +2072,22 @@ Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { // ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { - Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc}; + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { + Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc, DeactivationSymbol}; ConstantPtrAuthKeyType MapKey(ArgVec); LLVMContextImpl *pImpl = Ptr->getContext().pImpl; return pImpl->ConstantPtrAuths.getOrCreate(Ptr->getType(), MapKey); } ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { - return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator()); + return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator(), + getDeactivationSymbol()); } ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) : Constant(Ptr->getType(), Value::ConstantPtrAuthVal, AllocMarker) { assert(Ptr->getType()->isPointerTy()); assert(Key->getBitWidth() == 32); @@ -2094,6 +2097,7 @@ ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, setOperand(1, Key); setOperand(2, Disc); setOperand(3, AddrDisc); + setOperand(4, DeactivationSymbol); } /// Remove the constant from the constant table. diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index e5c9622e09927..bf9d8ab952271 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -545,7 +545,8 @@ struct ConstantPtrAuthKeyType { ConstantPtrAuth *create(TypeClass *Ty) const { return new ConstantPtrAuth(Operands[0], cast(Operands[1]), - cast(Operands[2]), Operands[3]); + cast(Operands[2]), Operands[3], + Operands[4]); } }; diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index f4b03e8cb8aa3..6190ebdac16d4 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -1687,7 +1687,9 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key, LLVMValueRef Disc, LLVMValueRef AddrDisc) { return wrap(ConstantPtrAuth::get( unwrap(Ptr), unwrap(Key), - unwrap(Disc), unwrap(AddrDisc))); + unwrap(Disc), unwrap(AddrDisc), + ConstantPointerNull::get( + cast(unwrap(AddrDisc)->getType())))); } /*-- Opcode mapping */ diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp index 3e13c935c4281..0a28cf9feeb4d 100644 --- a/llvm/lib/SandboxIR/Constant.cpp +++ b/llvm/lib/SandboxIR/Constant.cpp @@ -421,10 +421,12 @@ PointerType *NoCFIValue::getType() const { } ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { auto *LLVMC = llvm::ConstantPtrAuth::get( cast(Ptr->Val), cast(Key->Val), - cast(Disc->Val), cast(AddrDisc->Val)); + cast(Disc->Val), cast(AddrDisc->Val), + cast(DeactivationSymbol->Val)); return cast(Ptr->getContext().getOrCreateConstant(LLVMC)); } @@ -448,6 +450,11 @@ Constant *ConstantPtrAuth::getAddrDiscriminator() const { cast(Val)->getAddrDiscriminator()); } +Constant *ConstantPtrAuth::getDeactivationSymbol() const { + return Ctx.getOrCreateConstant( + cast(Val)->getDeactivationSymbol()); +} + ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { auto *LLVMC = cast(Val)->getWithSameSchema( cast(Pointer->Val)); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 49bb04daef26f..5ce64c9a3a091 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -195,7 +195,7 @@ class AArch64AsmPrinter : public AsmPrinter { const MCExpr *emitPAuthRelocationAsIRelative( const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, - bool HasAddressDiversity, bool IsDSOLocal); + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr); /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. @@ -2270,15 +2270,17 @@ static void emitAddress(MCStreamer &Streamer, MCRegister Reg, } static bool targetSupportsPAuthRelocation(const Triple &TT, - const MCExpr *Target) { + const MCExpr *Target, + const MCExpr *DSExpr) { // No released version of glibc supports PAuth relocations. if (TT.isOSGlibc()) return false; // We emit PAuth constants as IRELATIVE relocations in cases where the // constant cannot be represented as a PAuth relocation: - // 1) The signed value is not a symbol. - return !isa(Target); + // 1) There is a deactivation symbol. + // 2) The signed value is not a symbol. + return !DSExpr && !isa(Target); } static bool targetSupportsIRelativeRelocation(const Triple &TT) { @@ -2295,7 +2297,7 @@ static bool targetSupportsIRelativeRelocation(const Triple &TT) { const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, - bool HasAddressDiversity, bool IsDSOLocal) { + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) { const Triple &TT = TM.getTargetTriple(); // We only emit an IRELATIVE relocation if the target supports IRELATIVE and @@ -2358,6 +2360,18 @@ const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext()); OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef), *STI); + + if (DSExpr) { + auto *PrePACInstExpr = + MCSymbolRefExpr::create(PrePACInst, OutStreamer->getContext()); + OutStreamer->emitRelocDirective(*PrePACInstExpr, "R_AARCH64_INST32", DSExpr, + SMLoc(), *STI); + } + + // We need a RET despite the above tail call because the deactivation symbol + // may replace it with a NOP. + OutStreamer->emitInstruction(MCInstBuilder(AArch64::RET).addReg(AArch64::LR), + *STI); OutStreamer->popSection(); return MCSymbolRefExpr::create(IFuncSym, OutStreamer->getContext()); @@ -2388,6 +2402,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx); } + const MCExpr *DSExpr = nullptr; + if (auto *DS = dyn_cast(CPA.getDeactivationSymbol())) { + if (isa(DS)) + return Sym; + DSExpr = MCSymbolRefExpr::create(getSymbol(DS), Ctx); + } + uint64_t KeyID = CPA.getKey()->getZExtValue(); // We later rely on valid KeyID value in AArch64PACKeyIDToString call from // AArch64AuthMCExpr::printImpl, so fail fast. @@ -2404,9 +2425,13 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { // Check if we need to represent this with an IRELATIVE and emit it if so. if (auto *IFuncSym = emitPAuthRelocationAsIRelative( Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), - BaseGVB && BaseGVB->isDSOLocal())) + BaseGVB && BaseGVB->isDSOLocal(), DSExpr)) return IFuncSym; + if (DSExpr) + report_fatal_error("deactivation symbols unsupported in constant " + "expressions on this target"); + // Finally build the complete @AUTH expr. return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), Ctx); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 12dd4cec85f59..58b98d8d93464 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2946,9 +2946,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (NeedSign && isa(II->getArgOperand(4))) { auto *SignKey = cast(II->getArgOperand(3)); auto *SignDisc = cast(II->getArgOperand(4)); - auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy()); + auto *Null = ConstantPointerNull::get(Builder.getPtrTy()); auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey, - SignDisc, SignAddrDisc); + SignDisc, Null, Null); replaceInstUsesWith( *II, ConstantExpr::getPointerCast(NewCPA, II->getType())); return eraseInstFromFunction(*II); diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 5e50536a99206..320bef6c8f240 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -526,8 +526,9 @@ Value *Mapper::mapValue(const Value *V) { if (isa(C)) return getVM()[V] = ConstantVector::get(Ops); if (isa(C)) - return getVM()[V] = ConstantPtrAuth::get(Ops[0], cast(Ops[1]), - cast(Ops[2]), Ops[3]); + return getVM()[V] = + ConstantPtrAuth::get(Ops[0], cast(Ops[1]), + cast(Ops[2]), Ops[3], Ops[4]); // If this is a no-operand constant, it must be because the type was remapped. if (isa(C)) return getVM()[V] = PoisonValue::get(NewTy); diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp index 088264e0429fd..bb1646bc34545 100644 --- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp +++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp @@ -1169,7 +1169,7 @@ define ptr @foo() { // Check get(), getKey(), getDiscriminator(), getAddrDiscriminator(). auto *NewPtrAuth = sandboxir::ConstantPtrAuth::get( &F, PtrAuth->getKey(), PtrAuth->getDiscriminator(), - PtrAuth->getAddrDiscriminator()); + PtrAuth->getAddrDiscriminator(), PtrAuth->getDeactivationSymbol()); EXPECT_EQ(NewPtrAuth, PtrAuth); // Check hasAddressDiscriminator(). EXPECT_EQ(PtrAuth->hasAddressDiscriminator(), diff --git a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp index 86ad41fa7ad50..90b58236060d2 100644 --- a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp +++ b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp @@ -450,6 +450,10 @@ TEST(ValueMapperTest, mapValuePtrAuth) { PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage0"); std::unique_ptr Storage1 = std::make_unique( PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage1"); + std::unique_ptr DS0 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS0"); + std::unique_ptr DS1 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS1"); ConstantInt *ConstKey = ConstantInt::get(Int32Ty, 1); ConstantInt *ConstDisc = ConstantInt::get(Int64Ty, 1234); @@ -457,11 +461,12 @@ TEST(ValueMapperTest, mapValuePtrAuth) { ValueToValueMapTy VM; VM[Var0.get()] = Var1.get(); VM[Storage0.get()] = Storage1.get(); + VM[DS0.get()] = DS1.get(); - ConstantPtrAuth *Value = - ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, Storage0.get()); - ConstantPtrAuth *MappedValue = - ConstantPtrAuth::get(Var1.get(), ConstKey, ConstDisc, Storage1.get()); + ConstantPtrAuth *Value = ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, + Storage0.get(), DS0.get()); + ConstantPtrAuth *MappedValue = ConstantPtrAuth::get( + Var1.get(), ConstKey, ConstDisc, Storage1.get(), DS1.get()); EXPECT_EQ(ValueMapper(VM).mapValue(*Value), MappedValue); } From 52a1c573610329f7de90c4fffd6728427a3d19c5 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 27 Mar 2025 17:09:08 -0700 Subject: [PATCH 12/18] Add pointer field protection feature. Pointer field protection is a use-after-free vulnerability mitigation that works by changing how data structures' pointer fields are stored in memory. For more information, see the RFC: https://discourse.llvm.org/t/rfc-structure-protection-a-family-of-uaf-mitigation-techniques/85555 TODO: - Fix test failure. - Add more tests. - Add documentation. Pull Request: https://github.com/llvm/llvm-project/pull/133538 --- clang/include/clang/AST/ASTContext.h | 22 ++ clang/include/clang/Basic/Attr.td | 6 + clang/include/clang/Basic/Features.def | 3 + clang/include/clang/Basic/LangOptions.def | 3 + clang/include/clang/Basic/LangOptions.h | 11 + clang/include/clang/Basic/TokenKinds.def | 1 + clang/include/clang/Driver/Options.td | 6 + clang/lib/AST/ASTContext.cpp | 95 +++++++ clang/lib/AST/ExprConstant.cpp | 1 + clang/lib/AST/Type.cpp | 4 +- clang/lib/AST/TypePrinter.cpp | 3 + clang/lib/CodeGen/CGCall.cpp | 114 +++++++- clang/lib/CodeGen/CGClass.cpp | 52 +++- clang/lib/CodeGen/CGExpr.cpp | 8 +- clang/lib/CodeGen/CGExprAgg.cpp | 2 +- clang/lib/CodeGen/CGExprCXX.cpp | 10 + clang/lib/CodeGen/CGExprConstant.cpp | 39 ++- clang/lib/CodeGen/CodeGenFunction.cpp | 44 ++- clang/lib/CodeGen/CodeGenFunction.h | 8 +- clang/lib/CodeGen/CodeGenModule.cpp | 39 +++ clang/lib/CodeGen/CodeGenModule.h | 7 + clang/lib/CodeGen/ItaniumCXXABI.cpp | 1 + clang/lib/CodeGen/MicrosoftCXXABI.cpp | 1 + clang/lib/Driver/ToolChains/Clang.cpp | 4 + clang/lib/Sema/SemaDeclAttr.cpp | 8 + clang/lib/Sema/SemaExprCXX.cpp | 5 + clang/test/CodeGen/pfp-attribute-disable.cpp | 33 +++ clang/test/CodeGen/pfp-load-store.cpp | 40 +++ clang/test/CodeGen/pfp-memcpy.cpp | 19 ++ clang/test/CodeGen/pfp-null-init.cpp | 16 ++ clang/test/CodeGen/pfp-struct-gep.cpp | 25 ++ clang/test/CodeGenCXX/trivial_abi.cpp | 4 +- libcxx/include/__config | 23 ++ libcxx/include/__functional/function.h | 2 +- libcxx/include/__memory/shared_ptr.h | 4 +- libcxx/include/__memory/unique_ptr.h | 4 +- libcxx/include/__tree | 2 +- .../__type_traits/is_trivially_relocatable.h | 8 +- libcxx/include/__vector/vector.h | 2 +- libcxx/include/typeinfo | 2 +- .../libcxx/gdb/gdb_pretty_printer_test.sh.cpp | 9 + libcxxabi/include/__cxxabi_config.h | 10 + libcxxabi/src/private_typeinfo.h | 6 +- llvm/include/llvm/Analysis/PtrUseVisitor.h | 15 ++ llvm/include/llvm/IR/Intrinsics.td | 23 ++ llvm/include/llvm/Transforms/Utils/Local.h | 2 + llvm/lib/Analysis/PtrUseVisitor.cpp | 3 +- llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 255 ++++++++++++++++++ .../lib/Target/AArch64/AArch64InstrFormats.td | 3 + .../Transforms/InstCombine/InstCombinePHI.cpp | 6 +- llvm/lib/Transforms/Scalar/SROA.cpp | 40 ++- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 25 +- 52 files changed, 1023 insertions(+), 55 deletions(-) create mode 100644 clang/test/CodeGen/pfp-attribute-disable.cpp create mode 100644 clang/test/CodeGen/pfp-load-store.cpp create mode 100644 clang/test/CodeGen/pfp-memcpy.cpp create mode 100644 clang/test/CodeGen/pfp-null-init.cpp create mode 100644 clang/test/CodeGen/pfp-struct-gep.cpp diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index af8c49e99a7ce..abba83e1ff9c4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -183,6 +183,12 @@ struct TypeInfoChars { } }; +struct PFPField { + CharUnits structOffset; + CharUnits offset; + FieldDecl *field; +}; + /// Holds long-lived AST nodes (such as types and decls) that can be /// referred to throughout the semantic analysis of a file. class ASTContext : public RefCountedBase { @@ -3618,6 +3624,22 @@ OPT_LIST(V) StringRef getCUIDHash() const; + bool isPFPStruct(const RecordDecl *rec) const; + void findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, bool IncludeVBases) const; + bool hasPFPFields(QualType ty) const; + bool isPFPField(const FieldDecl *field) const; + + /// Returns whether this record's PFP fields (if any) are trivially + /// relocatable (i.e. may be memcpy'd). This may also return true if the + /// record does not have any PFP fields, so it may be necessary for the caller + /// to check for PFP fields, e.g. by calling hasPFPFields(). + bool arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const; + + llvm::SetVector PFPFieldsWithEvaluatedOffset; + void recordMemberDataPointerEvaluation(const ValueDecl *VD); + void recordOffsetOfEvaluation(const OffsetOfExpr *E); + private: /// All OMPTraitInfo objects live in this collection, one per /// `pragma omp [begin] declare variant` directive. diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 0999d8065e9f5..3d26c2f001812 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2460,6 +2460,12 @@ def CountedByOrNull : DeclOrTypeAttr { let LangOpts = [COnly]; } +def NoPointerFieldProtection : DeclOrTypeAttr { + let Spellings = [Clang<"no_field_protection">]; + let Subjects = SubjectList<[Field], ErrorDiag>; + let Documentation = [Undocumented]; +} + def SizedBy : DeclOrTypeAttr { let Spellings = [Clang<"sized_by">]; let Subjects = SubjectList<[Field], ErrorDiag>; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 05ce214935fad..d4ded24c5a87e 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -262,6 +262,9 @@ FEATURE(shadow_call_stack, FEATURE(tls, PP.getTargetInfo().isTLSSupported()) FEATURE(underlying_type, LangOpts.CPlusPlus) FEATURE(experimental_library, LangOpts.ExperimentalLibrary) +FEATURE(pointer_field_protection, + LangOpts.getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) // C11 features supported by other languages as extensions. EXTENSION(c_alignas, true) diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 3879cc7942877..8eacb6e066007 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -501,6 +501,9 @@ LANGOPT(RelativeCXXABIVTables, 1, 0, LANGOPT(OmitVTableRTTI, 1, 0, "Use an ABI-incompatible v-table layout that omits the RTTI component") +ENUM_LANGOPT(PointerFieldProtection, PointerFieldProtectionKind, 2, PointerFieldProtectionKind::None, + "Encode struct pointer fields to protect against UAF vulnerabilities") + LANGOPT(VScaleMin, 32, 0, "Minimum vscale value") LANGOPT(VScaleMax, 32, 0, "Maximum vscale value") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index e925e0f3b5d85..797a14038ba4b 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -365,6 +365,17 @@ class LangOptionsBase { BKey }; + enum class PointerFieldProtectionKind { + /// Pointer field protection disabled + None, + /// Pointer field protection enabled, allocator does not tag heap + /// allocations. + Untagged, + /// Pointer field protection enabled, allocator is expected to tag heap + /// allocations. + Tagged, + }; + enum class ThreadModelKind { /// POSIX Threads. POSIX, diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 1bf9f43f80986..142e13b3ee784 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -542,6 +542,7 @@ TYPE_TRAIT_2(__is_pointer_interconvertible_base_of, IsPointerInterconvertibleBas // Clang-only C++ Type Traits TYPE_TRAIT_1(__is_trivially_relocatable, IsTriviallyRelocatable, KEYCXX) +TYPE_TRAIT_1(__has_non_relocatable_fields, HasNonRelocatableFields, KEYCXX) TYPE_TRAIT_1(__is_trivially_equality_comparable, IsTriviallyEqualityComparable, KEYCXX) TYPE_TRAIT_1(__is_bounded_array, IsBoundedArray, KEYCXX) TYPE_TRAIT_1(__is_unbounded_array, IsUnboundedArray, KEYCXX) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index a7fcb160d3867..c903e0554319e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2957,6 +2957,12 @@ defm experimental_omit_vtable_rtti : BoolFOption<"experimental-omit-vtable-rtti" NegFlag, BothFlags<[], [CC1Option], " the RTTI component from virtual tables">>; +def experimental_pointer_field_protection_EQ : Joined<["-"], "fexperimental-pointer-field-protection=">, Group, + Visibility<[ClangOption, CC1Option]>, + Values<"none,untagged,tagged">, NormalizedValuesScope<"LangOptions::PointerFieldProtectionKind">, + NormalizedValues<["None", "Untagged", "Tagged"]>, + MarshallingInfoEnum, "None">; + def fcxx_abi_EQ : Joined<["-"], "fc++-abi=">, Group, Visibility<[ClangOption, CC1Option]>, HelpText<"C++ ABI to use. This will override the target C++ ABI.">; diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index c9d1bea4c623a..c3cbfec2c93d3 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -79,6 +79,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/Support/Capacity.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MD5.h" @@ -14948,3 +14949,97 @@ bool ASTContext::useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl, ThunksToBeAbbreviated[VirtualMethodDecl] = std::move(SimplifiedThunkNames); return Result; } + +bool ASTContext::arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::Tagged) + return !isa(RD) || + cast(RD)->hasTrivialDestructor(); + return true; +} + +bool ASTContext::isPFPStruct(const RecordDecl *rec) const { + if (getLangOpts().getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) + if (auto *cxxRec = dyn_cast(rec)) + return !cxxRec->isStandardLayout(); + return false; +} + +void ASTContext::findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, + bool IncludeVBases) const { + if (auto *AT = getAsConstantArrayType(Ty)) { + if (auto *ElemDecl = AT->getElementType()->getAsCXXRecordDecl()) { + const ASTRecordLayout &ElemRL = getASTRecordLayout(ElemDecl); + for (unsigned i = 0; i != AT->getSize(); ++i) { + findPFPFields(AT->getElementType(), Offset + i * ElemRL.getSize(), + Fields, true); + } + } + } + auto *Decl = Ty->getAsCXXRecordDecl(); + if (!Decl) + return; + const ASTRecordLayout &RL = getASTRecordLayout(Decl); + for (FieldDecl *field : Decl->fields()) { + CharUnits fieldOffset = + Offset + toCharUnitsFromBits(RL.getFieldOffset(field->getFieldIndex())); + if (isPFPField(field)) + Fields.push_back({Offset, fieldOffset, field}); + findPFPFields(field->getType(), fieldOffset, Fields, true); + } + for (auto &Base : Decl->bases()) { + if (Base.isVirtual()) + continue; + CharUnits BaseOffset = + Offset + RL.getBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + if (IncludeVBases) { + for (auto &Base : Decl->vbases()) { + CharUnits BaseOffset = + Offset + RL.getVBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + } +} + +bool ASTContext::hasPFPFields(QualType ty) const { + std::vector pfpFields; + findPFPFields(ty, CharUnits::Zero(), pfpFields, true); + return !pfpFields.empty(); +} + +bool ASTContext::isPFPField(const FieldDecl *field) const { + if (!isPFPStruct(field->getParent())) + return false; + return field->getType()->isPointerType() && + !field->hasAttr(); +} + +void ASTContext::recordMemberDataPointerEvaluation(const ValueDecl *VD) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None) + return; + auto *FD = dyn_cast(VD); + if (!FD) + FD = cast(cast(VD)->chain().back()); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} + +void ASTContext::recordOffsetOfEvaluation(const OffsetOfExpr *E) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None || + E->getNumComponents() == 0) + return; + OffsetOfNode Comp = E->getComponent(E->getNumComponents() - 1); + if (Comp.getKind() != OffsetOfNode::Field) + return; + FieldDecl *FD = Comp.getField(); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 95da7b067b459..51f319fd26f20 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -14932,6 +14932,7 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( } bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) { + Info.Ctx.recordOffsetOfEvaluation(OOE); CharUnits Result; unsigned n = OOE->getNumComponents(); if (n == 0) diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 08798219c0b83..ca7ccc7f2bb5a 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -2852,7 +2852,9 @@ bool QualType::isTriviallyRelocatableType(const ASTContext &Context) const { } else if (!BaseElementType->isObjectType()) { return false; } else if (const auto *RD = BaseElementType->getAsRecordDecl()) { - return RD->canPassInRegisters(); + return RD->canPassInRegisters() && + (Context.arePFPFieldsTriviallyRelocatable(RD) || + !Context.hasPFPFields(BaseElementType)); } else if (BaseElementType.isTriviallyCopyableType(Context)) { return true; } else { diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index 3982ca3b50604..1382b4a9edfd4 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -2096,6 +2096,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T, case attr::ExtVectorType: OS << "ext_vector_type"; break; + case attr::NoPointerFieldProtection: + OS << "no_field_protection"; + break; } OS << "))"; } diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 7aa77e55dbfcc..9d824231d02cb 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1298,7 +1298,8 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. -static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, +static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy, + llvm::Type *Ty, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src.getElementType(); @@ -1306,6 +1307,57 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, if (SrcTy == Ty) return CGF.Builder.CreateLoad(Src); + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct in registers has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + CGF.getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto LoadCoercedField = [&](CharUnits Offset, + llvm::Type *FieldType) -> llvm::Value * { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *FieldVal = CGF.Builder.CreateLoad(fieldAddr); + if (isa(FieldType)) + FieldVal = CGF.Builder.CreatePtrToInt(FieldVal, FieldType); + PFPFields.erase(PFPFields.begin()); + return FieldVal; + } + auto FieldAddr = CGF.Builder + .CreateConstInBoundsByteGEP( + Src.withElementType(CGF.Int8Ty), Offset) + .withElementType(FieldType); + return CGF.Builder.CreateLoad(FieldAddr); + }; + if (isa(Ty) || isa(Ty)) { + auto Addr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *Val = CGF.Builder.CreateLoad(Addr); + if (isa(Ty)) + Val = CGF.Builder.CreatePtrToInt(Val, Ty); + return Val; + } + if (auto *AT = dyn_cast(Ty)) { + auto *ET = AT->getElementType(); + CharUnits wordSize = CGF.getContext().toCharUnitsFromBits( + CGF.CGM.getDataLayout().getTypeSizeInBits(ET)); + CharUnits Offset = CharUnits::Zero(); + llvm::Value *Val = llvm::UndefValue::get(AT); + for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += wordSize) + Val = CGF.Builder.CreateInsertValue(Val, LoadCoercedField(Offset, ET), i); + return Val; + } + auto *ST = cast(Ty); + llvm::Value *Val = llvm::UndefValue::get(ST); + auto *SL = CGF.CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + Val = CGF.Builder.CreateInsertValue( + Val, LoadCoercedField(Offset, ST->getElementType(i)), i); + } + return Val; + } + llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); if (llvm::StructType *SrcSTy = dyn_cast(SrcTy)) { @@ -1374,7 +1426,9 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, return CGF.Builder.CreateLoad(Tmp); } -void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, +void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, + QualType SrcFETy, + Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile) { if (!DstSize) @@ -1395,6 +1449,52 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, } } + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct passed by value has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = EmitAddressOfPFPField(Dst, PFPFields[0]); + if (isa(FieldVal->getType())) + FieldVal = Builder.CreateIntToPtr(FieldVal, VoidPtrTy); + Builder.CreateStore(FieldVal, fieldAddr); + PFPFields.erase(PFPFields.begin()); + } else { + auto fieldAddr = + Builder + .CreateConstInBoundsByteGEP(Dst.withElementType(Int8Ty), Offset) + .withElementType(FieldVal->getType()); + Builder.CreateStore(FieldVal, fieldAddr); + } + }; + + if (isa(SrcTy) || isa(SrcTy)) { + if (isa(SrcTy)) + Src = Builder.CreateIntToPtr(Src, VoidPtrTy); + auto Addr = EmitAddressOfPFPField(Dst, PFPFields[0]); + Builder.CreateStore(Src, Addr); + } else if (auto *at = dyn_cast(SrcTy)) { + auto *et = at->getElementType(); + CharUnits wordSize = getContext().toCharUnitsFromBits( + CGM.getDataLayout().getTypeSizeInBits(et)); + CharUnits Offset = CharUnits::Zero(); + for (unsigned i = 0; i != at->getNumElements(); ++i, Offset += wordSize) + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } else { + auto *ST = cast(SrcTy); + auto *SL = CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } + } + return; + } + if (SrcSize.isScalable() || SrcSize <= DstSize) { if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() && SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) { @@ -3347,7 +3447,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, auto AI = Fn->getArg(FirstIRArg); AI->setName(Arg->getName() + ".coerce"); CreateCoercedStore( - AI, Ptr, + AI, Ty, Ptr, llvm::TypeSize::getFixed( getContext().getTypeSizeInChars(Ty).getQuantity() - ArgI.getDirectOffset()), @@ -3969,7 +4069,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If the value is offset in memory, apply the offset now. Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); + RV = CreateCoercedLoad(V, RetTy, RetAI.getCoerceToType(), *this); } // In ARC, end functions that return a retainable type with a call @@ -4020,6 +4120,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, auto eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + RetTy, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -5550,7 +5651,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); llvm::Value *Load = - CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); + CreateCoercedLoad(Src, I->Ty, ArgInfo.getCoerceToType(), *this); if (CallInfo.isCmseNSCall()) { // For certain parameter types, clear padding bits, as they may reveal @@ -5611,6 +5712,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + I->Ty, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -6105,7 +6207,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the value is offset in memory, apply the offset now. Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); CreateCoercedStore( - CI, StorePtr, + CI, RetTy, StorePtr, llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()), DestIsVolatile); } diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 98c93b5bb4883..ae1d78baed16a 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -594,12 +594,20 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, isBaseVirtual); } -static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { +static bool isMemcpyEquivalentSpecialMember(CodeGenModule &CGM, + const CXXMethodDecl *D) { auto *CD = dyn_cast(D); if (!(CD && CD->isCopyOrMoveConstructor()) && !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(D->getParent()) && + CGM.getContext().hasPFPFields( + QualType(D->getParent()->getTypeForDecl(), 0))) + return false; + // We can emit a memcpy for a trivial copy or move constructor/assignment. if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) return true; @@ -664,7 +672,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); if (BaseElementTy.isPODType(CGF.getContext()) || - (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { + (CE && isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor()))) { unsigned SrcArgIndex = CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); llvm::Value *SrcPtr @@ -928,6 +936,11 @@ namespace { Qualifiers Qual = F->getType().getQualifiers(); if (Qual.hasVolatile() || Qual.hasObjCLifetime()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGF.getContext().arePFPFieldsTriviallyRelocatable(ClassDecl) && + CGF.getContext().isPFPField(F)) + return false; return true; } @@ -1064,7 +1077,8 @@ namespace { CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); // Bail out on non-memcpyable, not-trivially-copyable members. - if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && + if (!(CE && + isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor())) && !(FieldType.isTriviallyCopyableType(CGF.getContext()) || FieldType->isReferenceType())) return false; @@ -1174,7 +1188,7 @@ namespace { return nullptr; } else if (CXXMemberCallExpr *MCE = dyn_cast(S)) { CXXMethodDecl *MD = dyn_cast(MCE->getCalleeDecl()); - if (!(MD && isMemcpyEquivalentSpecialMember(MD))) + if (!(MD && isMemcpyEquivalentSpecialMember(CGF.CGM, MD))) return nullptr; MemberExpr *IOA = dyn_cast(MCE->getImplicitObjectArgument()); if (!IOA) @@ -2142,7 +2156,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // If this is a trivial constructor, emit a memcpy now before we lose // the alignment information on the argument. // FIXME: It would be better to preserve alignment information into CallArg. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); const Expr *Arg = E->getArg(0); @@ -2201,6 +2215,22 @@ void CodeGenFunction::EmitCXXConstructorCall( EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); + // When initializing an object that has pointer field protection and whose + // fields are not trivially relocatable we must initialize any pointer fields + // to a valid signed pointer (any pointer value will do, but we just use null + // pointers). This is because if the object is subsequently copied, its copy + // constructor will need to read and authenticate any pointer fields in order + // to copy the object to a new address, which will fail if the pointers are + // uninitialized. + if (!getContext().arePFPFieldsTriviallyRelocatable(D->getParent())) { + std::vector PFPFields; + getContext().findPFPFields(QualType(ClassDecl->getTypeForDecl(), 0), + CharUnits::Zero(), PFPFields, Type != Ctor_Base); + for (auto &Field : PFPFields) + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), + EmitAddressOfPFPField(This, Field)); + } + if (D->isTrivial() && D->isDefaultConstructor()) { assert(Args.size() == 1 && "trivial default ctor with args"); return; @@ -2209,7 +2239,7 @@ void CodeGenFunction::EmitCXXConstructorCall( // If this is a trivial constructor, just emit what's needed. If this is a // union copy constructor, we must emit a memcpy, because the AST does not // model that copy. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); Address Src = makeNaturalAddressForPointer( @@ -2975,7 +3005,15 @@ void CodeGenFunction::EmitForwardingCallToLambda( QualType resultType = FPT->getReturnType(); ReturnValueSlot returnSlot; if (!resultType->isVoidType() && - calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && + (calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect || + // With pointer field protection, we need to set up the return slot when + // returning an object with trivial ABI to avoid the memcpy that would + // otherwise be generated by the call to EmitReturnOfRValue() below, as + // that may corrupt the pointer signature. It doesn't hurt to do this all + // the time as it results in slightly simpler codegen. + (resultType->isRecordType() && + resultType->getAsCXXRecordDecl() + ->hasTrivialCopyConstructorForCall())) && !hasScalarEvaluationKind(calleeFnInfo->getReturnType())) returnSlot = ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(), diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 5943ff9294e1a..ce8875897f7d1 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -4904,10 +4904,16 @@ static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, return emitAddrOfZeroSizeField(CGF, base, field); const RecordDecl *rec = field->getParent(); - unsigned idx = CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); + if (CGF.getContext().isPFPField(field)) { + const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(rec); + auto Offset = CGF.getContext().toCharUnitsFromBits( + RL.getFieldOffset(field->getFieldIndex())); + return CGF.EmitAddressOfPFPField(base, field, Offset); + } + return CGF.Builder.CreateStructGEP(base, idx, field->getName()); } diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index c8bdda375d1b1..bdff2bceda93b 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -137,7 +137,7 @@ class AggExprEmitter : public StmtVisitor { if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) { CGF.CreateCoercedStore( - Result, Dest.getAddress(), + Result, E->getType(), Dest.getAddress(), llvm::TypeSize::getFixed( Dest.getPreferredSize(CGF.getContext(), E->getType()) .getQuantity()), diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 5d96959065dd9..896185907688f 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -261,6 +261,16 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) && !MD->getParent()->mayInsertExtraPadding(); + // Assignment operators for objects with non-trivially-relocatable PFP fields + // aren't trivial, we need to auth and sign the fields one by one. + if (TrivialAssignment && + !CGM.getContext().arePFPFieldsTriviallyRelocatable(MD->getParent()) && + CGM.getContext().hasPFPFields( + QualType(MD->getParent()->getTypeForDecl(), 0))) { + TrivialForCodegen = false; + TrivialAssignment = false; + } + // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment // operator before the LHS. CallArgList RtlArgStorage; diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index b016c6e36d1a8..9b6f822ebef16 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -906,6 +906,29 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, if (!EltInit) return false; + if (CGM.getContext().isPFPField(*Field)) { + llvm::ConstantInt *Disc; + llvm::Constant *AddrDisc; + if (CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) { + uint64_t FieldSignature = + std::hash()(CGM.getPFPFieldName(*Field)) & 0xffff; + Disc = llvm::ConstantInt::get(CGM.Int64Ty, FieldSignature); + AddrDisc = llvm::ConstantPointerNull::get(CGM.VoidPtrTy); + } else if (Emitter.isAbstract()) { + return false; + } else { + Disc = llvm::ConstantInt::get( + CGM.Int64Ty, (-(Layout.getFieldOffset(FieldNo) / 8)) & 0xffff); + AddrDisc = Emitter.getCurrentAddrPrivate(); + } + EltInit = llvm::ConstantPtrAuth::get( + EltInit, llvm::ConstantInt::get(CGM.Int32Ty, 2), Disc, AddrDisc, + CGM.getPFPDeactivationSymbol(*Field)); + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) + Emitter.registerCurrentAddrPrivate(EltInit, + cast(AddrDisc)); + } + if (ZeroInitPadding) { if (!DoZeroInitPadding(Layout, FieldNo, **Field, AllowOverwrite, SizeSoFar, ZeroFieldSize)) @@ -1656,7 +1679,20 @@ ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value, llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) { initializeNonAbstract(D.getType().getAddressSpace()); - return markIfFailed(tryEmitPrivateForVarInit(D)); + llvm::Constant *Init = tryEmitPrivateForVarInit(D); + + // If a placeholder address was needed for a TLS variable, implying that the + // initializer's value depends on its address, then the object may not be + // initialized in .tdata because the initializer will be memcpy'd to the + // thread's TLS. Instead the initialization must be done in code. + if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) { + for (auto &entry : PlaceholderAddresses) + entry.second->eraseFromParent(); + PlaceholderAddresses.clear(); + Init = nullptr; + } + + return markIfFailed(Init); } llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E, @@ -2586,6 +2622,7 @@ CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) { return getCXXABI().EmitMemberFunctionPointer(method); // Otherwise, a member data pointer. + getContext().recordMemberDataPointerEvaluation(decl); uint64_t fieldOffset = getContext().getFieldOffset(decl); CharUnits chars = getContext().toCharUnitsFromBits((int64_t) fieldOffset); return getCXXABI().EmitMemberDataPointer(type, chars); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index dcf523f56bf1e..669fe867f75ac 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -2180,6 +2180,31 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(contBB); } +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const PFPField &Field) { + return EmitAddressOfPFPField( + Builder.CreateConstInBoundsByteGEP(RecordPtr.withElementType(Int8Ty), + Field.structOffset), + Field.field, Field.offset - Field.structOffset); +} + +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const FieldDecl *Field, + CharUnits Offset) { + return Address( + EmitRuntimeCall( + CGM.getIntrinsic(llvm::Intrinsic::protected_field_ptr), + {RecordPtr.getBasePointer(), Builder.getInt64(Offset.getQuantity()), + llvm::MetadataAsValue::get( + getLLVMContext(), + llvm::MDString::get(getLLVMContext(), + CGM.getPFPFieldName(Field))), + getContext().arePFPFieldsTriviallyRelocatable(Field->getParent()) + ? Builder.getFalse() + : Builder.getTrue()}), + VoidPtrTy, RecordPtr.getAlignment().alignmentAtOffset(Offset)); +} + void CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Ignore empty classes in C++. @@ -2242,13 +2267,22 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Get and call the appropriate llvm.memcpy overload. Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false); - return; + } else { + // Otherwise, just memset the whole thing to zero. This is legal + // because in LLVM, all default initializers (other than the ones we just + // handled above, and the case handled below) are guaranteed to have a bit + // pattern of all zeros. + Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); } - // Otherwise, just memset the whole thing to zero. This is legal - // because in LLVM, all default initializers (other than the ones we just - // handled above) are guaranteed to have a bit pattern of all zeros. - Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); + // With the pointer field protection feature, null pointers do not have a bit + // pattern of zero in memory, so we must initialize them separately. + std::vector PFPFields; + getContext().findPFPFields(Ty, CharUnits::Zero(), PFPFields, true); + for (auto &Field : PFPFields) { + auto addr = EmitAddressOfPFPField(DestPtr, Field); + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), addr); + } } llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) { diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index ca00a0e8c6cf4..fecb48033e6ef 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4993,8 +4993,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// Create a store to \arg DstPtr from \arg Src, truncating the stored value /// to at most \arg DstSize bytes. - void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, - bool DstIsVolatile); + void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst, + llvm::TypeSize DstSize, bool DstIsVolatile); /// EmitExtendGCLifetime - Given a pointer to an Objective-C object, /// make sure it survives garbage collection until this point. @@ -5468,6 +5468,10 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitRISCVMultiVersionResolver(llvm::Function *Resolver, ArrayRef Options); + Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field); + Address EmitAddressOfPFPField(Address RecordPtr, const FieldDecl *Field, + CharUnits Offset); + private: QualType getVarArgType(const Expr *Arg); diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 96ab4dd1837ce..8eb615ddd54fa 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -860,6 +860,7 @@ void CodeGenModule::Release() { applyGlobalValReplacements(); applyReplacements(); emitMultiVersionFunctions(); + emitPFPFieldsWithEvaluatedOffset(); if (Context.getLangOpts().IncrementalExtensions && GlobalTopLevelStmtBlockInFlight.first) { @@ -4410,6 +4411,35 @@ void CodeGenModule::emitMultiVersionFunctions() { emitMultiVersionFunctions(); } +llvm::GlobalValue *CodeGenModule::getPFPDeactivationSymbol(FieldDecl *FD) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *DS = TheModule.getNamedValue(DSName); + if (!DS) { + DS = new llvm::GlobalVariable(TheModule, Int8Ty, false, + llvm::GlobalVariable::ExternalWeakLinkage, + nullptr, DSName); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + } + return DS; +} + +void CodeGenModule::emitPFPFieldsWithEvaluatedOffset() { + llvm::Constant *Nop = llvm::ConstantExpr::getIntToPtr( + llvm::ConstantInt::get(Int64Ty, 0xd503201f), VoidPtrTy); + for (auto *FD : getContext().PFPFieldsWithEvaluatedOffset) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *OldDS = TheModule.getNamedValue(DSName); + llvm::GlobalValue *DS = llvm::GlobalAlias::create( + Int8Ty, 0, llvm::GlobalValue::ExternalLinkage, DSName, Nop, &TheModule); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } +} + static void replaceDeclarationWith(llvm::GlobalValue *Old, llvm::Constant *New) { assert(cast(Old)->isDeclaration() && "Not a declaration"); @@ -7938,3 +7968,12 @@ void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx); } + +std::string CodeGenModule::getPFPFieldName(const FieldDecl *FD) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCanonicalTypeName( + QualType(FD->getParent()->getTypeForDecl(), 0), Out, false); + Out << "." << FD->getName(); + return OutName; +} diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h index 6deb467b2cc9f..7c315d5f10b16 100644 --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -1812,6 +1812,9 @@ class CodeGenModule : public CodeGenTypeCache { return !getLangOpts().CPlusPlus; } + std::string getPFPFieldName(const FieldDecl *FD); + llvm::GlobalValue *getPFPDeactivationSymbol(FieldDecl *FD); + private: bool shouldDropDLLAttribute(const Decl *D, const llvm::GlobalValue *GV) const; @@ -2003,6 +2006,10 @@ class CodeGenModule : public CodeGenTypeCache { llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, StringRef Suffix); + + /// Emit deactivation symbols for any PFP fields whose offset is taken with + /// offsetof. + void emitPFPFieldsWithEvaluatedOffset(); }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index 77e995b4c933a..f3940b031ffe7 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -1224,6 +1224,7 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP, return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM); } + getContext().recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD)); return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset); diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 40371d99e23e1..60b2e68755515 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -2917,6 +2917,7 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP, // the class in which it was declared, and convert from there if necessary. // For indirect field decls, get the outermost anonymous field and use the // parent class. + Ctx.recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(MPD)); const FieldDecl *FD = dyn_cast(MPD); if (!FD) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index e1e8f57dd6455..027aa0b55d343 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7732,6 +7732,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, Args.addOptInFlag(CmdArgs, options::OPT_fexperimental_late_parse_attributes, options::OPT_fno_experimental_late_parse_attributes); + if (!IsCudaDevice) + Args.AddLastArg(CmdArgs, + options::OPT_experimental_pointer_field_protection_EQ); + // Setup statistics file output. SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D); if (!StatsFile.empty()) { diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 0a8a3e1c49414..60088dd56a48e 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -6441,6 +6441,10 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL)); } +static void handleNoPFPAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { + D->addAttr(NoPointerFieldProtectionAttr::Create(S.Context, AL)); +} + static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { auto *FD = dyn_cast(D); assert(FD); @@ -7452,6 +7456,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, handleCountedByAttrField(S, D, AL); break; + case ParsedAttr::AT_NoPointerFieldProtection: + handleNoPFPAttrField(S, D, AL); + break; + // Microsoft attributes: case ParsedAttr::AT_LayoutVersion: handleLayoutVersion(S, D, AL); diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index b2310628adc64..70d9ec9f6a7f2 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -5103,6 +5103,7 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT, // impose the same constraints. case UTT_IsTriviallyRelocatable: case UTT_IsTriviallyEqualityComparable: + case UTT_HasNonRelocatableFields: case UTT_CanPassInRegs: // Per the GCC type traits documentation, T shall be a complete type, cv void, // or an array of unknown bound. But GCC actually imposes the same constraints @@ -5672,6 +5673,10 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT, return C.hasUniqueObjectRepresentations(T); case UTT_IsTriviallyRelocatable: return T.isTriviallyRelocatableType(C); + case UTT_HasNonRelocatableFields: + return T->getAsCXXRecordDecl() && + !C.arePFPFieldsTriviallyRelocatable(T->getAsCXXRecordDecl()) && + C.hasPFPFields(T); case UTT_IsBitwiseCloneable: return T.isBitwiseCloneableType(C); case UTT_CanPassInRegs: diff --git a/clang/test/CodeGen/pfp-attribute-disable.cpp b/clang/test/CodeGen/pfp-attribute-disable.cpp new file mode 100644 index 0000000000000..a8efb08b9a7ee --- /dev/null +++ b/clang/test/CodeGen/pfp-attribute-disable.cpp @@ -0,0 +1,33 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + + +struct S { + int* ptr1; + __attribute__((no_field_protection)) int* ptr2; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers_without_no_field_protection +int* load_pointers_without_no_field_protection(S *t) { + return t->ptr1; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: load_pointers_with_no_field_protection +int* load_pointers_with_no_field_protection(S *t) { + return t->ptr2; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_without_no_field_protection +void store_pointers_without_no_field_protection(S *t, int *input) { + t->ptr1 = input; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_with_no_field_protection +void store_pointers_with_no_field_protection(S *t, int *input) { + t->ptr2 = input; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-load-store.cpp b/clang/test/CodeGen/pfp-load-store.cpp new file mode 100644 index 0000000000000..b5f6a11574270 --- /dev/null +++ b/clang/test/CodeGen/pfp-load-store.cpp @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -O1 -o - %s | FileCheck %s + +int val; + +struct Pointer { + int* ptr; +private: + int private_data; +}; + +struct ArrayType { + int* array[3]; +private: + int private_data; +}; + +struct Array { + ArrayType array; +private: + int private_data; +}; + +struct Struct { + Pointer ptr; +}; + +// CHECK-LABEL: test_pointer +Pointer test_pointer(Pointer t) { + t.ptr = &val; + return t; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + + + +// CHECK-LABEL: test_struct +int* test_struct(Struct *t) { + return (t->ptr).ptr; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-memcpy.cpp b/clang/test/CodeGen/pfp-memcpy.cpp new file mode 100644 index 0000000000000..3e92db6110ae3 --- /dev/null +++ b/clang/test/CodeGen/pfp-memcpy.cpp @@ -0,0 +1,19 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct ClassWithTrivialCopy { + ClassWithTrivialCopy(); + ~ClassWithTrivialCopy(); + void *a; +private: + void *c; +}; + +// Make sure we don't emit memcpy for operator= and constuctors. +void make_trivial_copy(ClassWithTrivialCopy *s1, ClassWithTrivialCopy *s2) { + *s1 = *s2; + ClassWithTrivialCopy s3(*s2); +} + +// CHECK-LABEL: define{{.*}} void @_Z17make_trivial_copyP20ClassWithTrivialCopyS0_ +// CHECK-NOT: memcpy +// CHECK: ret void \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-null-init.cpp b/clang/test/CodeGen/pfp-null-init.cpp new file mode 100644 index 0000000000000..94c1224f1be13 --- /dev/null +++ b/clang/test/CodeGen/pfp-null-init.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct S { + void *p; +private: + int private_data; +}; + +// CHECK-LABEL: null_init +void null_init() { + S s{}; +} + +// Check that the constructor was applied +// CHECK: call void @llvm.memset.{{.*}} +// CHECK: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.p", i1 false) \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-struct-gep.cpp b/clang/test/CodeGen/pfp-struct-gep.cpp new file mode 100644 index 0000000000000..964545efa9f4b --- /dev/null +++ b/clang/test/CodeGen/pfp-struct-gep.cpp @@ -0,0 +1,25 @@ +// RUN: %clang_cc1 -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOPFP +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-PFP + + +struct S { + int* ptr; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers +int* load_pointers(S *t) { + return t->ptr; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + +// CHECK-LABEL: store_pointers +void store_pointers(S* t, int* p) { + t->ptr = p; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + + diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp index 90054dbf37ae3..3d26a8639e13f 100644 --- a/clang/test/CodeGenCXX/trivial_abi.cpp +++ b/clang/test/CodeGenCXX/trivial_abi.cpp @@ -286,11 +286,9 @@ static_assert(sizeof(S) == 8 && sizeof(S2) == 8, ""); // CHECK: define{{.*}} @"_ZN3$_08__invokeEv"() // CHECK: %[[RETVAL:.*]] = alloca %[[STRUCT_SMALL]], align 8 -// CHECK: %[[COERCE:.*]] = alloca %[[STRUCT_SMALL]], align 8 // CHECK: %[[CALL:.*]] = call{{.*}} @"_ZNK3$_0clEv" -// CHECK: %[[COERCEDIVE:.*]] = getelementptr{{.*}} %[[COERCE]] +// CHECK: %[[RETVALDIVE:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[COERCEVALIP:.*]] = inttoptr{{.*}} %[[CALL]] -// CHECK: call {{.*}}memcpy{{.*}} %[[RETVAL]]{{.*}} %[[COERCE]] // CHECK: %[[COERCEDIVE1:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[TMP:.*]] = load{{.*}} %[[COERCEDIVE1]] // CHECK: %[[COERCEVALPI:.*]] = ptrtoint{{.*}} %[[TMP]] diff --git a/libcxx/include/__config b/libcxx/include/__config index c8224b07a6b81..27fd748e333e1 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1215,6 +1215,29 @@ typedef __char32_t char32_t; # define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 0 # endif +# if __has_feature(pointer_field_protection) +// Force a class to be non-standard layout by giving it two bases with the same +// type. This is useful when structure protection is enabled because structure +// protection cannot be applied to standard layout classes. We may use this in +// cases where the standard does not specify whether a standard library class is +// standard layout. See C++2a [class]p7: +// A class S is a standard-layout class if it: +// -- has at most one base class subobject of any given type +_LIBCPP_BEGIN_NAMESPACE_STD +_LIBCPP_DIAGNOSTIC_PUSH +_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winaccessible-base") +class __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout_base2 : __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout : __force_nonstandard_layout_base1, __force_nonstandard_layout_base2 {}; +_LIBCPP_DIAGNOSTIC_POP +_LIBCPP_END_NAMESPACE_STD +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT : __force_nonstandard_layout +# define _LIBCPP_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT +# define _LIBCPP_NO_PFP +# endif + #endif // __cplusplus #endif // _LIBCPP___CONFIG diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h index f33f424c66c22..55ca0fe4fc9a8 100644 --- a/libcxx/include/__functional/function.h +++ b/libcxx/include/__functional/function.h @@ -624,7 +624,7 @@ template class __policy_func; template -class __policy_func<_Rp(_ArgTypes...)> { +class __policy_func<_Rp(_ArgTypes...)> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { // Inline storage for small objects. __policy_storage __buf_; diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index e1d49c4594866..bd96841c68ebc 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -304,7 +304,7 @@ using __shared_ptr_nullptr_deleter_ctor_reqs _LIBCPP_NODEBUG = #endif template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { struct __nullptr_sfinae_tag {}; public: @@ -1202,7 +1202,7 @@ inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXC #endif // _LIBCPP_HAS_RTTI template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: #if _LIBCPP_STD_VER >= 17 typedef remove_extent_t<_Tp> element_type; diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h index 6f1dc98db5a9f..5e4fa857eb14b 100644 --- a/libcxx/include/__memory/unique_ptr.h +++ b/libcxx/include/__memory/unique_ptr.h @@ -126,7 +126,7 @@ struct __unique_ptr_deleter_sfinae<_Deleter&> { #endif template > -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; @@ -393,7 +393,7 @@ struct __unique_ptr_array_bounds_stored { }; template -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; diff --git a/libcxx/include/__tree b/libcxx/include/__tree index 08ae8996f8f7d..ef55bd5dede99 100644 --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -875,7 +875,7 @@ _LIBCPP_DIAGNOSE_WARNING(!__is_invocable_v<_Compare const&, _Tp const&, _Tp cons int __diagnose_non_const_comparator(); template -class __tree { +class __tree _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp value_type; typedef _Compare value_compare; diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h index 9b0e240de55f4..240b3b76adf36 100644 --- a/libcxx/include/__type_traits/is_trivially_relocatable.h +++ b/libcxx/include/__type_traits/is_trivially_relocatable.h @@ -36,8 +36,12 @@ struct __libcpp_is_trivially_relocatable : is_trivially_copyable<_Tp> {}; template struct __libcpp_is_trivially_relocatable<_Tp, - __enable_if_t::value> > - : true_type {}; + __enable_if_t::value +#if __has_builtin(__has_non_relocatable_fields) + && !__has_non_relocatable_fields(_Tp) +#endif + > > : true_type { +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 8818eb7dfe26e..a8e37525a75e6 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -84,7 +84,7 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template */> -class _LIBCPP_TEMPLATE_VIS vector { +class _LIBCPP_TEMPLATE_VIS vector _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: // // Types diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo index 799c6ebd5ecbb..1a59bc8ff149b 100644 --- a/libcxx/include/typeinfo +++ b/libcxx/include/typeinfo @@ -300,7 +300,7 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH type_info protected: typedef __type_info_implementations::__impl __impl; - __impl::__type_name_t __type_name; + _LIBCPP_NO_PFP __impl::__type_name_t __type_name; _LIBCPP_HIDE_FROM_ABI explicit type_info(const char* __n) : __type_name(__impl::__string_to_type_name(__n)) {} diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp index dcdce261298c1..7201340c21039 100644 --- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp +++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp @@ -255,9 +255,12 @@ void unique_ptr_test() { ComparePrettyPrintToRegex(std::move(forty_two), R"(std::unique_ptr containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::unique_ptr this_is_null; ComparePrettyPrintToChars(std::move(this_is_null), R"(std::unique_ptr is nullptr)"); +#endif } void bitset_test() { @@ -475,10 +478,13 @@ void vector_test() { "std::vector of length " "3, capacity 3 = {5, 6, 7}"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::vector> test3({7, 8}); ComparePrettyPrintToChars(std::move(test3), "std::vector of length " "2, capacity 2 = {7, 8}"); +#endif } void set_iterator_test() { @@ -649,8 +655,11 @@ void shared_ptr_test() { test0, R"(std::shared_ptr count [3\?], weak [0\?]( \(libc\+\+ missing debug info\))? containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::shared_ptr test3; ComparePrettyPrintToChars(test3, "std::shared_ptr is nullptr"); +#endif } void streampos_test() { diff --git a/libcxxabi/include/__cxxabi_config.h b/libcxxabi/include/__cxxabi_config.h index 759445dac91f9..00e1357cdab85 100644 --- a/libcxxabi/include/__cxxabi_config.h +++ b/libcxxabi/include/__cxxabi_config.h @@ -109,4 +109,14 @@ # define _LIBCXXABI_NOEXCEPT noexcept #endif +#if defined(_LIBCXXABI_COMPILER_CLANG) +# if __has_feature(pointer_field_protection) +# define _LIBCXXABI_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCXXABI_NO_PFP +# endif +#else +# define _LIBCXXABI_NO_PFP +#endif + #endif // ____CXXABI_CONFIG_H diff --git a/libcxxabi/src/private_typeinfo.h b/libcxxabi/src/private_typeinfo.h index 328a02edef5c1..a3bc0bffd41bc 100644 --- a/libcxxabi/src/private_typeinfo.h +++ b/libcxxabi/src/private_typeinfo.h @@ -145,7 +145,7 @@ class _LIBCXXABI_TYPE_VIS __class_type_info : public __shim_type_info { // Has one non-virtual public base class at offset zero class _LIBCXXABI_TYPE_VIS __si_class_type_info : public __class_type_info { public: - const __class_type_info *__base_type; + _LIBCXXABI_NO_PFP const __class_type_info *__base_type; _LIBCXXABI_HIDDEN virtual ~__si_class_type_info(); @@ -204,7 +204,7 @@ class _LIBCXXABI_TYPE_VIS __vmi_class_type_info : public __class_type_info { class _LIBCXXABI_TYPE_VIS __pbase_type_info : public __shim_type_info { public: unsigned int __flags; - const __shim_type_info *__pointee; + _LIBCXXABI_NO_PFP const __shim_type_info *__pointee; enum __masks { __const_mask = 0x1, @@ -245,7 +245,7 @@ class _LIBCXXABI_TYPE_VIS __pointer_type_info : public __pbase_type_info { class _LIBCXXABI_TYPE_VIS __pointer_to_member_type_info : public __pbase_type_info { public: - const __class_type_info *__context; + _LIBCXXABI_NO_PFP const __class_type_info *__context; _LIBCXXABI_HIDDEN virtual ~__pointer_to_member_type_info(); _LIBCXXABI_HIDDEN virtual bool can_catch(const __shim_type_info *, diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h index c9d3874e7dd96..862ce9b5f18e8 100644 --- a/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -134,6 +134,7 @@ class PtrUseVisitorBase { UseAndIsOffsetKnownPair UseAndIsOffsetKnown; APInt Offset; + Value *ProtectedField; }; /// The worklist of to-visit uses. @@ -158,6 +159,8 @@ class PtrUseVisitorBase { /// The constant offset of the use if that is known. APInt Offset; + Value *ProtectedField; + /// @} /// Note that the constructor is protected because this class must be a base @@ -230,6 +233,7 @@ class PtrUseVisitor : protected InstVisitor, IntegerType *IntIdxTy = cast(DL.getIndexType(I.getType())); IsOffsetKnown = true; Offset = APInt(IntIdxTy->getBitWidth(), 0); + ProtectedField = nullptr; PI.reset(); // Enqueue the uses of this pointer. @@ -242,6 +246,7 @@ class PtrUseVisitor : protected InstVisitor, IsOffsetKnown = ToVisit.UseAndIsOffsetKnown.getInt(); if (IsOffsetKnown) Offset = std::move(ToVisit.Offset); + ProtectedField = ToVisit.ProtectedField; Instruction *I = cast(U->getUser()); static_cast(this)->visit(I); @@ -301,6 +306,16 @@ class PtrUseVisitor : protected InstVisitor, case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: return; // No-op intrinsics. + + case Intrinsic::protected_field_ptr: { + auto *OffsetConst = dyn_cast(II.getArgOperand(1)); + if (!IsOffsetKnown || !OffsetConst) + return Base::visitIntrinsicInst(II); + Offset += OffsetConst->getValue().sextOrTrunc(Offset.getBitWidth()); + ProtectedField = II.getArgOperand(2); + enqueueUsers(II); + break; + } } } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index d10b07ccd91c2..b6369c40235bb 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2534,6 +2534,29 @@ def int_hwasan_check_memaccess_shortgranules_fixedshadow : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i64_ty], [ImmArg>, ImmArg>]>; +// Returns a pointer which may be used to store a pointer at the address +// (first argument + second argument) with protection against use-after-free +// vulnerabilities. Stores via the pointer will cause the stored pointer to be +// blended with some combination of: +// - the field identifier (third argument) +// - the struct address (first argument), if the struct is not trivially +// copyable (fourth argument is true) +// before being stored. When loading from the pointer, the inverse operation +// is done on the loaded pointer after it is loaded. Specifically, when +// targeting AArch64 with pointer authentication enabled and when the fourth +// argument is true, the pointer is signed using the struct address before +// being stored, and authenticated after being loaded. Otherwise it is rotated +// left by 16 bits and a hash of the field identifier is subtracted before +// being stored, and the hash is added and the pointer is rotated right by +// 16 bits after being loaded. If the pointer is used otherwise than for +// loading or storing (e.g. its address escapes), it will cause all blending +// operations using the same field identifier to be disabled throughout the +// program. This is done using special linker relocations. +def int_protected_field_ptr : + DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, + llvm_metadata_ty, llvm_i1_ty], + [IntrNoMem, ImmArg>, ImmArg>]>; + // Xray intrinsics //===----------------------------------------------------------------------===// // Custom event logging for x-ray. diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index db064e1f41f02..847bedeb694d9 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -175,6 +175,8 @@ bool EliminateDuplicatePHINodes(BasicBlock *BB); bool EliminateDuplicatePHINodes(BasicBlock *BB, SmallPtrSetImpl &ToRemove); +bool shouldFoldLoadStoreWithPointerOperandThroughPhi(Value *Ptr); + /// This function is used to do simplification of a CFG. For example, it /// adjusts branches to branches to eliminate the extra hop, it eliminates /// unreachable basic blocks, and does other peephole optimization of the CFG. diff --git a/llvm/lib/Analysis/PtrUseVisitor.cpp b/llvm/lib/Analysis/PtrUseVisitor.cpp index 9c79546f491ef..b34f380b7db5c 100644 --- a/llvm/lib/Analysis/PtrUseVisitor.cpp +++ b/llvm/lib/Analysis/PtrUseVisitor.cpp @@ -22,7 +22,8 @@ void detail::PtrUseVisitorBase::enqueueUsers(Value &I) { if (VisitedUses.insert(&U).second) { UseToVisit NewU = { UseToVisit::UseAndIsOffsetKnownPair(&U, IsOffsetKnown), - Offset + Offset, + ProtectedField, }; Worklist.push_back(std::move(NewU)); } diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 9dc1764b49e46..58cb0370357e4 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -21,9 +21,11 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" @@ -36,6 +38,8 @@ #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h" +#include + using namespace llvm; /// Threshold to leave statically sized memory intrinsic calls. Calls of known @@ -441,6 +445,254 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { return Changed; } +namespace { + +enum class PointerEncoding { + Rotate, + PACCopyable, + PACNonCopyable, +}; + +bool expandProtectedFieldPtr(Function &Intr) { + Module &M = *Intr.getParent(); + bool IsAArch64 = Triple(M.getTargetTriple()).isAArch64(); + + std::set NonPFPFields; + std::set LoadsStores; + + Type *Int8Ty = Type::getInt8Ty(M.getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); + PointerType *PtrTy = PointerType::get(M.getContext(), 0); + + Function *SignIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_sign, {}); + Function *AuthIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_auth, {}); + + auto *EmuFnTy = FunctionType::get(Int64Ty, {Int64Ty, Int64Ty}, false); + FunctionCallee EmuSignIntr = M.getOrInsertFunction("__emupac_pacda", EmuFnTy); + FunctionCallee EmuAuthIntr = M.getOrInsertFunction("__emupac_autda", EmuFnTy); + + auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(SignIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuSignIntr, {Val, Disc}, DSBundle); + }; + + auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(AuthIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuAuthIntr, {Val, Disc}, DSBundle); + }; + + for (User *U : Intr.users()) { + auto *Call = cast(U); + auto *FieldName = cast( + cast(Call->getArgOperand(2))->getMetadata()); + std::set VisitedPhis; + + std::function FindLoadsStores; + FindLoadsStores = [&](Instruction *I) { + for (Use &U : I->uses()) { + if (auto *LI = dyn_cast(U.getUser())) { + if (isa(LI->getType())) { + LoadsStores.insert(LI); + continue; + } + } + if (auto *SI = dyn_cast(U.getUser())) { + if (U.getOperandNo() == 1 && + isa(SI->getValueOperand()->getType())) { + LoadsStores.insert(SI); + continue; + } + } + if (auto *P = dyn_cast(U.getUser())) { + if (VisitedPhis.insert(P).second) + FindLoadsStores(P); + continue; + } + // Comparisons against null cannot be used to recover the original + // pointer so we allow them. + if (auto *CI = dyn_cast(U.getUser())) { + if (auto *Op = dyn_cast(CI->getOperand(0))) + if (Op->isNullValue()) + continue; + if (auto *Op = dyn_cast(CI->getOperand(1))) + if (Op->isNullValue()) + continue; + } + NonPFPFields.insert(FieldName); + } + }; + + FindLoadsStores(Call); + } + + for (Instruction *I : LoadsStores) { + std::set Offsets; + std::set Fields; + std::set VisitedPhis; + bool IsNonTriviallyCopyable = false; + + std::function FindFields; + FindFields = [&](Value *V) { + if (auto *Call = dyn_cast(V)) { + if (Call->getCalledOperand() == &Intr) { + Offsets.insert(Call->getArgOperand(1)); + auto *Field = cast( + cast(Call->getArgOperand(2))->getMetadata()); + Fields.insert(Field); + if (cast(Call->getArgOperand(3))->getZExtValue()) + IsNonTriviallyCopyable = true; + return; + } + } + if (auto *P = dyn_cast(V)) { + if (VisitedPhis.insert(P).second) + for (Value *V : P->incoming_values()) + FindFields(V); + return; + } + Fields.insert(nullptr); + }; + FindFields(isa(I) ? cast(I)->getPointerOperand() + : cast(I)->getPointerOperand()); + if (Fields.size() != 1 || Offsets.size() != 1) { + for (Metadata *Field : Fields) + if (Field) + NonPFPFields.insert(Field); + continue; + } + + std::string FieldName = cast(*Fields.begin())->getString().str(); + uint64_t FieldSignature = std::hash()(FieldName); + + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *DS = M.getNamedValue(DSName); + if (!DS) { + DS = new GlobalVariable(M, Int8Ty, false, + GlobalVariable::ExternalWeakLinkage, nullptr, + DSName); + DS->setVisibility(GlobalValue::HiddenVisibility); + } + OperandBundleDef DSBundle("deactivation-symbol", DS); + + PointerEncoding Encoding; + if (!IsAArch64) + Encoding = PointerEncoding::Rotate; + else if (IsNonTriviallyCopyable) + Encoding = PointerEncoding::PACNonCopyable; + else + Encoding = PointerEncoding::PACCopyable; + + if (auto *LI = dyn_cast(I)) { + auto *FieldAddr = LI->getPointerOperand(); + IRBuilder<> B(LI->getNextNode()); + auto *LIInt = cast(B.CreatePtrToInt(LI, B.getInt64Ty())); + Value *Auth; + switch (Encoding) { + case PointerEncoding::Rotate: + Auth = B.CreateAdd(LIInt, B.getInt64(FieldSignature & 0xff)); + Auth = B.CreateOr(B.CreateLShr(Auth, 16), B.CreateShl(Auth, 48)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Auth = CreateAuth(B, LIInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Auth = + CreateAuth(B, LIInt, B.getInt64(FieldSignature & 0xffff), DSBundle); + break; + } + LI->replaceAllUsesWith(B.CreateIntToPtr(Auth, B.getPtrTy())); + LIInt->setOperand(0, LI); + } else if (auto *SI = dyn_cast(I)) { + IRBuilder<> B(SI); + auto *FieldAddr = SI->getPointerOperand(); + auto *SIValInt = + B.CreatePtrToInt(SI->getValueOperand(), B.getInt64Ty()); + Value *Sign; + switch (Encoding) { + case PointerEncoding::Rotate: + Sign = + B.CreateOr(B.CreateLShr(SIValInt, 48), B.CreateShl(SIValInt, 16)); + Sign = B.CreateSub(Sign, B.getInt64(FieldSignature & 0xff)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Sign = CreateSign(B, SIValInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Sign = CreateSign(B, SIValInt, B.getInt64(FieldSignature & 0xffff), + DSBundle); + break; + } + SI->setOperand(0, B.CreateIntToPtr(Sign, B.getPtrTy())); + } + } + + for (User *U : llvm::make_early_inc_range(Intr.users())) { + auto *Call = cast(U); + auto *Struct = Call->getArgOperand(0); + auto *Offset = Call->getArgOperand(1); + + IRBuilder<> B(Call); + if (cast(Offset)->getZExtValue() == 0) + Call->replaceAllUsesWith(Struct); + else + Call->replaceAllUsesWith(B.CreateGEP(B.getInt8Ty(), Struct, {Offset})); + Call->eraseFromParent(); + } + + if (!NonPFPFields.empty()) { + Constant *Nop = + ConstantExpr::getIntToPtr(ConstantInt::get(Int64Ty, 0xd503201f), PtrTy); + std::set LocalNonPFPFieldNames; + for (auto *Field : NonPFPFields) + LocalNonPFPFieldNames.insert(cast(Field)->getString().str()); + for (auto &FieldName : LocalNonPFPFieldNames) { + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *OldDS = M.getNamedValue(DSName); + GlobalValue *DS = GlobalAlias::create( + Int8Ty, 0, GlobalValue::ExternalLinkage, DSName, Nop, &M); + DS->setVisibility(GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } + } + return true; +} + +} + bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { bool Changed = false; for (Function &F : M) { @@ -572,6 +824,9 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { return lowerUnaryVectorIntrinsicAsLoop(M, CI); }); break; + case Intrinsic::protected_field_ptr: + Changed |= expandProtectedFieldPtr(F); + break; } } return Changed; diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 255cd0ec5840c..c26c4562a17fe 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -2403,6 +2403,7 @@ class BImm pattern> let Inst{25-0} = addr; let DecoderMethod = "DecodeUnconditionalBranch"; + let supportsDeactivationSymbol = true; } class BranchImm pattern> @@ -2460,6 +2461,7 @@ class SignAuthOneData opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = Rn; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthZero opcode_prefix, bits<2> opcode, string asm, @@ -2473,6 +2475,7 @@ class SignAuthZero opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = 0b11111; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthTwoOperand opc, string asm, diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 80308bf92dbbc..4041c40de9cec 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -697,8 +697,7 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) { Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LoadInst *FirstLI = cast(PN.getIncomingValue(0)); - // Can't forward swifterror through a phi. - if (FirstLI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(FirstLI->getOperand(0))) return nullptr; // FIXME: This is overconservative; this transform is allowed in some cases @@ -737,8 +736,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LI->getPointerAddressSpace() != LoadAddrSpace) return nullptr; - // Can't forward swifterror through a phi. - if (LI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(LI->getOperand(0))) return nullptr; // We can't sink the load if the loaded value could be modified between diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 86be20c799a68..ca428f4f5b851 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -534,9 +534,9 @@ class Slice { public: Slice() = default; - Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) + Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable, Value *ProtectedField) : BeginOffset(BeginOffset), EndOffset(EndOffset), - UseAndIsSplittable(U, IsSplittable) {} + UseAndIsSplittable(U, IsSplittable), ProtectedField(ProtectedField) {} uint64_t beginOffset() const { return BeginOffset; } uint64_t endOffset() const { return EndOffset; } @@ -549,6 +549,8 @@ class Slice { bool isDead() const { return getUse() == nullptr; } void kill() { UseAndIsSplittable.setPointer(nullptr); } + Value *ProtectedField; + /// Support for ordering ranges. /// /// This provides an ordering over ranges such that start offsets are @@ -1075,7 +1077,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { EndOffset = AllocSize; } - AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); + AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable, ProtectedField)); } void visitBitCastInst(BitCastInst &BC) { @@ -4595,7 +4597,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PLoad << "\n"); @@ -4751,10 +4753,12 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { LLVMContext::MD_access_group}); // Now build a new slice for the alloca. + // ProtectedField==nullptr is a lie, but it doesn't matter because we + // already determined that all accesses are consistent. NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PStore->getOperandUse(PStore->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PStore << "\n"); @@ -5602,6 +5606,32 @@ SROA::runOnAlloca(AllocaInst &AI) { return {Changed, CFGChanged}; } + for (auto &P : AS.partitions()) { + std::optional ProtectedField; + // For now, we can't split if a field is accessed both via protected + // field and not. + for (Slice &S : P) { + if (auto *II = dyn_cast(S.getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S.ProtectedField; + if (*ProtectedField != S.ProtectedField) + return {Changed, CFGChanged}; + } + for (Slice *S : P.splitSliceTails()) { + if (auto *II = dyn_cast(S->getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S->ProtectedField; + if (*ProtectedField != S->ProtectedField) + return {Changed, CFGChanged}; + } + } + // Delete all the dead users of this alloca before splitting and rewriting it. for (Instruction *DeadUser : AS.getDeadUsers()) { // Free up everything used by this instruction. diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 02f1d08759129..80562255acae7 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -2148,6 +2148,22 @@ static bool replacingOperandWithVariableIsCheap(const Instruction *I, return !isa(I); } +bool llvm::shouldFoldLoadStoreWithPointerOperandThroughPhi(Value *Ptr) { + // swifterror pointers can only be used by a load or store; sinking a load + // or store would require introducing a select for the pointer operand, + // which isn't allowed for swifterror pointers. + if (Ptr->isSwiftError()) + return false; + + // Protected pointer field loads/stores should be paired with the intrinsic + // to avoid unnecessary address escapes. + if (auto *II = dyn_cast(Ptr)) + if (II->getIntrinsicID() == Intrinsic::protected_field_ptr) + return false; + + return true; +} + // All instructions in Insts belong to different blocks that all unconditionally // branch to a common successor. Analyze each instruction and return true if it // would be possible to sink them into their successor, creating one common @@ -2190,12 +2206,11 @@ static bool canSinkInstructions( if (!I->isSameOperationAs(I0, Instruction::CompareUsingIntersectedAttrs)) return false; - // swifterror pointers can only be used by a load or store; sinking a load - // or store would require introducing a select for the pointer operand, - // which isn't allowed for swifterror pointers. - if (isa(I) && I->getOperand(1)->isSwiftError()) + if (isa(I) && + !shouldFoldLoadStoreWithPointerOperandThroughPhi(I->getOperand(1))) return false; - if (isa(I) && I->getOperand(0)->isSwiftError()) + if (isa(I) && + !shouldFoldLoadStoreWithPointerOperandThroughPhi(I->getOperand(0))) return false; // Treat MMRAs conservatively. This pass can be quite aggressive and From 7fd12b3b325bfbb1d7a9f07bce1ad89b0aed92aa Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Thu, 27 Mar 2025 16:34:39 -0700 Subject: [PATCH 13/18] personal change --- compiler-rt/lib/builtins/aarch64/emupac.cpp | 4 + llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 4 + .../utils/gn/build/toolchain/target_flags.gni | 23 ++++ .../compiler-rt/lib/builtins/BUILD.gn | 4 + .../gn/secondary/llvm/lib/CodeGen/BUILD.gn | 5 + llvm/utils/pfp-bazel | 17 +++ llvm/utils/pfp-check-libcxx | 90 +++++++++++++ llvm/utils/pfp-cmake | 17 +++ llvm/utils/pfp-patches/dart.patch | 127 ++++++++++++++++++ 9 files changed, 291 insertions(+) create mode 100755 llvm/utils/pfp-bazel create mode 100755 llvm/utils/pfp-check-libcxx create mode 100755 llvm/utils/pfp-cmake create mode 100644 llvm/utils/pfp-patches/dart.patch diff --git a/compiler-rt/lib/builtins/aarch64/emupac.cpp b/compiler-rt/lib/builtins/aarch64/emupac.cpp index 95a21c969da3d..c27c2aa5dee28 100644 --- a/compiler-rt/lib/builtins/aarch64/emupac.cpp +++ b/compiler-rt/lib/builtins/aarch64/emupac.cpp @@ -47,7 +47,11 @@ const uint64_t kTTBR1Mask = 1ULL << 55; static bool pac_supported() { register uintptr_t x30 __asm__("x30") = 1ULL << 55; __asm__ __volatile__("xpaclri" : "+r"(x30)); +#ifdef FORCE_NON_PAC + return !(x30 & (1ULL << 54)); +#else return x30 & (1ULL << 54); +#endif } // This asm snippet is used to force the creation of a frame record when diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 58cb0370357e4..6d57eca420fb7 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -475,19 +475,23 @@ bool expandProtectedFieldPtr(Function &Intr) { auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc, OperandBundleDef DSBundle) { +#ifndef FORCE_EMUPAC Function *F = B.GetInsertBlock()->getParent(); Attribute FSAttr = F->getFnAttribute("target-features"); if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) return B.CreateCall(SignIntr, {Val, B.getInt32(2), Disc}, DSBundle); +#endif return B.CreateCall(EmuSignIntr, {Val, Disc}, DSBundle); }; auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc, OperandBundleDef DSBundle) { +#ifndef FORCE_EMUPAC Function *F = B.GetInsertBlock()->getParent(); Attribute FSAttr = F->getFnAttribute("target-features"); if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) return B.CreateCall(AuthIntr, {Val, B.getInt32(2), Disc}, DSBundle); +#endif return B.CreateCall(EmuAuthIntr, {Val, Disc}, DSBundle); }; diff --git a/llvm/utils/gn/build/toolchain/target_flags.gni b/llvm/utils/gn/build/toolchain/target_flags.gni index cdfab75ed8bcd..82323e66b3ae5 100644 --- a/llvm/utils/gn/build/toolchain/target_flags.gni +++ b/llvm/utils/gn/build/toolchain/target_flags.gni @@ -2,6 +2,12 @@ import("//llvm/triples.gni") import("//llvm/utils/gn/build/mac_sdk.gni") import("//llvm/utils/gn/build/toolchain/compiler.gni") +declare_args() { + stage2_pointer_field_protection = "none" + emupac_force = false + emupac_force_non_pac = false +} + # Flags in this file are passed both to the compiler that's building # compiler-rt at build time (via normal gn cflags/ldflags), as well as to the # compiler building compiler-rt test programs at test time (via @@ -63,3 +69,20 @@ if (current_cpu == "x86") { target_flags += [ "-m32" ] } } + +if (current_toolchain != host_toolchain && current_cpu == "arm64") { + target_flags += [ + "-march=armv8.3a", + ] +} +if (current_toolchain != host_toolchain) { + target_flags += [ + "-fexperimental-pointer-field-protection=$stage2_pointer_field_protection", + ] + target_ldflags += [ + "--rtlib=compiler-rt", + "--unwindlib=libunwind", + "-static-libgcc", + "-L" + rebase_path(root_build_dir) + "/lib", + ] +} diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn index 445c1c269e37e..d1a97568a2f03 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn @@ -1,5 +1,6 @@ import("//compiler-rt/target.gni") import("//llvm/utils/gn/build/buildflags.gni") +import("//llvm/utils/gn/build/toolchain/target_flags.gni") declare_args() { # Skip the atomic builtin (should normally be provided by a shared library). @@ -510,6 +511,9 @@ static_library("builtins") { } } + if (emupac_force_non_pac) { + defines = [ "FORCE_NON_PAC" ] + } if (current_cpu == "avr") { sources += [ "avr/divmodhi4.S", diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn index b99676b52aea7..4d48c5ff41da0 100644 --- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn @@ -1,3 +1,5 @@ +import("//llvm/utils/gn/build/toolchain/target_flags.gni") + static_library("CodeGen") { output_name = "LLVMCodeGen" public_deps = [ @@ -254,4 +256,7 @@ static_library("CodeGen") { "WindowScheduler.cpp", "XRayInstrumentation.cpp", ] + if (emupac_force) { + defines = [ "FORCE_EMUPAC" ] + } } diff --git a/llvm/utils/pfp-bazel b/llvm/utils/pfp-bazel new file mode 100755 index 0000000000000..ee9caff17ae8e --- /dev/null +++ b/llvm/utils/pfp-bazel @@ -0,0 +1,17 @@ +#!/bin/sh -xe + +shopt -s globstar + +tc="$(dirname $0)/../../ra" + +mtime_before="$(stat -c %Y $tc/bin/clang || true)" +ninja -C $tc clang +mtime_after="$(stat -c %Y $tc/bin/clang)" +if [ "$mtime_before" != "$mtime_after" ] ; then + rm -f $tc/stage2_*/**/*.o +fi +ninja -C $tc libcxx + +cflags=$(grep COMPILER_RT_TEST_COMPILER_CFLAGS= $tc/stage2_unix/toolchain.ninja | sed -e 's/\\\$ /:/g' | grep -o COMPILER_RT_TEST_COMPILER_CFLAGS='[^ ]*' | cut -d= -f2-) +cflags="$cflags:-Wno-unused-command-line-argument" +BAZEL_CXXOPTS=$cflags:-stdlib=libc++ BAZEL_LINKLIBS=$cflags:-Wl,-Bstatic:-lc++:-lc++abi:-Wl,-Bdynamic:-lm CC=clang PATH=$tc/bin:$PATH bazel "$@" diff --git a/llvm/utils/pfp-check-libcxx b/llvm/utils/pfp-check-libcxx new file mode 100755 index 0000000000000..dd9b944d71192 --- /dev/null +++ b/llvm/utils/pfp-check-libcxx @@ -0,0 +1,90 @@ +#!/bin/sh -xe + +shopt -s globstar + +tc="$(cd $(dirname $0)/../../ra && pwd)" + +mtime_before="$(stat -c %Y $tc/bin/clang || true)" +ninja -C $tc clang +mtime_after="$(stat -c %Y $tc/bin/clang)" +if [ "$mtime_before" != "$mtime_after" ] ; then + rm -f $tc/stage2_*/**/*.o +fi +ninja -C $tc libcxx + +cflags=$(grep COMPILER_RT_TEST_COMPILER_CFLAGS= $tc/stage2_unix/toolchain.ninja | sed -e 's/\\\$ /:/g' | grep -o COMPILER_RT_TEST_COMPILER_CFLAGS='[^ ]*' | sed -e 's/:/ /g' | cut -d= -f2-) +cflags="$cflags -Wno-unused-command-line-argument" + +mkdir -p $tc/check-libcxx +cat > $tc/check-libcxx/lit.site.cfg < $tc/check-libcxx/cmake-bridge.cfg < +Date: Mon, 3 Mar 2025 20:07:20 -0800 +Subject: [PATCH] PFP fixes + +--- + runtime/platform/allocation.h | 4 ++++ + runtime/vm/dart.cc | 5 ++--- + runtime/vm/isolate.cc | 5 ++++- + runtime/vm/isolate.h | 4 ++-- + runtime/vm/timeline.cc | 4 ++++ + 5 files changed, 16 insertions(+), 6 deletions(-) + +diff --git a/runtime/platform/allocation.h b/runtime/platform/allocation.h +index 26001ac54f2..5bc2a10960b 100644 +--- a/runtime/platform/allocation.h ++++ b/runtime/platform/allocation.h +@@ -40,6 +40,10 @@ class MallocAllocated { + public: + MallocAllocated() {} + ++ // Placement new. ++ void* operator new(size_t size, void *p) { return p; } ++ void* operator new[](size_t size, void *p) { return p; } ++ + // Intercept operator new to produce clearer error messages when we run out + // of memory. Don't do this when running under ASAN so it can continue to + // check malloc/new/new[] are paired with free/delete/delete[] respectively. +diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc +index 490549316ef..d78779a511a 100644 +--- a/runtime/vm/dart.cc ++++ b/runtime/vm/dart.cc +@@ -165,12 +165,12 @@ class DartInitializationState : public AllStatic { + std::atomic DartInitializationState::state_ = {kUnInitialized}; + std::atomic DartInitializationState::in_use_count_ = {0}; + +-#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + static void CheckOffsets() { + #if !defined(IS_SIMARM_HOST64) + // These offsets are embedded in precompiled instructions. We need the + // compiler and the runtime to agree. + bool ok = true; ++ (void)ok; + #define CHECK_OFFSET(expr, offset) \ + if ((expr) != (offset)) { \ + OS::PrintErr("%s got %" Pd ", %s expected %" Pd "\n", #expr, \ +@@ -226,7 +226,6 @@ static void CheckOffsets() { + } \ + } + #define CHECK_CONSTANT(Class, Name) CHECK_OFFSET(Class::Name, Class##_##Name); +-#endif // defined(DART_PRECOMPILED_RUNTIME) + + COMMON_OFFSETS_LIST(CHECK_FIELD, CHECK_ARRAY, CHECK_SIZEOF, + CHECK_ARRAY_SIZEOF, CHECK_PAYLOAD_SIZEOF, CHECK_RANGE, +@@ -258,8 +257,8 @@ static void CheckOffsets() { + #endif // defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + + char* Dart::DartInit(const Dart_InitializeParams* params) { +-#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + CheckOffsets(); ++#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + #elif defined(ARCH_IS_64_BIT) != defined(TARGET_ARCH_IS_64_BIT) + return Utils::StrDup( + "JIT cannot simulate target architecture with different word size than " +diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc +index 11cb004f448..c48623b8907 100644 +--- a/runtime/vm/isolate.cc ++++ b/runtime/vm/isolate.cc +@@ -448,6 +448,8 @@ IsolateGroup::~IsolateGroup() { + delete debugger_; + debugger_ = nullptr; + #endif ++ ++ delete object_store_; + } + + void IsolateGroup::RegisterIsolate(Isolate* isolate) { +@@ -1112,7 +1114,8 @@ bool Isolate::SendInternalLibMessage(Dart_Port main_port, + } + + void IsolateGroup::set_object_store(ObjectStore* object_store) { +- object_store_.reset(object_store); ++ delete object_store_; ++ object_store_ = object_store; + } + + class IsolateMessageHandler : public MessageHandler { +diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h +index 9f5d27fb404..5b501b3fc4f 100644 +--- a/runtime/vm/isolate.h ++++ b/runtime/vm/isolate.h +@@ -506,7 +506,7 @@ class IsolateGroup : public IntrusiveDListEntry { + void DropOriginalClassTable(); + + StoreBuffer* store_buffer() const { return store_buffer_.get(); } +- ObjectStore* object_store() const { return object_store_.get(); } ++ ObjectStore* object_store() const { return object_store_; } + Mutex* symbols_mutex() { return &symbols_mutex_; } + Mutex* type_canonicalization_mutex() { return &type_canonicalization_mutex_; } + Mutex* type_arguments_canonicalization_mutex() { +@@ -819,7 +819,7 @@ class IsolateGroup : public IntrusiveDListEntry { + // Accessed from generated code. + ClassTable* class_table_; + AcqRelAtomic cached_class_table_table_; +- std::unique_ptr object_store_; ++ ObjectStore* object_store_; + // End accessed from generated code. + + ClassTableAllocator class_table_allocator_; +diff --git a/runtime/vm/timeline.cc b/runtime/vm/timeline.cc +index 211f0e28741..ba07afac27c 100644 +--- a/runtime/vm/timeline.cc ++++ b/runtime/vm/timeline.cc +@@ -1595,6 +1595,10 @@ TimelineEventFixedBufferRecorder::TimelineEventFixedBufferRecorder( + if (memory_ == nullptr) { + OUT_OF_MEMORY(); + } ++ for (intptr_t i = 0; i != num_blocks_; ++i) { ++ new (reinterpret_cast(memory_->address()) + ++ i * sizeof(TimelineEventBlock)) TimelineEventBlock(i); ++ } + blocks_ = reinterpret_cast(memory_->address()); + } + +-- +2.48.1 + From 725376b45c8504946fbe5ed26dd13584d019ef9a Mon Sep 17 00:00:00 2001 From: Qinkun Bao Date: Fri, 4 Apr 2025 18:00:15 +0000 Subject: [PATCH 14/18] Implement a simple e2e test for PFP. --- .../cmake/Modules/AllSupportedArchDefs.cmake | 4 ++ compiler-rt/cmake/config-ix.cmake | 4 ++ compiler-rt/test/CMakeLists.txt | 3 + compiler-rt/test/pfp/CMakeLists.txt | 55 +++++++++++++++++++ compiler-rt/test/pfp/lit.cfg.py | 37 +++++++++++++ compiler-rt/test/pfp/lit.site.cfg.py.in | 16 ++++++ compiler-rt/test/pfp/use-after-free-fixed.cpp | 43 +++++++++++++++ compiler-rt/test/pfp/use-after-free.cpp | 45 +++++++++++++++ 8 files changed, 207 insertions(+) create mode 100644 compiler-rt/test/pfp/CMakeLists.txt create mode 100644 compiler-rt/test/pfp/lit.cfg.py create mode 100644 compiler-rt/test/pfp/lit.site.cfg.py.in create mode 100644 compiler-rt/test/pfp/use-after-free-fixed.cpp create mode 100644 compiler-rt/test/pfp/use-after-free.cpp diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index 2683259e93e37..1ad927f4677e2 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -119,3 +119,7 @@ endif() if (WIN32) set(ALL_ORC_SUPPORTED_ARCH ${X86_64}) endif() + +if (UNIX) + set(ALL_PFP_SUPPORTED_ARCH ${X86_64} ${ARM64}) +endif() diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index cf729c3adb1f5..8b45685132ab3 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -686,6 +686,9 @@ if(APPLE) list_intersect(ORC_SUPPORTED_ARCH ALL_ORC_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(PFP_SUPPORTED_ARCH + ALL_PFP_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) else() # Architectures supported by compiler-rt libraries. @@ -721,6 +724,7 @@ else() filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH}) filter_available_targets(NSAN_SUPPORTED_ARCH ${ALL_NSAN_SUPPORTED_ARCH}) filter_available_targets(ORC_SUPPORTED_ARCH ${ALL_ORC_SUPPORTED_ARCH}) + filter_available_targets(PFP_SUPPORTED_ARCH ${ALL_PFP_SUPPORTED_ARCH}) endif() if (MSVC) diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index fad5b7e03925e..9dd90aef7ebd9 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -106,6 +106,9 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS) # ShadowCallStack does not yet provide a runtime with compiler-rt, the tests # include their own minimal runtime add_subdirectory(shadowcallstack) + # PointerFieldProtection does not yet provide a runtime with compiler-rt, the tests + # include their own minimal runtime + add_subdirectory(pfp) endif() # Now that we've traversed all the directories and know all the lit testsuites, diff --git a/compiler-rt/test/pfp/CMakeLists.txt b/compiler-rt/test/pfp/CMakeLists.txt new file mode 100644 index 0000000000000..11e9985ddd3c4 --- /dev/null +++ b/compiler-rt/test/pfp/CMakeLists.txt @@ -0,0 +1,55 @@ +set(PFP_TESTSUITES) +set(PFP_TEST_DEPS + ${SANITIZER_COMMON_LIT_TEST_DEPS}) + + +macro(add_pfp_testsuite arch lld thinlto compilerrt libunwind) + set(PFP_TEST_TARGET_ARCH ${arch}) + get_test_cc_for_arch(${arch} PFP_TEST_TARGET_CC PFP_TEST_TARGET_CFLAGS) + + string(TOUPPER ${arch} CONFIG_NAME) + + if (${thinlto}) + set(CONFIG_NAME "thinlto-${CONFIG_NAME}") + list(APPEND PFP_TEST_DEPS LTO) + endif() + if (${libunwind}) + set(CONFIG_NAME "libunwind-${CONFIG_NAME}") + list(APPEND PFP_TEST_DEPS unwind) + endif() + if (${lld}) + set(CONFIG_NAME "lld-${CONFIG_NAME}") + if (TARGET lld) + list(APPEND PFP_TEST_DEPS lld) + endif() + endif() + if (${compilerrt}) + set(CONFIG_NAME "compiler-rt-${CONFIG_NAME}") + list(APPEND PFP_TEST_DEPS compiler-rt) + endif() + set(PFP_TEST_USE_THINLTO ${thinlto}) + set(PFP_TEST_USE_LLD ${lld}) + + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py) + list(APPEND PFP_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}) +endmacro() + +set(PFP_TEST_ARCH ${PFP_SUPPORTED_ARCH}) + + +foreach(arch ${PFP_TEST_ARCH}) + if (arch STREQUAL ${X86_64}) + add_pfp_testsuite(${arch} False False False False) + endif() + if (arch STREQUAL ${ARM64}) + add_pfp_testsuite(${arch} True True True True) + endif() +endforeach() + +message(STATUS "PFP tests. " "Arch: ${PFP_TEST_ARCH} Deps: ${PFP_TEST_DEPS}") +add_lit_testsuite(check-pfp "Running the PointerFieldProtection tests" + ${PFP_TESTSUITES} + DEPENDS ${PFP_TEST_DEPS}) + diff --git a/compiler-rt/test/pfp/lit.cfg.py b/compiler-rt/test/pfp/lit.cfg.py new file mode 100644 index 0000000000000..39a6252236509 --- /dev/null +++ b/compiler-rt/test/pfp/lit.cfg.py @@ -0,0 +1,37 @@ +# -*- Python -*- + +import os + +from lit.llvm import llvm_config +from lit.llvm.subst import ToolSubst, FindTool + +# Setup config name. +config.name = "pfp" + config.name_suffix + +# Default test suffixes. +config.suffixes = [".c", ".cpp"] + +if config.host_os not in ["Linux"]: + config.unsupported = True + +# Setup source root. +config.test_source_root = os.path.dirname(__file__) +# Setup default compiler flags used with -fsanitize=memory option. +clang_cflags = [config.target_cflags] + config.debug_info_flags +clang_cxxflags = config.cxx_mode_flags + clang_cflags +clang_pfp_tagged_common_cflags = clang_cflags + [ + "-fexperimental-pointer-field-protection=tagged" +] + + +clang_pfp_cxxflags = config.cxx_mode_flags + clang_pfp_tagged_common_cflags +if config.target_arch == 'aarch64': + clang_pfp_cxxflags = clang_pfp_cxxflags + ["-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind -static-libgcc"] + + +def build_invocation(compile_flags): + return " " + " ".join([config.clang] + compile_flags) + " " + + +config.substitutions.append(("%clangxx ", build_invocation(clang_cxxflags))) +config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags))) diff --git a/compiler-rt/test/pfp/lit.site.cfg.py.in b/compiler-rt/test/pfp/lit.site.cfg.py.in new file mode 100644 index 0000000000000..ec39759982a1d --- /dev/null +++ b/compiler-rt/test/pfp/lit.site.cfg.py.in @@ -0,0 +1,16 @@ +@LIT_SITE_CFG_IN_HEADER@ + +# Tool-specific config options. +config.name_suffix = "@PFP_TEST_CONFIG_SUFFIX@" +config.target_cflags = "@PFP_TEST_TARGET_CFLAGS@" +config.target_arch = "@PFP_TEST_TARGET_ARCH@" +config.use_lld = @PFP_TEST_USE_LLD@ +config.use_thinlto = @PFP_TEST_USE_THINLTO@ +config.libunwind_shared = "@LIBUNWIND_ENABLE_SHARED@" +config.libunwind_install_dir = "@LLVM_BINARY_DIR@/@LIBUNWIND_INSTALL_LIBRARY_DIR@" + +# Load common config for all compiler-rt lit tests. +lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") + +# Load tool-specific config that would do the real work. +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py") diff --git a/compiler-rt/test/pfp/use-after-free-fixed.cpp b/compiler-rt/test/pfp/use-after-free-fixed.cpp new file mode 100644 index 0000000000000..2fe21111cdd49 --- /dev/null +++ b/compiler-rt/test/pfp/use-after-free-fixed.cpp @@ -0,0 +1,43 @@ +// RUN: %clangxx_pfp %s -o %t1 +// RUN: %run %t1 2>&1 +// RUN: %clangxx %s -o %t2 +// RUN: %run %t2 2>&1 + +#include + +// Struct1.ptr and Struct2.ptr have different locks. +struct Struct1 { + int *ptr; + Struct1() : num(1), ptr(&num) {} + +private: + int num; +}; + +struct Struct2 { + int *ptr; + Struct2() : num(2), ptr(&num) {} + +private: + int num; +}; + +Struct1 *new_object1() { + Struct1 *ptr = new Struct1; + return ptr; +} + +Struct2 *new_object2() { + Struct2 *ptr = new Struct2; + return ptr; +} + +int main() { + Struct1 *obj1 = new_object1(); + Struct2 *obj2 = new_object2(); + std::cout << "Struct2: " << *(obj2->ptr) << "\n"; + std::cout << "Struct1: " << *(obj1->ptr) << "\n"; + delete obj1; + delete obj2; + return 0; +} diff --git a/compiler-rt/test/pfp/use-after-free.cpp b/compiler-rt/test/pfp/use-after-free.cpp new file mode 100644 index 0000000000000..17b7661b2a904 --- /dev/null +++ b/compiler-rt/test/pfp/use-after-free.cpp @@ -0,0 +1,45 @@ +// RUN: %clangxx_pfp %s -o %t1 +// RUN: %expect_crash %run %t1 2>&1 +// RUN: %clangxx %s -o %t2 +// RUN: %run %t2 2>&1 + +#include + +// Struct1.ptr and Struct2.ptr have different locks. +struct Struct1 { + int *ptr; + Struct1() : num(1), ptr(&num) {} + +private: + int num; +}; + +struct Struct2 { + int *ptr; + Struct2() : num(2), ptr(&num) {} + +private: + int num; +}; + +Struct1 *new_object1() { + Struct1 *ptr = new Struct1; + return ptr; +} + +Struct2 *new_object2() { + Struct2 *ptr = new Struct2; + return ptr; +} + +int main() { + Struct1 *obj1 = new_object1(); + delete obj1; + // obj1's memory will be reused. + Struct2 *obj2 = new_object2(); + std::cout << "Struct2: " << *(obj2->ptr) << "\n"; + // Uses a wrong lock. The Program should crash when "-fexperimental-pointer-field-protection=tagged". + std::cout << "Struct1: " << *(obj1->ptr) << "\n"; + delete obj2; + return 0; +} From 858863e1e603dfc1443d0ccca75463fc64c48a4c Mon Sep 17 00:00:00 2001 From: Qinkun Bao Date: Sat, 5 Apr 2025 00:12:44 +0000 Subject: [PATCH 15/18] address comments --- .../cmake/Modules/AllSupportedArchDefs.cmake | 2 +- compiler-rt/test/CMakeLists.txt | 2 -- compiler-rt/test/pfp/CMakeLists.txt | 33 ++++--------------- compiler-rt/test/pfp/lit.cfg.py | 19 ++++++----- 4 files changed, 18 insertions(+), 38 deletions(-) diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index 1ad927f4677e2..e75115fbbee9a 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -120,6 +120,6 @@ if (WIN32) set(ALL_ORC_SUPPORTED_ARCH ${X86_64}) endif() -if (UNIX) +if (OS_NAME MATCHES "Linux") set(ALL_PFP_SUPPORTED_ARCH ${X86_64} ${ARM64}) endif() diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index 9dd90aef7ebd9..c4d2d181fd5df 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -106,8 +106,6 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS) # ShadowCallStack does not yet provide a runtime with compiler-rt, the tests # include their own minimal runtime add_subdirectory(shadowcallstack) - # PointerFieldProtection does not yet provide a runtime with compiler-rt, the tests - # include their own minimal runtime add_subdirectory(pfp) endif() diff --git a/compiler-rt/test/pfp/CMakeLists.txt b/compiler-rt/test/pfp/CMakeLists.txt index 11e9985ddd3c4..631b73cc2165f 100644 --- a/compiler-rt/test/pfp/CMakeLists.txt +++ b/compiler-rt/test/pfp/CMakeLists.txt @@ -1,34 +1,19 @@ set(PFP_TESTSUITES) -set(PFP_TEST_DEPS - ${SANITIZER_COMMON_LIT_TEST_DEPS}) +set(PFP_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} unwind lld compiler-rt) +set(PFP_TEST_USE_LLD True) - -macro(add_pfp_testsuite arch lld thinlto compilerrt libunwind) +macro(add_pfp_testsuite arch thinlto) set(PFP_TEST_TARGET_ARCH ${arch}) get_test_cc_for_arch(${arch} PFP_TEST_TARGET_CC PFP_TEST_TARGET_CFLAGS) string(TOUPPER ${arch} CONFIG_NAME) if (${thinlto}) + set(PFP_TEST_USE_THINLTO ${thinlto}) set(CONFIG_NAME "thinlto-${CONFIG_NAME}") list(APPEND PFP_TEST_DEPS LTO) - endif() - if (${libunwind}) - set(CONFIG_NAME "libunwind-${CONFIG_NAME}") - list(APPEND PFP_TEST_DEPS unwind) - endif() - if (${lld}) - set(CONFIG_NAME "lld-${CONFIG_NAME}") - if (TARGET lld) - list(APPEND PFP_TEST_DEPS lld) - endif() - endif() - if (${compilerrt}) - set(CONFIG_NAME "compiler-rt-${CONFIG_NAME}") - list(APPEND PFP_TEST_DEPS compiler-rt) endif() set(PFP_TEST_USE_THINLTO ${thinlto}) - set(PFP_TEST_USE_LLD ${lld}) configure_lit_site_cfg( ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in @@ -38,17 +23,11 @@ endmacro() set(PFP_TEST_ARCH ${PFP_SUPPORTED_ARCH}) - foreach(arch ${PFP_TEST_ARCH}) - if (arch STREQUAL ${X86_64}) - add_pfp_testsuite(${arch} False False False False) - endif() - if (arch STREQUAL ${ARM64}) - add_pfp_testsuite(${arch} True True True True) - endif() + add_pfp_testsuite(${arch} False) + add_pfp_testsuite(${arch} True) endforeach() -message(STATUS "PFP tests. " "Arch: ${PFP_TEST_ARCH} Deps: ${PFP_TEST_DEPS}") add_lit_testsuite(check-pfp "Running the PointerFieldProtection tests" ${PFP_TESTSUITES} DEPENDS ${PFP_TEST_DEPS}) diff --git a/compiler-rt/test/pfp/lit.cfg.py b/compiler-rt/test/pfp/lit.cfg.py index 39a6252236509..6c2773ea07e53 100644 --- a/compiler-rt/test/pfp/lit.cfg.py +++ b/compiler-rt/test/pfp/lit.cfg.py @@ -11,9 +11,6 @@ # Default test suffixes. config.suffixes = [".c", ".cpp"] -if config.host_os not in ["Linux"]: - config.unsupported = True - # Setup source root. config.test_source_root = os.path.dirname(__file__) # Setup default compiler flags used with -fsanitize=memory option. @@ -25,13 +22,19 @@ clang_pfp_cxxflags = config.cxx_mode_flags + clang_pfp_tagged_common_cflags -if config.target_arch == 'aarch64': - clang_pfp_cxxflags = clang_pfp_cxxflags + ["-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind -static-libgcc"] +clang_pfp_cxxflags = clang_pfp_cxxflags + ["-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind -static-libgcc"] -def build_invocation(compile_flags): - return " " + " ".join([config.clang] + compile_flags) + " " +def build_invocation(compile_flags, with_lto=False): + lto_flags = [] + if with_lto and config.lto_supported: + lto_flags += config.lto_flags + + return " " + " ".join([config.clang] + lto_flags + compile_flags) + " " config.substitutions.append(("%clangxx ", build_invocation(clang_cxxflags))) -config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags))) +if config.target_arch == 'aarch64': + config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags, True))) +else: + config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags))) From c2a6405bf4e87c93e39713d28a40cc20e19895a9 Mon Sep 17 00:00:00 2001 From: Qinkun Bao Date: Sat, 5 Apr 2025 00:49:15 +0000 Subject: [PATCH 16/18] Remove PFP_TEST_USE_LLD --- compiler-rt/test/pfp/CMakeLists.txt | 1 - compiler-rt/test/pfp/lit.site.cfg.py.in | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler-rt/test/pfp/CMakeLists.txt b/compiler-rt/test/pfp/CMakeLists.txt index 631b73cc2165f..737955998e3de 100644 --- a/compiler-rt/test/pfp/CMakeLists.txt +++ b/compiler-rt/test/pfp/CMakeLists.txt @@ -1,6 +1,5 @@ set(PFP_TESTSUITES) set(PFP_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} unwind lld compiler-rt) -set(PFP_TEST_USE_LLD True) macro(add_pfp_testsuite arch thinlto) set(PFP_TEST_TARGET_ARCH ${arch}) diff --git a/compiler-rt/test/pfp/lit.site.cfg.py.in b/compiler-rt/test/pfp/lit.site.cfg.py.in index ec39759982a1d..7bc6331a0a376 100644 --- a/compiler-rt/test/pfp/lit.site.cfg.py.in +++ b/compiler-rt/test/pfp/lit.site.cfg.py.in @@ -4,7 +4,7 @@ config.name_suffix = "@PFP_TEST_CONFIG_SUFFIX@" config.target_cflags = "@PFP_TEST_TARGET_CFLAGS@" config.target_arch = "@PFP_TEST_TARGET_ARCH@" -config.use_lld = @PFP_TEST_USE_LLD@ +config.use_lld = True config.use_thinlto = @PFP_TEST_USE_THINLTO@ config.libunwind_shared = "@LIBUNWIND_ENABLE_SHARED@" config.libunwind_install_dir = "@LLVM_BINARY_DIR@/@LIBUNWIND_INSTALL_LIBRARY_DIR@" From 98295661ce32b539055fb47d87ce3fe33cf6a22f Mon Sep 17 00:00:00 2001 From: Qinkun Bao Date: Sat, 5 Apr 2025 01:08:10 +0000 Subject: [PATCH 17/18] pass config.use_thinlto to build_invocation --- compiler-rt/test/pfp/lit.cfg.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compiler-rt/test/pfp/lit.cfg.py b/compiler-rt/test/pfp/lit.cfg.py index 6c2773ea07e53..2d58972804039 100644 --- a/compiler-rt/test/pfp/lit.cfg.py +++ b/compiler-rt/test/pfp/lit.cfg.py @@ -34,7 +34,4 @@ def build_invocation(compile_flags, with_lto=False): config.substitutions.append(("%clangxx ", build_invocation(clang_cxxflags))) -if config.target_arch == 'aarch64': - config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags, True))) -else: - config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags))) +config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags, config.use_thinlto))) From 162b0d07e0ebfcff9f3d6e69232b7d9c530ca299 Mon Sep 17 00:00:00 2001 From: Qinkun Bao Date: Wed, 9 Apr 2025 21:20:04 +0000 Subject: [PATCH 18/18] make lld as optional deps --- compiler-rt/test/pfp/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler-rt/test/pfp/CMakeLists.txt b/compiler-rt/test/pfp/CMakeLists.txt index 737955998e3de..1ee8eea8dbaac 100644 --- a/compiler-rt/test/pfp/CMakeLists.txt +++ b/compiler-rt/test/pfp/CMakeLists.txt @@ -1,5 +1,8 @@ set(PFP_TESTSUITES) -set(PFP_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} unwind lld compiler-rt) +set(PFP_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} unwind compiler-rt) +if(COMPILER_RT_HAS_LLD AND TARGET lld) + list(APPEND PFP_TEST_DEPS lld) +endif() macro(add_pfp_testsuite arch thinlto) set(PFP_TEST_TARGET_ARCH ${arch})