diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index af8c49e99a7ce..abba83e1ff9c4 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -183,6 +183,12 @@ struct TypeInfoChars { } }; +struct PFPField { + CharUnits structOffset; + CharUnits offset; + FieldDecl *field; +}; + /// Holds long-lived AST nodes (such as types and decls) that can be /// referred to throughout the semantic analysis of a file. class ASTContext : public RefCountedBase { @@ -3618,6 +3624,22 @@ OPT_LIST(V) StringRef getCUIDHash() const; + bool isPFPStruct(const RecordDecl *rec) const; + void findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, bool IncludeVBases) const; + bool hasPFPFields(QualType ty) const; + bool isPFPField(const FieldDecl *field) const; + + /// Returns whether this record's PFP fields (if any) are trivially + /// relocatable (i.e. may be memcpy'd). This may also return true if the + /// record does not have any PFP fields, so it may be necessary for the caller + /// to check for PFP fields, e.g. by calling hasPFPFields(). + bool arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const; + + llvm::SetVector PFPFieldsWithEvaluatedOffset; + void recordMemberDataPointerEvaluation(const ValueDecl *VD); + void recordOffsetOfEvaluation(const OffsetOfExpr *E); + private: /// All OMPTraitInfo objects live in this collection, one per /// `pragma omp [begin] declare variant` directive. diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 0999d8065e9f5..3d26c2f001812 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2460,6 +2460,12 @@ def CountedByOrNull : DeclOrTypeAttr { let LangOpts = [COnly]; } +def NoPointerFieldProtection : DeclOrTypeAttr { + let Spellings = [Clang<"no_field_protection">]; + let Subjects = SubjectList<[Field], ErrorDiag>; + let Documentation = [Undocumented]; +} + def SizedBy : DeclOrTypeAttr { let Spellings = [Clang<"sized_by">]; let Subjects = SubjectList<[Field], ErrorDiag>; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 05ce214935fad..d4ded24c5a87e 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -262,6 +262,9 @@ FEATURE(shadow_call_stack, FEATURE(tls, PP.getTargetInfo().isTLSSupported()) FEATURE(underlying_type, LangOpts.CPlusPlus) FEATURE(experimental_library, LangOpts.ExperimentalLibrary) +FEATURE(pointer_field_protection, + LangOpts.getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) // C11 features supported by other languages as extensions. EXTENSION(c_alignas, true) diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 3879cc7942877..8eacb6e066007 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -501,6 +501,9 @@ LANGOPT(RelativeCXXABIVTables, 1, 0, LANGOPT(OmitVTableRTTI, 1, 0, "Use an ABI-incompatible v-table layout that omits the RTTI component") +ENUM_LANGOPT(PointerFieldProtection, PointerFieldProtectionKind, 2, PointerFieldProtectionKind::None, + "Encode struct pointer fields to protect against UAF vulnerabilities") + LANGOPT(VScaleMin, 32, 0, "Minimum vscale value") LANGOPT(VScaleMax, 32, 0, "Maximum vscale value") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index e925e0f3b5d85..797a14038ba4b 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -365,6 +365,17 @@ class LangOptionsBase { BKey }; + enum class PointerFieldProtectionKind { + /// Pointer field protection disabled + None, + /// Pointer field protection enabled, allocator does not tag heap + /// allocations. + Untagged, + /// Pointer field protection enabled, allocator is expected to tag heap + /// allocations. + Tagged, + }; + enum class ThreadModelKind { /// POSIX Threads. POSIX, diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 1bf9f43f80986..142e13b3ee784 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -542,6 +542,7 @@ TYPE_TRAIT_2(__is_pointer_interconvertible_base_of, IsPointerInterconvertibleBas // Clang-only C++ Type Traits TYPE_TRAIT_1(__is_trivially_relocatable, IsTriviallyRelocatable, KEYCXX) +TYPE_TRAIT_1(__has_non_relocatable_fields, HasNonRelocatableFields, KEYCXX) TYPE_TRAIT_1(__is_trivially_equality_comparable, IsTriviallyEqualityComparable, KEYCXX) TYPE_TRAIT_1(__is_bounded_array, IsBoundedArray, KEYCXX) TYPE_TRAIT_1(__is_unbounded_array, IsUnboundedArray, KEYCXX) diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index a7fcb160d3867..c903e0554319e 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -2957,6 +2957,12 @@ defm experimental_omit_vtable_rtti : BoolFOption<"experimental-omit-vtable-rtti" NegFlag, BothFlags<[], [CC1Option], " the RTTI component from virtual tables">>; +def experimental_pointer_field_protection_EQ : Joined<["-"], "fexperimental-pointer-field-protection=">, Group, + Visibility<[ClangOption, CC1Option]>, + Values<"none,untagged,tagged">, NormalizedValuesScope<"LangOptions::PointerFieldProtectionKind">, + NormalizedValues<["None", "Untagged", "Tagged"]>, + MarshallingInfoEnum, "None">; + def fcxx_abi_EQ : Joined<["-"], "fc++-abi=">, Group, Visibility<[ClangOption, CC1Option]>, HelpText<"C++ ABI to use. This will override the target C++ ABI.">; diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index c9d1bea4c623a..c3cbfec2c93d3 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -79,6 +79,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" #include "llvm/Support/Capacity.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MD5.h" @@ -14948,3 +14949,97 @@ bool ASTContext::useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl, ThunksToBeAbbreviated[VirtualMethodDecl] = std::move(SimplifiedThunkNames); return Result; } + +bool ASTContext::arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::Tagged) + return !isa(RD) || + cast(RD)->hasTrivialDestructor(); + return true; +} + +bool ASTContext::isPFPStruct(const RecordDecl *rec) const { + if (getLangOpts().getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) + if (auto *cxxRec = dyn_cast(rec)) + return !cxxRec->isStandardLayout(); + return false; +} + +void ASTContext::findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, + bool IncludeVBases) const { + if (auto *AT = getAsConstantArrayType(Ty)) { + if (auto *ElemDecl = AT->getElementType()->getAsCXXRecordDecl()) { + const ASTRecordLayout &ElemRL = getASTRecordLayout(ElemDecl); + for (unsigned i = 0; i != AT->getSize(); ++i) { + findPFPFields(AT->getElementType(), Offset + i * ElemRL.getSize(), + Fields, true); + } + } + } + auto *Decl = Ty->getAsCXXRecordDecl(); + if (!Decl) + return; + const ASTRecordLayout &RL = getASTRecordLayout(Decl); + for (FieldDecl *field : Decl->fields()) { + CharUnits fieldOffset = + Offset + toCharUnitsFromBits(RL.getFieldOffset(field->getFieldIndex())); + if (isPFPField(field)) + Fields.push_back({Offset, fieldOffset, field}); + findPFPFields(field->getType(), fieldOffset, Fields, true); + } + for (auto &Base : Decl->bases()) { + if (Base.isVirtual()) + continue; + CharUnits BaseOffset = + Offset + RL.getBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + if (IncludeVBases) { + for (auto &Base : Decl->vbases()) { + CharUnits BaseOffset = + Offset + RL.getVBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + } +} + +bool ASTContext::hasPFPFields(QualType ty) const { + std::vector pfpFields; + findPFPFields(ty, CharUnits::Zero(), pfpFields, true); + return !pfpFields.empty(); +} + +bool ASTContext::isPFPField(const FieldDecl *field) const { + if (!isPFPStruct(field->getParent())) + return false; + return field->getType()->isPointerType() && + !field->hasAttr(); +} + +void ASTContext::recordMemberDataPointerEvaluation(const ValueDecl *VD) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None) + return; + auto *FD = dyn_cast(VD); + if (!FD) + FD = cast(cast(VD)->chain().back()); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} + +void ASTContext::recordOffsetOfEvaluation(const OffsetOfExpr *E) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None || + E->getNumComponents() == 0) + return; + OffsetOfNode Comp = E->getComponent(E->getNumComponents() - 1); + if (Comp.getKind() != OffsetOfNode::Field) + return; + FieldDecl *FD = Comp.getField(); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 95da7b067b459..51f319fd26f20 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -14932,6 +14932,7 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( } bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) { + Info.Ctx.recordOffsetOfEvaluation(OOE); CharUnits Result; unsigned n = OOE->getNumComponents(); if (n == 0) diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp index 08798219c0b83..ca7ccc7f2bb5a 100644 --- a/clang/lib/AST/Type.cpp +++ b/clang/lib/AST/Type.cpp @@ -2852,7 +2852,9 @@ bool QualType::isTriviallyRelocatableType(const ASTContext &Context) const { } else if (!BaseElementType->isObjectType()) { return false; } else if (const auto *RD = BaseElementType->getAsRecordDecl()) { - return RD->canPassInRegisters(); + return RD->canPassInRegisters() && + (Context.arePFPFieldsTriviallyRelocatable(RD) || + !Context.hasPFPFields(BaseElementType)); } else if (BaseElementType.isTriviallyCopyableType(Context)) { return true; } else { diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index 3982ca3b50604..1382b4a9edfd4 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -2096,6 +2096,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T, case attr::ExtVectorType: OS << "ext_vector_type"; break; + case attr::NoPointerFieldProtection: + OS << "no_field_protection"; + break; } OS << "))"; } diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index 7aa77e55dbfcc..9d824231d02cb 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1298,7 +1298,8 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. -static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, +static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy, + llvm::Type *Ty, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src.getElementType(); @@ -1306,6 +1307,57 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, if (SrcTy == Ty) return CGF.Builder.CreateLoad(Src); + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct in registers has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + CGF.getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto LoadCoercedField = [&](CharUnits Offset, + llvm::Type *FieldType) -> llvm::Value * { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *FieldVal = CGF.Builder.CreateLoad(fieldAddr); + if (isa(FieldType)) + FieldVal = CGF.Builder.CreatePtrToInt(FieldVal, FieldType); + PFPFields.erase(PFPFields.begin()); + return FieldVal; + } + auto FieldAddr = CGF.Builder + .CreateConstInBoundsByteGEP( + Src.withElementType(CGF.Int8Ty), Offset) + .withElementType(FieldType); + return CGF.Builder.CreateLoad(FieldAddr); + }; + if (isa(Ty) || isa(Ty)) { + auto Addr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *Val = CGF.Builder.CreateLoad(Addr); + if (isa(Ty)) + Val = CGF.Builder.CreatePtrToInt(Val, Ty); + return Val; + } + if (auto *AT = dyn_cast(Ty)) { + auto *ET = AT->getElementType(); + CharUnits wordSize = CGF.getContext().toCharUnitsFromBits( + CGF.CGM.getDataLayout().getTypeSizeInBits(ET)); + CharUnits Offset = CharUnits::Zero(); + llvm::Value *Val = llvm::UndefValue::get(AT); + for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += wordSize) + Val = CGF.Builder.CreateInsertValue(Val, LoadCoercedField(Offset, ET), i); + return Val; + } + auto *ST = cast(Ty); + llvm::Value *Val = llvm::UndefValue::get(ST); + auto *SL = CGF.CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + Val = CGF.Builder.CreateInsertValue( + Val, LoadCoercedField(Offset, ST->getElementType(i)), i); + } + return Val; + } + llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); if (llvm::StructType *SrcSTy = dyn_cast(SrcTy)) { @@ -1374,7 +1426,9 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, return CGF.Builder.CreateLoad(Tmp); } -void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, +void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, + QualType SrcFETy, + Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile) { if (!DstSize) @@ -1395,6 +1449,52 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, } } + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct passed by value has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = EmitAddressOfPFPField(Dst, PFPFields[0]); + if (isa(FieldVal->getType())) + FieldVal = Builder.CreateIntToPtr(FieldVal, VoidPtrTy); + Builder.CreateStore(FieldVal, fieldAddr); + PFPFields.erase(PFPFields.begin()); + } else { + auto fieldAddr = + Builder + .CreateConstInBoundsByteGEP(Dst.withElementType(Int8Ty), Offset) + .withElementType(FieldVal->getType()); + Builder.CreateStore(FieldVal, fieldAddr); + } + }; + + if (isa(SrcTy) || isa(SrcTy)) { + if (isa(SrcTy)) + Src = Builder.CreateIntToPtr(Src, VoidPtrTy); + auto Addr = EmitAddressOfPFPField(Dst, PFPFields[0]); + Builder.CreateStore(Src, Addr); + } else if (auto *at = dyn_cast(SrcTy)) { + auto *et = at->getElementType(); + CharUnits wordSize = getContext().toCharUnitsFromBits( + CGM.getDataLayout().getTypeSizeInBits(et)); + CharUnits Offset = CharUnits::Zero(); + for (unsigned i = 0; i != at->getNumElements(); ++i, Offset += wordSize) + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } else { + auto *ST = cast(SrcTy); + auto *SL = CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } + } + return; + } + if (SrcSize.isScalable() || SrcSize <= DstSize) { if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() && SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) { @@ -3347,7 +3447,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, auto AI = Fn->getArg(FirstIRArg); AI->setName(Arg->getName() + ".coerce"); CreateCoercedStore( - AI, Ptr, + AI, Ty, Ptr, llvm::TypeSize::getFixed( getContext().getTypeSizeInChars(Ty).getQuantity() - ArgI.getDirectOffset()), @@ -3969,7 +4069,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, // If the value is offset in memory, apply the offset now. Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); + RV = CreateCoercedLoad(V, RetTy, RetAI.getCoerceToType(), *this); } // In ARC, end functions that return a retainable type with a call @@ -4020,6 +4120,7 @@ void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, auto eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + RetTy, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -5550,7 +5651,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); llvm::Value *Load = - CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); + CreateCoercedLoad(Src, I->Ty, ArgInfo.getCoerceToType(), *this); if (CallInfo.isCmseNSCall()) { // For certain parameter types, clear padding bits, as they may reveal @@ -5611,6 +5712,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + I->Ty, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -6105,7 +6207,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the value is offset in memory, apply the offset now. Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); CreateCoercedStore( - CI, StorePtr, + CI, RetTy, StorePtr, llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()), DestIsVolatile); } diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 98c93b5bb4883..ae1d78baed16a 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -594,12 +594,20 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, isBaseVirtual); } -static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { +static bool isMemcpyEquivalentSpecialMember(CodeGenModule &CGM, + const CXXMethodDecl *D) { auto *CD = dyn_cast(D); if (!(CD && CD->isCopyOrMoveConstructor()) && !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(D->getParent()) && + CGM.getContext().hasPFPFields( + QualType(D->getParent()->getTypeForDecl(), 0))) + return false; + // We can emit a memcpy for a trivial copy or move constructor/assignment. if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) return true; @@ -664,7 +672,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); if (BaseElementTy.isPODType(CGF.getContext()) || - (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { + (CE && isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor()))) { unsigned SrcArgIndex = CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); llvm::Value *SrcPtr @@ -928,6 +936,11 @@ namespace { Qualifiers Qual = F->getType().getQualifiers(); if (Qual.hasVolatile() || Qual.hasObjCLifetime()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGF.getContext().arePFPFieldsTriviallyRelocatable(ClassDecl) && + CGF.getContext().isPFPField(F)) + return false; return true; } @@ -1064,7 +1077,8 @@ namespace { CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); // Bail out on non-memcpyable, not-trivially-copyable members. - if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && + if (!(CE && + isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor())) && !(FieldType.isTriviallyCopyableType(CGF.getContext()) || FieldType->isReferenceType())) return false; @@ -1174,7 +1188,7 @@ namespace { return nullptr; } else if (CXXMemberCallExpr *MCE = dyn_cast(S)) { CXXMethodDecl *MD = dyn_cast(MCE->getCalleeDecl()); - if (!(MD && isMemcpyEquivalentSpecialMember(MD))) + if (!(MD && isMemcpyEquivalentSpecialMember(CGF.CGM, MD))) return nullptr; MemberExpr *IOA = dyn_cast(MCE->getImplicitObjectArgument()); if (!IOA) @@ -2142,7 +2156,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // If this is a trivial constructor, emit a memcpy now before we lose // the alignment information on the argument. // FIXME: It would be better to preserve alignment information into CallArg. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); const Expr *Arg = E->getArg(0); @@ -2201,6 +2215,22 @@ void CodeGenFunction::EmitCXXConstructorCall( EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); + // When initializing an object that has pointer field protection and whose + // fields are not trivially relocatable we must initialize any pointer fields + // to a valid signed pointer (any pointer value will do, but we just use null + // pointers). This is because if the object is subsequently copied, its copy + // constructor will need to read and authenticate any pointer fields in order + // to copy the object to a new address, which will fail if the pointers are + // uninitialized. + if (!getContext().arePFPFieldsTriviallyRelocatable(D->getParent())) { + std::vector PFPFields; + getContext().findPFPFields(QualType(ClassDecl->getTypeForDecl(), 0), + CharUnits::Zero(), PFPFields, Type != Ctor_Base); + for (auto &Field : PFPFields) + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), + EmitAddressOfPFPField(This, Field)); + } + if (D->isTrivial() && D->isDefaultConstructor()) { assert(Args.size() == 1 && "trivial default ctor with args"); return; @@ -2209,7 +2239,7 @@ void CodeGenFunction::EmitCXXConstructorCall( // If this is a trivial constructor, just emit what's needed. If this is a // union copy constructor, we must emit a memcpy, because the AST does not // model that copy. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); Address Src = makeNaturalAddressForPointer( @@ -2975,7 +3005,15 @@ void CodeGenFunction::EmitForwardingCallToLambda( QualType resultType = FPT->getReturnType(); ReturnValueSlot returnSlot; if (!resultType->isVoidType() && - calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && + (calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect || + // With pointer field protection, we need to set up the return slot when + // returning an object with trivial ABI to avoid the memcpy that would + // otherwise be generated by the call to EmitReturnOfRValue() below, as + // that may corrupt the pointer signature. It doesn't hurt to do this all + // the time as it results in slightly simpler codegen. + (resultType->isRecordType() && + resultType->getAsCXXRecordDecl() + ->hasTrivialCopyConstructorForCall())) && !hasScalarEvaluationKind(calleeFnInfo->getReturnType())) returnSlot = ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(), diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 5943ff9294e1a..ce8875897f7d1 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -4904,10 +4904,16 @@ static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, return emitAddrOfZeroSizeField(CGF, base, field); const RecordDecl *rec = field->getParent(); - unsigned idx = CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); + if (CGF.getContext().isPFPField(field)) { + const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(rec); + auto Offset = CGF.getContext().toCharUnitsFromBits( + RL.getFieldOffset(field->getFieldIndex())); + return CGF.EmitAddressOfPFPField(base, field, Offset); + } + return CGF.Builder.CreateStructGEP(base, idx, field->getName()); } diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index c8bdda375d1b1..bdff2bceda93b 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -137,7 +137,7 @@ class AggExprEmitter : public StmtVisitor { if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) { CGF.CreateCoercedStore( - Result, Dest.getAddress(), + Result, E->getType(), Dest.getAddress(), llvm::TypeSize::getFixed( Dest.getPreferredSize(CGF.getContext(), E->getType()) .getQuantity()), diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 5d96959065dd9..896185907688f 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -261,6 +261,16 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) && !MD->getParent()->mayInsertExtraPadding(); + // Assignment operators for objects with non-trivially-relocatable PFP fields + // aren't trivial, we need to auth and sign the fields one by one. + if (TrivialAssignment && + !CGM.getContext().arePFPFieldsTriviallyRelocatable(MD->getParent()) && + CGM.getContext().hasPFPFields( + QualType(MD->getParent()->getTypeForDecl(), 0))) { + TrivialForCodegen = false; + TrivialAssignment = false; + } + // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment // operator before the LHS. CallArgList RtlArgStorage; diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index b016c6e36d1a8..9b6f822ebef16 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -906,6 +906,29 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, if (!EltInit) return false; + if (CGM.getContext().isPFPField(*Field)) { + llvm::ConstantInt *Disc; + llvm::Constant *AddrDisc; + if (CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) { + uint64_t FieldSignature = + std::hash()(CGM.getPFPFieldName(*Field)) & 0xffff; + Disc = llvm::ConstantInt::get(CGM.Int64Ty, FieldSignature); + AddrDisc = llvm::ConstantPointerNull::get(CGM.VoidPtrTy); + } else if (Emitter.isAbstract()) { + return false; + } else { + Disc = llvm::ConstantInt::get( + CGM.Int64Ty, (-(Layout.getFieldOffset(FieldNo) / 8)) & 0xffff); + AddrDisc = Emitter.getCurrentAddrPrivate(); + } + EltInit = llvm::ConstantPtrAuth::get( + EltInit, llvm::ConstantInt::get(CGM.Int32Ty, 2), Disc, AddrDisc, + CGM.getPFPDeactivationSymbol(*Field)); + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) + Emitter.registerCurrentAddrPrivate(EltInit, + cast(AddrDisc)); + } + if (ZeroInitPadding) { if (!DoZeroInitPadding(Layout, FieldNo, **Field, AllowOverwrite, SizeSoFar, ZeroFieldSize)) @@ -1656,7 +1679,20 @@ ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value, llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) { initializeNonAbstract(D.getType().getAddressSpace()); - return markIfFailed(tryEmitPrivateForVarInit(D)); + llvm::Constant *Init = tryEmitPrivateForVarInit(D); + + // If a placeholder address was needed for a TLS variable, implying that the + // initializer's value depends on its address, then the object may not be + // initialized in .tdata because the initializer will be memcpy'd to the + // thread's TLS. Instead the initialization must be done in code. + if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) { + for (auto &entry : PlaceholderAddresses) + entry.second->eraseFromParent(); + PlaceholderAddresses.clear(); + Init = nullptr; + } + + return markIfFailed(Init); } llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E, @@ -2586,6 +2622,7 @@ CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) { return getCXXABI().EmitMemberFunctionPointer(method); // Otherwise, a member data pointer. + getContext().recordMemberDataPointerEvaluation(decl); uint64_t fieldOffset = getContext().getFieldOffset(decl); CharUnits chars = getContext().toCharUnitsFromBits((int64_t) fieldOffset); return getCXXABI().EmitMemberDataPointer(type, chars); diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp index 4b032306ead72..2d72fef470af6 100644 --- a/clang/lib/CodeGen/CGPointerAuth.cpp +++ b/clang/lib/CodeGen/CGPointerAuth.cpp @@ -308,9 +308,9 @@ CodeGenModule::getConstantSignedPointer(llvm::Constant *Pointer, unsigned Key, IntegerDiscriminator = llvm::ConstantInt::get(Int64Ty, 0); } - return llvm::ConstantPtrAuth::get(Pointer, - llvm::ConstantInt::get(Int32Ty, Key), - IntegerDiscriminator, AddressDiscriminator); + return llvm::ConstantPtrAuth::get( + Pointer, llvm::ConstantInt::get(Int32Ty, Key), IntegerDiscriminator, + AddressDiscriminator, llvm::Constant::getNullValue(UnqualPtrTy)); } /// Does a given PointerAuthScheme require us to sign a value diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index dcf523f56bf1e..669fe867f75ac 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -2180,6 +2180,31 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(contBB); } +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const PFPField &Field) { + return EmitAddressOfPFPField( + Builder.CreateConstInBoundsByteGEP(RecordPtr.withElementType(Int8Ty), + Field.structOffset), + Field.field, Field.offset - Field.structOffset); +} + +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const FieldDecl *Field, + CharUnits Offset) { + return Address( + EmitRuntimeCall( + CGM.getIntrinsic(llvm::Intrinsic::protected_field_ptr), + {RecordPtr.getBasePointer(), Builder.getInt64(Offset.getQuantity()), + llvm::MetadataAsValue::get( + getLLVMContext(), + llvm::MDString::get(getLLVMContext(), + CGM.getPFPFieldName(Field))), + getContext().arePFPFieldsTriviallyRelocatable(Field->getParent()) + ? Builder.getFalse() + : Builder.getTrue()}), + VoidPtrTy, RecordPtr.getAlignment().alignmentAtOffset(Offset)); +} + void CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Ignore empty classes in C++. @@ -2242,13 +2267,22 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Get and call the appropriate llvm.memcpy overload. Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false); - return; + } else { + // Otherwise, just memset the whole thing to zero. This is legal + // because in LLVM, all default initializers (other than the ones we just + // handled above, and the case handled below) are guaranteed to have a bit + // pattern of all zeros. + Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); } - // Otherwise, just memset the whole thing to zero. This is legal - // because in LLVM, all default initializers (other than the ones we just - // handled above) are guaranteed to have a bit pattern of all zeros. - Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); + // With the pointer field protection feature, null pointers do not have a bit + // pattern of zero in memory, so we must initialize them separately. + std::vector PFPFields; + getContext().findPFPFields(Ty, CharUnits::Zero(), PFPFields, true); + for (auto &Field : PFPFields) { + auto addr = EmitAddressOfPFPField(DestPtr, Field); + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), addr); + } } llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) { diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index ca00a0e8c6cf4..fecb48033e6ef 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -4993,8 +4993,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// Create a store to \arg DstPtr from \arg Src, truncating the stored value /// to at most \arg DstSize bytes. - void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, - bool DstIsVolatile); + void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst, + llvm::TypeSize DstSize, bool DstIsVolatile); /// EmitExtendGCLifetime - Given a pointer to an Objective-C object, /// make sure it survives garbage collection until this point. @@ -5468,6 +5468,10 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitRISCVMultiVersionResolver(llvm::Function *Resolver, ArrayRef Options); + Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field); + Address EmitAddressOfPFPField(Address RecordPtr, const FieldDecl *Field, + CharUnits Offset); + private: QualType getVarArgType(const Expr *Arg); diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 96ab4dd1837ce..8eb615ddd54fa 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -860,6 +860,7 @@ void CodeGenModule::Release() { applyGlobalValReplacements(); applyReplacements(); emitMultiVersionFunctions(); + emitPFPFieldsWithEvaluatedOffset(); if (Context.getLangOpts().IncrementalExtensions && GlobalTopLevelStmtBlockInFlight.first) { @@ -4410,6 +4411,35 @@ void CodeGenModule::emitMultiVersionFunctions() { emitMultiVersionFunctions(); } +llvm::GlobalValue *CodeGenModule::getPFPDeactivationSymbol(FieldDecl *FD) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *DS = TheModule.getNamedValue(DSName); + if (!DS) { + DS = new llvm::GlobalVariable(TheModule, Int8Ty, false, + llvm::GlobalVariable::ExternalWeakLinkage, + nullptr, DSName); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + } + return DS; +} + +void CodeGenModule::emitPFPFieldsWithEvaluatedOffset() { + llvm::Constant *Nop = llvm::ConstantExpr::getIntToPtr( + llvm::ConstantInt::get(Int64Ty, 0xd503201f), VoidPtrTy); + for (auto *FD : getContext().PFPFieldsWithEvaluatedOffset) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *OldDS = TheModule.getNamedValue(DSName); + llvm::GlobalValue *DS = llvm::GlobalAlias::create( + Int8Ty, 0, llvm::GlobalValue::ExternalLinkage, DSName, Nop, &TheModule); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } +} + static void replaceDeclarationWith(llvm::GlobalValue *Old, llvm::Constant *New) { assert(cast(Old)->isDeclaration() && "Not a declaration"); @@ -7938,3 +7968,12 @@ void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx); } + +std::string CodeGenModule::getPFPFieldName(const FieldDecl *FD) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCanonicalTypeName( + QualType(FD->getParent()->getTypeForDecl(), 0), Out, false); + Out << "." << FD->getName(); + return OutName; +} diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h index 6deb467b2cc9f..7c315d5f10b16 100644 --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -1812,6 +1812,9 @@ class CodeGenModule : public CodeGenTypeCache { return !getLangOpts().CPlusPlus; } + std::string getPFPFieldName(const FieldDecl *FD); + llvm::GlobalValue *getPFPDeactivationSymbol(FieldDecl *FD); + private: bool shouldDropDLLAttribute(const Decl *D, const llvm::GlobalValue *GV) const; @@ -2003,6 +2006,10 @@ class CodeGenModule : public CodeGenTypeCache { llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, StringRef Suffix); + + /// Emit deactivation symbols for any PFP fields whose offset is taken with + /// offsetof. + void emitPFPFieldsWithEvaluatedOffset(); }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index 77e995b4c933a..f3940b031ffe7 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -1224,6 +1224,7 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP, return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM); } + getContext().recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD)); return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset); diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index 40371d99e23e1..60b2e68755515 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -2917,6 +2917,7 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP, // the class in which it was declared, and convert from there if necessary. // For indirect field decls, get the outermost anonymous field and use the // parent class. + Ctx.recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(MPD)); const FieldDecl *FD = dyn_cast(MPD); if (!FD) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index e1e8f57dd6455..027aa0b55d343 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7732,6 +7732,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, Args.addOptInFlag(CmdArgs, options::OPT_fexperimental_late_parse_attributes, options::OPT_fno_experimental_late_parse_attributes); + if (!IsCudaDevice) + Args.AddLastArg(CmdArgs, + options::OPT_experimental_pointer_field_protection_EQ); + // Setup statistics file output. SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D); if (!StatsFile.empty()) { diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 0a8a3e1c49414..60088dd56a48e 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -6441,6 +6441,10 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL)); } +static void handleNoPFPAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { + D->addAttr(NoPointerFieldProtectionAttr::Create(S.Context, AL)); +} + static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { auto *FD = dyn_cast(D); assert(FD); @@ -7452,6 +7456,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, handleCountedByAttrField(S, D, AL); break; + case ParsedAttr::AT_NoPointerFieldProtection: + handleNoPFPAttrField(S, D, AL); + break; + // Microsoft attributes: case ParsedAttr::AT_LayoutVersion: handleLayoutVersion(S, D, AL); diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp index b2310628adc64..70d9ec9f6a7f2 100644 --- a/clang/lib/Sema/SemaExprCXX.cpp +++ b/clang/lib/Sema/SemaExprCXX.cpp @@ -5103,6 +5103,7 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S, TypeTrait UTT, // impose the same constraints. case UTT_IsTriviallyRelocatable: case UTT_IsTriviallyEqualityComparable: + case UTT_HasNonRelocatableFields: case UTT_CanPassInRegs: // Per the GCC type traits documentation, T shall be a complete type, cv void, // or an array of unknown bound. But GCC actually imposes the same constraints @@ -5672,6 +5673,10 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT, return C.hasUniqueObjectRepresentations(T); case UTT_IsTriviallyRelocatable: return T.isTriviallyRelocatableType(C); + case UTT_HasNonRelocatableFields: + return T->getAsCXXRecordDecl() && + !C.arePFPFieldsTriviallyRelocatable(T->getAsCXXRecordDecl()) && + C.hasPFPFields(T); case UTT_IsBitwiseCloneable: return T.isBitwiseCloneableType(C); case UTT_CanPassInRegs: diff --git a/clang/test/CodeGen/pfp-attribute-disable.cpp b/clang/test/CodeGen/pfp-attribute-disable.cpp new file mode 100644 index 0000000000000..a8efb08b9a7ee --- /dev/null +++ b/clang/test/CodeGen/pfp-attribute-disable.cpp @@ -0,0 +1,33 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + + +struct S { + int* ptr1; + __attribute__((no_field_protection)) int* ptr2; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers_without_no_field_protection +int* load_pointers_without_no_field_protection(S *t) { + return t->ptr1; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: load_pointers_with_no_field_protection +int* load_pointers_with_no_field_protection(S *t) { + return t->ptr2; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_without_no_field_protection +void store_pointers_without_no_field_protection(S *t, int *input) { + t->ptr1 = input; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_with_no_field_protection +void store_pointers_with_no_field_protection(S *t, int *input) { + t->ptr2 = input; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-load-store.cpp b/clang/test/CodeGen/pfp-load-store.cpp new file mode 100644 index 0000000000000..b5f6a11574270 --- /dev/null +++ b/clang/test/CodeGen/pfp-load-store.cpp @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -O1 -o - %s | FileCheck %s + +int val; + +struct Pointer { + int* ptr; +private: + int private_data; +}; + +struct ArrayType { + int* array[3]; +private: + int private_data; +}; + +struct Array { + ArrayType array; +private: + int private_data; +}; + +struct Struct { + Pointer ptr; +}; + +// CHECK-LABEL: test_pointer +Pointer test_pointer(Pointer t) { + t.ptr = &val; + return t; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + + + +// CHECK-LABEL: test_struct +int* test_struct(Struct *t) { + return (t->ptr).ptr; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-memcpy.cpp b/clang/test/CodeGen/pfp-memcpy.cpp new file mode 100644 index 0000000000000..3e92db6110ae3 --- /dev/null +++ b/clang/test/CodeGen/pfp-memcpy.cpp @@ -0,0 +1,19 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct ClassWithTrivialCopy { + ClassWithTrivialCopy(); + ~ClassWithTrivialCopy(); + void *a; +private: + void *c; +}; + +// Make sure we don't emit memcpy for operator= and constuctors. +void make_trivial_copy(ClassWithTrivialCopy *s1, ClassWithTrivialCopy *s2) { + *s1 = *s2; + ClassWithTrivialCopy s3(*s2); +} + +// CHECK-LABEL: define{{.*}} void @_Z17make_trivial_copyP20ClassWithTrivialCopyS0_ +// CHECK-NOT: memcpy +// CHECK: ret void \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-null-init.cpp b/clang/test/CodeGen/pfp-null-init.cpp new file mode 100644 index 0000000000000..94c1224f1be13 --- /dev/null +++ b/clang/test/CodeGen/pfp-null-init.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct S { + void *p; +private: + int private_data; +}; + +// CHECK-LABEL: null_init +void null_init() { + S s{}; +} + +// Check that the constructor was applied +// CHECK: call void @llvm.memset.{{.*}} +// CHECK: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.p", i1 false) \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-struct-gep.cpp b/clang/test/CodeGen/pfp-struct-gep.cpp new file mode 100644 index 0000000000000..964545efa9f4b --- /dev/null +++ b/clang/test/CodeGen/pfp-struct-gep.cpp @@ -0,0 +1,25 @@ +// RUN: %clang_cc1 -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOPFP +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-PFP + + +struct S { + int* ptr; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers +int* load_pointers(S *t) { + return t->ptr; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + +// CHECK-LABEL: store_pointers +void store_pointers(S* t, int* p) { + t->ptr = p; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + + diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp index 90054dbf37ae3..3d26a8639e13f 100644 --- a/clang/test/CodeGenCXX/trivial_abi.cpp +++ b/clang/test/CodeGenCXX/trivial_abi.cpp @@ -286,11 +286,9 @@ static_assert(sizeof(S) == 8 && sizeof(S2) == 8, ""); // CHECK: define{{.*}} @"_ZN3$_08__invokeEv"() // CHECK: %[[RETVAL:.*]] = alloca %[[STRUCT_SMALL]], align 8 -// CHECK: %[[COERCE:.*]] = alloca %[[STRUCT_SMALL]], align 8 // CHECK: %[[CALL:.*]] = call{{.*}} @"_ZNK3$_0clEv" -// CHECK: %[[COERCEDIVE:.*]] = getelementptr{{.*}} %[[COERCE]] +// CHECK: %[[RETVALDIVE:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[COERCEVALIP:.*]] = inttoptr{{.*}} %[[CALL]] -// CHECK: call {{.*}}memcpy{{.*}} %[[RETVAL]]{{.*}} %[[COERCE]] // CHECK: %[[COERCEDIVE1:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[TMP:.*]] = load{{.*}} %[[COERCEDIVE1]] // CHECK: %[[COERCEVALPI:.*]] = ptrtoint{{.*}} %[[TMP]] diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake index c3e734f72392f..2f6fd4ba1e4c9 100644 --- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake +++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake @@ -172,7 +172,7 @@ function(add_compiler_rt_runtime name type) cmake_parse_arguments(LIB "" "PARENT_TARGET" - "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS;EXTENSIONS" + "OS;ARCHS;SOURCES;CFLAGS;LINK_FLAGS;DEFS;DEPS;LINK_LIBS;OBJECT_LIBS;ADDITIONAL_HEADERS;EXTENSIONS;C_STANDARD;CXX_STANDARD" ${ARGN}) set(libnames) # Until we support this some other way, build compiler-rt runtime without LTO @@ -360,6 +360,12 @@ function(add_compiler_rt_runtime name type) set_target_link_flags(${libname} ${extra_link_flags_${libname}}) set_property(TARGET ${libname} APPEND PROPERTY COMPILE_DEFINITIONS ${LIB_DEFS}) + if(LIB_C_STANDARD) + set_property(TARGET ${libname} PROPERTY C_STANDARD ${LIB_C_STANDARD}) + endif() + if(LIB_CXX_STANDARD) + set_property(TARGET ${libname} PROPERTY CXX_STANDARD ${LIB_CXX_STANDARD}) + endif() set_target_output_directories(${libname} ${output_dir_${libname}}) install(TARGETS ${libname} ARCHIVE DESTINATION ${install_dir_${libname}} diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake index 2683259e93e37..e75115fbbee9a 100644 --- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake +++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake @@ -119,3 +119,7 @@ endif() if (WIN32) set(ALL_ORC_SUPPORTED_ARCH ${X86_64}) endif() + +if (OS_NAME MATCHES "Linux") + set(ALL_PFP_SUPPORTED_ARCH ${X86_64} ${ARM64}) +endif() diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake index 7bd3269bd999d..7bdd30ee67f3d 100644 --- a/compiler-rt/cmake/builtin-config-ix.cmake +++ b/compiler-rt/cmake/builtin-config-ix.cmake @@ -26,6 +26,7 @@ builtin_check_c_compiler_flag("-Xclang -mcode-object-version=none" COMPILER_RT_H builtin_check_c_compiler_flag(-Wbuiltin-declaration-mismatch COMPILER_RT_HAS_WBUILTIN_DECLARATION_MISMATCH_FLAG) builtin_check_c_compiler_flag(/Zl COMPILER_RT_HAS_ZL_FLAG) builtin_check_c_compiler_flag(-fcf-protection=full COMPILER_RT_HAS_FCF_PROTECTION_FLAG) +builtin_check_c_compiler_flag(-nostdinc++ COMPILER_RT_HAS_NOSTDINCXX_FLAG) builtin_check_c_compiler_source(COMPILER_RT_HAS_ATOMIC_KEYWORD " diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake index cf729c3adb1f5..8b45685132ab3 100644 --- a/compiler-rt/cmake/config-ix.cmake +++ b/compiler-rt/cmake/config-ix.cmake @@ -686,6 +686,9 @@ if(APPLE) list_intersect(ORC_SUPPORTED_ARCH ALL_ORC_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) + list_intersect(PFP_SUPPORTED_ARCH + ALL_PFP_SUPPORTED_ARCH + SANITIZER_COMMON_SUPPORTED_ARCH) else() # Architectures supported by compiler-rt libraries. @@ -721,6 +724,7 @@ else() filter_available_targets(GWP_ASAN_SUPPORTED_ARCH ${ALL_GWP_ASAN_SUPPORTED_ARCH}) filter_available_targets(NSAN_SUPPORTED_ARCH ${ALL_NSAN_SUPPORTED_ARCH}) filter_available_targets(ORC_SUPPORTED_ARCH ${ALL_ORC_SUPPORTED_ARCH}) + filter_available_targets(PFP_SUPPORTED_ARCH ${ALL_PFP_SUPPORTED_ARCH}) endif() if (MSVC) diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index 19316c52d12ce..ca5cc7d218223 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -6,7 +6,7 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) cmake_minimum_required(VERSION 3.20.0) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) - project(CompilerRTBuiltins C ASM) + project(CompilerRTBuiltins C CXX ASM) set(COMPILER_RT_STANDALONE_BUILD TRUE) set(COMPILER_RT_BUILTINS_STANDALONE_BUILD TRUE) @@ -64,6 +64,8 @@ include(CMakePushCheckState) option(COMPILER_RT_BUILTINS_HIDE_SYMBOLS "Do not export any symbols from the static library." ON) +include_directories(../../../third-party/siphash/include) + # TODO: Need to add a mechanism for logging errors when builtin source files are # added to a sub-directory and not this CMakeLists file. set(GENERIC_SOURCES @@ -570,6 +572,7 @@ set(aarch64_SOURCES ${GENERIC_TF_SOURCES} ${GENERIC_SOURCES} cpu_model/aarch64.c + aarch64/emupac.cpp aarch64/fp_mode.c ) @@ -802,7 +805,7 @@ else () append_list_if(COMPILER_RT_ENABLE_CET -fcf-protection=full BUILTIN_CFLAGS) endif() - append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 BUILTIN_CFLAGS) + append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ BUILTIN_CFLAGS) append_list_if(COMPILER_RT_HAS_WBUILTIN_DECLARATION_MISMATCH_FLAG -Werror=builtin-declaration-mismatch BUILTIN_CFLAGS) # Don't embed directives for picking any specific CRT @@ -917,6 +920,8 @@ else () SOURCES ${${arch}_SOURCES} DEFS ${BUILTIN_DEFS} CFLAGS ${BUILTIN_CFLAGS_${arch}} + C_STANDARD 11 + CXX_STANDARD 17 PARENT_TARGET builtins) cmake_pop_check_state() endif () diff --git a/compiler-rt/lib/builtins/aarch64/emupac.cpp b/compiler-rt/lib/builtins/aarch64/emupac.cpp new file mode 100644 index 0000000000000..c27c2aa5dee28 --- /dev/null +++ b/compiler-rt/lib/builtins/aarch64/emupac.cpp @@ -0,0 +1,136 @@ +//===--- emupac.cpp - Emulated PAC implementation -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements Emulated PAC using SipHash_2_4 as the IMPDEF hashing +// scheme. +// +//===----------------------------------------------------------------------===// + +#include + +#include "siphash/SipHash.h" + +// EmuPAC implements runtime emulation of PAC instructions. If the current +// CPU supports PAC, EmuPAC uses real PAC instructions. Otherwise, it uses the +// emulation, which is effectively an implementation of PAC with an IMPDEF +// hashing scheme based on SipHash_2_4. +// +// The purpose of the emulation is to allow programs to be built to be portable +// to machines without PAC support, with some performance loss and increased +// probability of false positives (due to not being able to portably determine +// the VA size), while being functionally almost equivalent to running on a +// machine with PAC support. One example of a use case is if PAC is used in +// production as a security mitigation, but the testing environment is +// heterogeneous (i.e. some machines lack PAC support). In this case we would +// like the testing machines to be able to detect issues resulting +// from the use of PAC instructions that would affect production by running +// tests. This can be achieved by building test binaries with EmuPAC and +// production binaries with real PAC. +// +// The emulation assumes that the VA size is at most 48 bits. The architecture +// as of ARMv8.2, which was the last architecture version in which PAC was not +// mandatory, permitted VA size up to 52 bits via ARMv8.2-LVA, but we are +// unaware of an ARMv8.2 CPU that implemented ARMv8.2-LVA. + +const uint64_t kMaxVASize = 48; +const uint64_t kPACMask = ((1ULL << 55) - 1) & ~((1ULL << kMaxVASize) - 1); +const uint64_t kTTBR1Mask = 1ULL << 55; + +// Determine whether PAC is supported without accessing memory. This utilizes +// the XPACLRI instruction which will copy bit 55 of x30 into at least bit 54 if +// PAC is supported and acts as a NOP if PAC is not supported. +static bool pac_supported() { + register uintptr_t x30 __asm__("x30") = 1ULL << 55; + __asm__ __volatile__("xpaclri" : "+r"(x30)); +#ifdef FORCE_NON_PAC + return !(x30 & (1ULL << 54)); +#else + return x30 & (1ULL << 54); +#endif +} + +// This asm snippet is used to force the creation of a frame record when +// calling the EmuPAC functions. This is important because the EmuPAC functions +// may crash if an auth failure is detected and may be unwound past using a +// frame pointer based unwinder. +#ifdef __GCC_HAVE_DWARF2_CFI_ASM +#define frame_pointer_wrap(sym) \ + "stp x29, x30, [sp, #-16]!\n" \ + ".cfi_def_cfa_offset 16\n" \ + "mov x29, sp\n" \ + ".cfi_def_cfa w29, 16\n" \ + ".cfi_offset w30, -8\n" \ + ".cfi_offset w29, -16\n" \ + "bl " #sym "\n" \ + ".cfi_def_cfa wsp, 16\n" \ + "ldp x29, x30, [sp], #16\n" \ + ".cfi_def_cfa_offset 0\n" \ + ".cfi_restore w30\n" \ + ".cfi_restore w29\n" \ + "ret" +#else +#define frame_pointer_wrap(sym) \ + "stp x29, x30, [sp, #-16]!\n" \ + "mov x29, sp\n" \ + "bl " #sym "\n" \ + "ldp x29, x30, [sp], #16\n" \ + "ret" +#endif + +static const uint8_t K[16] = {0xb5, 0xd4, 0xc9, 0xeb, 0x79, 0x10, 0x4a, 0x79, + 0x6f, 0xec, 0x8b, 0x1b, 0x42, 0x87, 0x81, 0xd4}; + +__attribute__((flatten)) +extern "C" uint64_t __emupac_pacda_impl(uint64_t ptr, uint64_t disc) { + if (pac_supported()) { + __asm__ __volatile__(".arch_extension pauth\npacda %0, %1" + : "+r"(ptr) + : "r"(disc)); + return ptr; + } + if (ptr & kTTBR1Mask) { + if ((ptr & kPACMask) != kPACMask) { + return ptr | kPACMask; + } + } else { + if (ptr & kPACMask) { + return ptr & ~kPACMask; + } + } + uint64_t hash; + siphash<2, 4>(reinterpret_cast(&ptr), 8, K, + *reinterpret_cast(&hash)); + return (ptr & ~kPACMask) | (hash & kPACMask); +} + +extern "C" __attribute__((naked)) uint64_t __emupac_pacda(uint64_t ptr, uint64_t disc) { + __asm__(frame_pointer_wrap(__emupac_pacda_impl)); +} + +__attribute__((flatten)) +extern "C" uint64_t __emupac_autda_impl(uint64_t ptr, uint64_t disc) { + if (pac_supported()) { + __asm__ __volatile__(".arch_extension pauth\nautda %0, %1" + : "+r"(ptr) + : "r"(disc)); + return ptr; + } + uint64_t ptr_without_pac = + (ptr & kTTBR1Mask) ? (ptr | kPACMask) : (ptr & ~kPACMask); + uint64_t hash; + siphash<2, 4>(reinterpret_cast(&ptr_without_pac), 8, K, + *reinterpret_cast(&hash)); + if (((ptr & ~kPACMask) | (hash & kPACMask)) != ptr) { + __builtin_trap(); + } + return ptr_without_pac; +} + +extern "C" __attribute__((naked)) uint64_t __emupac_autda(uint64_t ptr, uint64_t disc) { + __asm__(frame_pointer_wrap(__emupac_autda_impl)); +} diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt index fad5b7e03925e..c4d2d181fd5df 100644 --- a/compiler-rt/test/CMakeLists.txt +++ b/compiler-rt/test/CMakeLists.txt @@ -106,6 +106,7 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS) # ShadowCallStack does not yet provide a runtime with compiler-rt, the tests # include their own minimal runtime add_subdirectory(shadowcallstack) + add_subdirectory(pfp) endif() # Now that we've traversed all the directories and know all the lit testsuites, diff --git a/compiler-rt/test/pfp/CMakeLists.txt b/compiler-rt/test/pfp/CMakeLists.txt new file mode 100644 index 0000000000000..1ee8eea8dbaac --- /dev/null +++ b/compiler-rt/test/pfp/CMakeLists.txt @@ -0,0 +1,36 @@ +set(PFP_TESTSUITES) +set(PFP_TEST_DEPS ${SANITIZER_COMMON_LIT_TEST_DEPS} unwind compiler-rt) +if(COMPILER_RT_HAS_LLD AND TARGET lld) + list(APPEND PFP_TEST_DEPS lld) +endif() + +macro(add_pfp_testsuite arch thinlto) + set(PFP_TEST_TARGET_ARCH ${arch}) + get_test_cc_for_arch(${arch} PFP_TEST_TARGET_CC PFP_TEST_TARGET_CFLAGS) + + string(TOUPPER ${arch} CONFIG_NAME) + + if (${thinlto}) + set(PFP_TEST_USE_THINLTO ${thinlto}) + set(CONFIG_NAME "thinlto-${CONFIG_NAME}") + list(APPEND PFP_TEST_DEPS LTO) + endif() + set(PFP_TEST_USE_THINLTO ${thinlto}) + + configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py) + list(APPEND PFP_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}) +endmacro() + +set(PFP_TEST_ARCH ${PFP_SUPPORTED_ARCH}) + +foreach(arch ${PFP_TEST_ARCH}) + add_pfp_testsuite(${arch} False) + add_pfp_testsuite(${arch} True) +endforeach() + +add_lit_testsuite(check-pfp "Running the PointerFieldProtection tests" + ${PFP_TESTSUITES} + DEPENDS ${PFP_TEST_DEPS}) + diff --git a/compiler-rt/test/pfp/lit.cfg.py b/compiler-rt/test/pfp/lit.cfg.py new file mode 100644 index 0000000000000..2d58972804039 --- /dev/null +++ b/compiler-rt/test/pfp/lit.cfg.py @@ -0,0 +1,37 @@ +# -*- Python -*- + +import os + +from lit.llvm import llvm_config +from lit.llvm.subst import ToolSubst, FindTool + +# Setup config name. +config.name = "pfp" + config.name_suffix + +# Default test suffixes. +config.suffixes = [".c", ".cpp"] + +# Setup source root. +config.test_source_root = os.path.dirname(__file__) +# Setup default compiler flags used with -fsanitize=memory option. +clang_cflags = [config.target_cflags] + config.debug_info_flags +clang_cxxflags = config.cxx_mode_flags + clang_cflags +clang_pfp_tagged_common_cflags = clang_cflags + [ + "-fexperimental-pointer-field-protection=tagged" +] + + +clang_pfp_cxxflags = config.cxx_mode_flags + clang_pfp_tagged_common_cflags +clang_pfp_cxxflags = clang_pfp_cxxflags + ["-fuse-ld=lld --rtlib=compiler-rt --unwindlib=libunwind -static-libgcc"] + + +def build_invocation(compile_flags, with_lto=False): + lto_flags = [] + if with_lto and config.lto_supported: + lto_flags += config.lto_flags + + return " " + " ".join([config.clang] + lto_flags + compile_flags) + " " + + +config.substitutions.append(("%clangxx ", build_invocation(clang_cxxflags))) +config.substitutions.append(("%clangxx_pfp ", build_invocation(clang_pfp_cxxflags, config.use_thinlto))) diff --git a/compiler-rt/test/pfp/lit.site.cfg.py.in b/compiler-rt/test/pfp/lit.site.cfg.py.in new file mode 100644 index 0000000000000..7bc6331a0a376 --- /dev/null +++ b/compiler-rt/test/pfp/lit.site.cfg.py.in @@ -0,0 +1,16 @@ +@LIT_SITE_CFG_IN_HEADER@ + +# Tool-specific config options. +config.name_suffix = "@PFP_TEST_CONFIG_SUFFIX@" +config.target_cflags = "@PFP_TEST_TARGET_CFLAGS@" +config.target_arch = "@PFP_TEST_TARGET_ARCH@" +config.use_lld = True +config.use_thinlto = @PFP_TEST_USE_THINLTO@ +config.libunwind_shared = "@LIBUNWIND_ENABLE_SHARED@" +config.libunwind_install_dir = "@LLVM_BINARY_DIR@/@LIBUNWIND_INSTALL_LIBRARY_DIR@" + +# Load common config for all compiler-rt lit tests. +lit_config.load_config(config, "@COMPILER_RT_BINARY_DIR@/test/lit.common.configured") + +# Load tool-specific config that would do the real work. +lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py") diff --git a/compiler-rt/test/pfp/use-after-free-fixed.cpp b/compiler-rt/test/pfp/use-after-free-fixed.cpp new file mode 100644 index 0000000000000..2fe21111cdd49 --- /dev/null +++ b/compiler-rt/test/pfp/use-after-free-fixed.cpp @@ -0,0 +1,43 @@ +// RUN: %clangxx_pfp %s -o %t1 +// RUN: %run %t1 2>&1 +// RUN: %clangxx %s -o %t2 +// RUN: %run %t2 2>&1 + +#include + +// Struct1.ptr and Struct2.ptr have different locks. +struct Struct1 { + int *ptr; + Struct1() : num(1), ptr(&num) {} + +private: + int num; +}; + +struct Struct2 { + int *ptr; + Struct2() : num(2), ptr(&num) {} + +private: + int num; +}; + +Struct1 *new_object1() { + Struct1 *ptr = new Struct1; + return ptr; +} + +Struct2 *new_object2() { + Struct2 *ptr = new Struct2; + return ptr; +} + +int main() { + Struct1 *obj1 = new_object1(); + Struct2 *obj2 = new_object2(); + std::cout << "Struct2: " << *(obj2->ptr) << "\n"; + std::cout << "Struct1: " << *(obj1->ptr) << "\n"; + delete obj1; + delete obj2; + return 0; +} diff --git a/compiler-rt/test/pfp/use-after-free.cpp b/compiler-rt/test/pfp/use-after-free.cpp new file mode 100644 index 0000000000000..17b7661b2a904 --- /dev/null +++ b/compiler-rt/test/pfp/use-after-free.cpp @@ -0,0 +1,45 @@ +// RUN: %clangxx_pfp %s -o %t1 +// RUN: %expect_crash %run %t1 2>&1 +// RUN: %clangxx %s -o %t2 +// RUN: %run %t2 2>&1 + +#include + +// Struct1.ptr and Struct2.ptr have different locks. +struct Struct1 { + int *ptr; + Struct1() : num(1), ptr(&num) {} + +private: + int num; +}; + +struct Struct2 { + int *ptr; + Struct2() : num(2), ptr(&num) {} + +private: + int num; +}; + +Struct1 *new_object1() { + Struct1 *ptr = new Struct1; + return ptr; +} + +Struct2 *new_object2() { + Struct2 *ptr = new Struct2; + return ptr; +} + +int main() { + Struct1 *obj1 = new_object1(); + delete obj1; + // obj1's memory will be reused. + Struct2 *obj2 = new_object2(); + std::cout << "Struct2: " << *(obj2->ptr) << "\n"; + // Uses a wrong lock. The Program should crash when "-fexperimental-pointer-field-protection=tagged". + std::cout << "Struct1: " << *(obj1->ptr) << "\n"; + delete obj2; + return 0; +} diff --git a/libcxx/include/__config b/libcxx/include/__config index c8224b07a6b81..27fd748e333e1 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1215,6 +1215,29 @@ typedef __char32_t char32_t; # define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 0 # endif +# if __has_feature(pointer_field_protection) +// Force a class to be non-standard layout by giving it two bases with the same +// type. This is useful when structure protection is enabled because structure +// protection cannot be applied to standard layout classes. We may use this in +// cases where the standard does not specify whether a standard library class is +// standard layout. See C++2a [class]p7: +// A class S is a standard-layout class if it: +// -- has at most one base class subobject of any given type +_LIBCPP_BEGIN_NAMESPACE_STD +_LIBCPP_DIAGNOSTIC_PUSH +_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winaccessible-base") +class __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout_base2 : __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout : __force_nonstandard_layout_base1, __force_nonstandard_layout_base2 {}; +_LIBCPP_DIAGNOSTIC_POP +_LIBCPP_END_NAMESPACE_STD +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT : __force_nonstandard_layout +# define _LIBCPP_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT +# define _LIBCPP_NO_PFP +# endif + #endif // __cplusplus #endif // _LIBCPP___CONFIG diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h index f33f424c66c22..55ca0fe4fc9a8 100644 --- a/libcxx/include/__functional/function.h +++ b/libcxx/include/__functional/function.h @@ -624,7 +624,7 @@ template class __policy_func; template -class __policy_func<_Rp(_ArgTypes...)> { +class __policy_func<_Rp(_ArgTypes...)> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { // Inline storage for small objects. __policy_storage __buf_; diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index e1d49c4594866..bd96841c68ebc 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -304,7 +304,7 @@ using __shared_ptr_nullptr_deleter_ctor_reqs _LIBCPP_NODEBUG = #endif template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS shared_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { struct __nullptr_sfinae_tag {}; public: @@ -1202,7 +1202,7 @@ inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXC #endif // _LIBCPP_HAS_RTTI template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS weak_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: #if _LIBCPP_STD_VER >= 17 typedef remove_extent_t<_Tp> element_type; diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h index 6f1dc98db5a9f..5e4fa857eb14b 100644 --- a/libcxx/include/__memory/unique_ptr.h +++ b/libcxx/include/__memory/unique_ptr.h @@ -126,7 +126,7 @@ struct __unique_ptr_deleter_sfinae<_Deleter&> { #endif template > -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; @@ -393,7 +393,7 @@ struct __unique_ptr_array_bounds_stored { }; template -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI _LIBCPP_TEMPLATE_VIS unique_ptr<_Tp[], _Dp> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; diff --git a/libcxx/include/__tree b/libcxx/include/__tree index 08ae8996f8f7d..ef55bd5dede99 100644 --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -875,7 +875,7 @@ _LIBCPP_DIAGNOSE_WARNING(!__is_invocable_v<_Compare const&, _Tp const&, _Tp cons int __diagnose_non_const_comparator(); template -class __tree { +class __tree _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp value_type; typedef _Compare value_compare; diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h index 9b0e240de55f4..240b3b76adf36 100644 --- a/libcxx/include/__type_traits/is_trivially_relocatable.h +++ b/libcxx/include/__type_traits/is_trivially_relocatable.h @@ -36,8 +36,12 @@ struct __libcpp_is_trivially_relocatable : is_trivially_copyable<_Tp> {}; template struct __libcpp_is_trivially_relocatable<_Tp, - __enable_if_t::value> > - : true_type {}; + __enable_if_t::value +#if __has_builtin(__has_non_relocatable_fields) + && !__has_non_relocatable_fields(_Tp) +#endif + > > : true_type { +}; _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 8818eb7dfe26e..a8e37525a75e6 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -84,7 +84,7 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template */> -class _LIBCPP_TEMPLATE_VIS vector { +class _LIBCPP_TEMPLATE_VIS vector _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: // // Types diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo index 799c6ebd5ecbb..1a59bc8ff149b 100644 --- a/libcxx/include/typeinfo +++ b/libcxx/include/typeinfo @@ -300,7 +300,7 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH type_info protected: typedef __type_info_implementations::__impl __impl; - __impl::__type_name_t __type_name; + _LIBCPP_NO_PFP __impl::__type_name_t __type_name; _LIBCPP_HIDE_FROM_ABI explicit type_info(const char* __n) : __type_name(__impl::__string_to_type_name(__n)) {} diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py index 254a61a8c633e..630b90c9d77a6 100644 --- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py +++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.py @@ -30,7 +30,7 @@ # we exit. has_run_tests = False -has_execute_mi = tuple(map(int, gdb.VERSION.split("."))) >= (14, 2) +has_execute_mi = 'execute_mi' in gdb.__dict__ class CheckResult(gdb.Command): def __init__(self): diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp index dcdce261298c1..7201340c21039 100644 --- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp +++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp @@ -255,9 +255,12 @@ void unique_ptr_test() { ComparePrettyPrintToRegex(std::move(forty_two), R"(std::unique_ptr containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::unique_ptr this_is_null; ComparePrettyPrintToChars(std::move(this_is_null), R"(std::unique_ptr is nullptr)"); +#endif } void bitset_test() { @@ -475,10 +478,13 @@ void vector_test() { "std::vector of length " "3, capacity 3 = {5, 6, 7}"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::vector> test3({7, 8}); ComparePrettyPrintToChars(std::move(test3), "std::vector of length " "2, capacity 2 = {7, 8}"); +#endif } void set_iterator_test() { @@ -649,8 +655,11 @@ void shared_ptr_test() { test0, R"(std::shared_ptr count [3\?], weak [0\?]( \(libc\+\+ missing debug info\))? containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::shared_ptr test3; ComparePrettyPrintToChars(test3, "std::shared_ptr is nullptr"); +#endif } void streampos_test() { diff --git a/libcxxabi/include/__cxxabi_config.h b/libcxxabi/include/__cxxabi_config.h index 759445dac91f9..00e1357cdab85 100644 --- a/libcxxabi/include/__cxxabi_config.h +++ b/libcxxabi/include/__cxxabi_config.h @@ -109,4 +109,14 @@ # define _LIBCXXABI_NOEXCEPT noexcept #endif +#if defined(_LIBCXXABI_COMPILER_CLANG) +# if __has_feature(pointer_field_protection) +# define _LIBCXXABI_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCXXABI_NO_PFP +# endif +#else +# define _LIBCXXABI_NO_PFP +#endif + #endif // ____CXXABI_CONFIG_H diff --git a/libcxxabi/src/private_typeinfo.h b/libcxxabi/src/private_typeinfo.h index 328a02edef5c1..a3bc0bffd41bc 100644 --- a/libcxxabi/src/private_typeinfo.h +++ b/libcxxabi/src/private_typeinfo.h @@ -145,7 +145,7 @@ class _LIBCXXABI_TYPE_VIS __class_type_info : public __shim_type_info { // Has one non-virtual public base class at offset zero class _LIBCXXABI_TYPE_VIS __si_class_type_info : public __class_type_info { public: - const __class_type_info *__base_type; + _LIBCXXABI_NO_PFP const __class_type_info *__base_type; _LIBCXXABI_HIDDEN virtual ~__si_class_type_info(); @@ -204,7 +204,7 @@ class _LIBCXXABI_TYPE_VIS __vmi_class_type_info : public __class_type_info { class _LIBCXXABI_TYPE_VIS __pbase_type_info : public __shim_type_info { public: unsigned int __flags; - const __shim_type_info *__pointee; + _LIBCXXABI_NO_PFP const __shim_type_info *__pointee; enum __masks { __const_mask = 0x1, @@ -245,7 +245,7 @@ class _LIBCXXABI_TYPE_VIS __pointer_type_info : public __pbase_type_info { class _LIBCXXABI_TYPE_VIS __pointer_to_member_type_info : public __pbase_type_info { public: - const __class_type_info *__context; + _LIBCXXABI_NO_PFP const __class_type_info *__context; _LIBCXXABI_HIDDEN virtual ~__pointer_to_member_type_info(); _LIBCXXABI_HIDDEN virtual bool can_catch(const __shim_type_info *, diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp index 9538dd4a70bae..110d087230a9c 100644 --- a/lld/ELF/Arch/AArch64.cpp +++ b/lld/ELF/Arch/AArch64.cpp @@ -135,6 +135,7 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, switch (type) { case R_AARCH64_ABS16: case R_AARCH64_ABS32: + case R_AARCH64_INST32: case R_AARCH64_ABS64: case R_AARCH64_ADD_ABS_LO12_NC: case R_AARCH64_LDST128_ABS_LO12_NC: @@ -278,6 +279,7 @@ int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const { case R_AARCH64_PREL16: return SignExtend64<16>(read16(ctx, buf)); case R_AARCH64_ABS32: + case R_AARCH64_INST32: case R_AARCH64_PREL32: return SignExtend64<32>(read32(ctx, buf)); case R_AARCH64_ABS64: @@ -505,6 +507,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel, checkIntUInt(ctx, loc, val, 32, rel); write32(ctx, loc, val); break; + case R_AARCH64_INST32: + if (!rel.sym->isUndefined()) { + checkIntUInt(ctx, loc, val, 32, rel); + write32(ctx, loc, val); + } + break; case R_AARCH64_PLT32: case R_AARCH64_GOTPCREL32: checkInt(ctx, loc, val, 32, rel); diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index 03b3cd4771f49..e611e5059dc19 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -366,6 +366,7 @@ struct Config { bool zCopyreloc; bool zForceBti; bool zForceIbt; + bool zGlibc228Compat; bool zGlobal; bool zHazardplt; bool zIfuncNoplt; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 4555a85a4d216..5a173565e478f 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1559,6 +1559,7 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) { ctx.arg.zForceBti = hasZOption(args, "force-bti"); ctx.arg.zForceIbt = hasZOption(args, "force-ibt"); ctx.arg.zGcs = getZGcs(ctx, args); + ctx.arg.zGlibc228Compat = hasZOption(args, "glibc-228-compat"); ctx.arg.zGlobal = hasZOption(args, "global"); ctx.arg.zGnustack = getZGnuStack(args); ctx.arg.zHazardplt = hasZOption(args, "hazardplt"); diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 629702b45965b..46117a75ccea8 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -869,7 +869,8 @@ static void addRelativeReloc(Ctx &ctx, InputSectionBase &isec, // relrDyn sections don't support odd offsets. Also, relrDyn sections // don't store the addend values, so we must write it to the relocated // address. - if (part.relrDyn && isec.addralign >= 2 && offsetInSec % 2 == 0) { + if (part.relrDyn && isec.addralign >= 2 && offsetInSec % 2 == 0 && + !sym.isGnuIFunc()) { isec.addReloc({expr, type, offsetInSec, addend, &sym}); if (shard) part.relrDyn->relocsVec[parallel::getThreadIndex()].push_back( @@ -1107,8 +1108,6 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset, } } else if (needsPlt(expr)) { sym.setFlags(NEEDS_PLT); - } else if (LLVM_UNLIKELY(isIfunc)) { - sym.setFlags(HAS_DIRECT_RELOC); } // If the relocation is known to be a link-time constant, we know no dynamic @@ -1194,6 +1193,9 @@ void RelocationScanner::processAux(RelExpr expr, RelType type, uint64_t offset, } } + if (LLVM_UNLIKELY(isIfunc && !needsGot(expr) && !needsPlt(expr))) + sym.setFlags(HAS_DIRECT_RELOC); + // When producing an executable, we can perform copy relocations (for // STT_OBJECT) and canonical PLT (for STT_FUNC) if sym is defined by a DSO. // Copy relocations/canonical PLT entries are unsupported for diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h index 64f2f6eaa8d09..d4280f367d23b 100644 --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -42,6 +42,8 @@ void printTraceSymbol(const Symbol &sym, StringRef name); enum { NEEDS_GOT = 1 << 0, NEEDS_PLT = 1 << 1, + // True if this is an ifunc with a direct relocation that cannot be + // represented as a RELATIVE relocation. HAS_DIRECT_RELOC = 1 << 2, // True if this symbol needs a canonical PLT entry, or (during // postScanRelocations) a copy relocation. diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index b03c4282ab1aa..afa0482bae3ba 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -1707,9 +1707,13 @@ void RelocationBaseSection::mergeRels() { } void RelocationBaseSection::partitionRels() { + const RelType relativeRel = ctx.target->relativeRel; + const RelType iRelativeRel = ctx.target->iRelativeRel; + for (auto &r : relocs) + if (r.type == relativeRel && r.sym->isGnuIFunc()) + r.type = iRelativeRel; if (!combreloc) return; - const RelType relativeRel = ctx.target->relativeRel; numRelativeRelocs = std::stable_partition(relocs.begin(), relocs.end(), [=](auto &r) { return r.type == relativeRel; }) - diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 2cea6a44b391a..c266aff3a5153 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -2277,6 +2277,19 @@ static uint64_t computeFlags(Ctx &ctx, uint64_t flags) { return PF_R | PF_W | PF_X; if (ctx.arg.executeOnly && (flags & PF_X)) return flags & ~PF_R; + + // The -z glibc-228-compat flag is used for binaries utilizing IFUNCs which + // need to be compatible with glibc versions containing a bug that was fixed + // in commit b5c45e83753b27dc538dff2d55d4410c385cf3a4 which was released in + // version 2.29. The bug causes glibc to mprotect the .text section as RW + // while calling ifunc resolvers in binaries linked with -z notext, leading to + // a SIGSEGV at startup time. By setting the W flag on the executable section + // we work around the bug by avoiding the code path that does the mprotect. It + // is recommended that binaries linked with this flag contain startup code + // (e.g. in .init_array) that remaps the executable section as non-writable. + if (ctx.arg.zGlibc228Compat && !ctx.arg.zText && (flags & PF_X)) + return flags | PF_W; + return flags; } diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h index c9d3874e7dd96..862ce9b5f18e8 100644 --- a/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -134,6 +134,7 @@ class PtrUseVisitorBase { UseAndIsOffsetKnownPair UseAndIsOffsetKnown; APInt Offset; + Value *ProtectedField; }; /// The worklist of to-visit uses. @@ -158,6 +159,8 @@ class PtrUseVisitorBase { /// The constant offset of the use if that is known. APInt Offset; + Value *ProtectedField; + /// @} /// Note that the constructor is protected because this class must be a base @@ -230,6 +233,7 @@ class PtrUseVisitor : protected InstVisitor, IntegerType *IntIdxTy = cast(DL.getIndexType(I.getType())); IsOffsetKnown = true; Offset = APInt(IntIdxTy->getBitWidth(), 0); + ProtectedField = nullptr; PI.reset(); // Enqueue the uses of this pointer. @@ -242,6 +246,7 @@ class PtrUseVisitor : protected InstVisitor, IsOffsetKnown = ToVisit.UseAndIsOffsetKnown.getInt(); if (IsOffsetKnown) Offset = std::move(ToVisit.Offset); + ProtectedField = ToVisit.ProtectedField; Instruction *I = cast(U->getUser()); static_cast(this)->visit(I); @@ -301,6 +306,16 @@ class PtrUseVisitor : protected InstVisitor, case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: return; // No-op intrinsics. + + case Intrinsic::protected_field_ptr: { + auto *OffsetConst = dyn_cast(II.getArgOperand(1)); + if (!IsOffsetKnown || !OffsetConst) + return Base::visitIntrinsicInst(II); + Offset += OffsetConst->getValue().sextOrTrunc(Offset.getBitWidth()); + ProtectedField = II.getArgOperand(2); + enqueueUsers(II); + break; + } } } diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def index 05b79eae573f7..c9f17ee4e0c7a 100644 --- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def @@ -61,6 +61,7 @@ ELF_RELOC(R_AARCH64_LD64_GOT_LO12_NC, 0x138) ELF_RELOC(R_AARCH64_LD64_GOTPAGE_LO15, 0x139) ELF_RELOC(R_AARCH64_PLT32, 0x13a) ELF_RELOC(R_AARCH64_GOTPCREL32, 0x13b) +ELF_RELOC(R_AARCH64_INST32, 0x13c) // General dynamic TLS relocations ELF_RELOC(R_AARCH64_TLSGD_ADR_PREL21, 0x200) ELF_RELOC(R_AARCH64_TLSGD_ADR_PAGE21, 0x201) diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index ec2535ac85966..13521ba6cd00f 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -431,6 +431,7 @@ enum ConstantsCodes { CST_CODE_CE_GEP_WITH_INRANGE = 31, // [opty, flags, range, n x operands] CST_CODE_CE_GEP = 32, // [opty, flags, n x operands] CST_CODE_PTRAUTH = 33, // [ptr, key, disc, addrdisc] + CST_CODE_PTRAUTH2 = 34, // [ptr, key, disc, addrdisc, DeactivationSymbol] }; /// CastOpcodes - These are values used in the bitcode files to encode which diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h index 9c8226660e087..66f9090a967c2 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -161,6 +161,8 @@ class CallLowering { /// True if this call results in convergent operations. bool IsConvergent = true; + + GlobalValue *DeactivationSymbol = nullptr; }; /// Argument handling is mostly uniform between the four places that diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index f9dcbeb370bef..4fc69a6ef67d5 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -55,6 +55,7 @@ struct MachineIRBuilderState { MDNode *PCSections = nullptr; /// MMRA Metadata to be set on any instruction we create. MDNode *MMRA = nullptr; + Value *DS = nullptr; /// \name Fields describing the insertion point. /// @{ @@ -368,6 +369,7 @@ class MachineIRBuilder { State.II = MI.getIterator(); setPCSections(MI.getPCSections()); setMMRAMetadata(MI.getMMRAMetadata()); + setDeactivationSymbol(MI.getDeactivationSymbol()); } /// @} @@ -404,6 +406,9 @@ class MachineIRBuilder { /// Set the PC sections metadata to \p MD for all the next build instructions. void setMMRAMetadata(MDNode *MMRA) { State.MMRA = MMRA; } + Value *getDeactivationSymbol() { return State.DS; } + void setDeactivationSymbol(Value *DS) { State.DS = DS; } + /// Get the current instruction's MMRA metadata. MDNode *getMMRAMetadata() { return State.MMRA; } diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h index 59f31f8443947..17db9352de4d9 100644 --- a/llvm/include/llvm/CodeGen/ISDOpcodes.h +++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h @@ -1524,6 +1524,8 @@ enum NodeType { // Outputs: Output Chain CLEAR_CACHE, + DEACTIVATION_SYMBOL, + /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. BUILTIN_OP_END diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index 997d6a5554e06..2a8f6eaa6e8b3 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -878,6 +878,15 @@ class MachineInstr return nullptr; } + // FIXME: Move to Info. + Value *DeactivationSymbol = nullptr; + Value *getDeactivationSymbol() const { + return DeactivationSymbol; + } + void setDeactivationSymbol(MachineFunction &MF, Value *DeactivationSymbol) { + this->DeactivationSymbol = DeactivationSymbol; + } + /// Helper to extract a CFI type hash if one has been added. uint32_t getCFIType() const { if (!Info) diff --git a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h index 49d9402d0c273..2a475b97101f7 100644 --- a/llvm/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/llvm/include/llvm/CodeGen/MachineInstrBuilder.h @@ -68,6 +68,47 @@ enum { } // end namespace RegState +/// Set of metadata that should be preserved when using BuildMI(). This provides +/// a more convenient way of preserving certain data from the original +/// instruction. +class MIMetadata { +public: + MIMetadata() = default; + MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr, + Value *DeactivationSymbol = nullptr) + : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA), + DeactivationSymbol(DeactivationSymbol) {} + MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, + MDNode *MMRA = nullptr) + : DL(DI), PCSections(PCSections), MMRA(MMRA) {} + explicit MIMetadata(const Instruction &From) + : DL(From.getDebugLoc()), + PCSections(From.getMetadata(LLVMContext::MD_pcsections)), + DeactivationSymbol(getDeactivationSymbol(&From)) {} + explicit MIMetadata(const MachineInstr &From) + : DL(From.getDebugLoc()), PCSections(From.getPCSections()), + DeactivationSymbol(From.getDeactivationSymbol()) {} + + const DebugLoc &getDL() const { return DL; } + MDNode *getPCSections() const { return PCSections; } + MDNode *getMMRAMetadata() const { return MMRA; } + Value *getDeactivationSymbol() const { return DeactivationSymbol; } + +private: + DebugLoc DL; + MDNode *PCSections = nullptr; + MDNode *MMRA = nullptr; + Value *DeactivationSymbol = nullptr; + + static inline Value *getDeactivationSymbol(const Instruction *I) { + if (auto *CB = dyn_cast(I)) + if (auto Bundle = + CB->getOperandBundle(llvm::LLVMContext::OB_deactivation_symbol)) + return Bundle->Inputs[0].get(); + return nullptr; + } +}; + class MachineInstrBuilder { MachineFunction *MF = nullptr; MachineInstr *MI = nullptr; @@ -317,15 +358,13 @@ class MachineInstrBuilder { } } - const MachineInstrBuilder &setPCSections(MDNode *MD) const { - if (MD) - MI->setPCSections(*MF, MD); - return *this; - } - - const MachineInstrBuilder &setMMRAMetadata(MDNode *MMRA) const { - if (MMRA) - MI->setMMRAMetadata(*MF, MMRA); + const MachineInstrBuilder ©MIMetadata(const MIMetadata &MIMD) const { + if (MIMD.getPCSections()) + MI->setPCSections(*MF, MIMD.getPCSections()); + if (MIMD.getMMRAMetadata()) + MI->setMMRAMetadata(*MF, MIMD.getMMRAMetadata()); + if (MIMD.getDeactivationSymbol()) + MI->setDeactivationSymbol(*MF, MIMD.getDeactivationSymbol()); return *this; } @@ -343,38 +382,11 @@ class MachineInstrBuilder { } }; -/// Set of metadata that should be preserved when using BuildMI(). This provides -/// a more convenient way of preserving DebugLoc, PCSections and MMRA. -class MIMetadata { -public: - MIMetadata() = default; - MIMetadata(DebugLoc DL, MDNode *PCSections = nullptr, MDNode *MMRA = nullptr) - : DL(std::move(DL)), PCSections(PCSections), MMRA(MMRA) {} - MIMetadata(const DILocation *DI, MDNode *PCSections = nullptr, - MDNode *MMRA = nullptr) - : DL(DI), PCSections(PCSections), MMRA(MMRA) {} - explicit MIMetadata(const Instruction &From) - : DL(From.getDebugLoc()), - PCSections(From.getMetadata(LLVMContext::MD_pcsections)) {} - explicit MIMetadata(const MachineInstr &From) - : DL(From.getDebugLoc()), PCSections(From.getPCSections()) {} - - const DebugLoc &getDL() const { return DL; } - MDNode *getPCSections() const { return PCSections; } - MDNode *getMMRAMetadata() const { return MMRA; } - -private: - DebugLoc DL; - MDNode *PCSections = nullptr; - MDNode *MMRA = nullptr; -}; - /// Builder interface. Specify how to create the initial instruction itself. inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, MIMD.getDL())) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } /// This version of the builder sets up the first operand as a @@ -382,8 +394,7 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, inline MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID, Register DestReg) { return MachineInstrBuilder(MF, MF.CreateMachineInstr(MCID, MIMD.getDL())) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -398,8 +409,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -417,8 +427,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); } @@ -450,8 +459,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, @@ -462,8 +470,7 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr *MI = MF.CreateMachineInstr(MCID, MIMD.getDL()); BB.insert(I, MI); return MachineInstrBuilder(MF, MI) - .setPCSections(MIMD.getPCSections()) - .setMMRAMetadata(MIMD.getMMRAMetadata()); + .copyMIMetadata(MIMD); } inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, MachineInstr &I, diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 15a2370e5d8b8..3dba2c6d55e13 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -752,6 +752,7 @@ class SelectionDAG { int64_t offset = 0, unsigned TargetFlags = 0) { return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags); } + SDValue getDeactivationSymbol(const GlobalValue *GV); SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false); SDValue getTargetFrameIndex(int FI, EVT VT) { return getFrameIndex(FI, VT, true); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 55f8f19d437a0..34528fb09b7a0 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -152,6 +152,7 @@ class SelectionDAGISel { OPC_RecordChild7, OPC_RecordMemRef, OPC_CaptureGlueInput, + OPC_CaptureDeactivationSymbol, OPC_MoveChild, OPC_MoveChild0, OPC_MoveChild1, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 2283f99202e2f..94a4262e9b986 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1928,6 +1928,23 @@ class GlobalAddressSDNode : public SDNode { } }; +class DeactivationSymbolSDNode : public SDNode { + friend class SelectionDAG; + + const GlobalValue *TheGlobal; + + DeactivationSymbolSDNode(const GlobalValue *GV, SDVTList VTs) + : SDNode(ISD::DEACTIVATION_SYMBOL, 0, DebugLoc(), VTs), + TheGlobal(GV) {} + +public: + const GlobalValue *getGlobal() const { return TheGlobal; } + + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::DEACTIVATION_SYMBOL; + } +}; + class FrameIndexSDNode : public SDNode { friend class SelectionDAG; diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 58ac87206b9a6..35cbeea57566b 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -4612,6 +4612,7 @@ class TargetLowering : public TargetLoweringBase { SmallVector InVals; const ConstantInt *CFIType = nullptr; SDValue ConvergenceControlToken; + GlobalValue *DeactivationSymbol = nullptr; std::optional PAI; @@ -4757,6 +4758,11 @@ class TargetLowering : public TargetLoweringBase { return *this; } + CallLoweringInfo &setDeactivationSymbol(GlobalValue *Sym) { + DeactivationSymbol = Sym; + return *this; + } + ArgListTy &getArgs() { return Args; } diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h index a50217078d0ed..45d5352bf06a6 100644 --- a/llvm/include/llvm/IR/Constants.h +++ b/llvm/include/llvm/IR/Constants.h @@ -1022,10 +1022,10 @@ class ConstantPtrAuth final : public Constant { friend struct ConstantPtrAuthKeyType; friend class Constant; - constexpr static IntrusiveOperandsAllocMarker AllocMarker{4}; + constexpr static IntrusiveOperandsAllocMarker AllocMarker{5}; ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, - Constant *AddrDisc); + Constant *AddrDisc, Constant *DeactivationSymbol); void *operator new(size_t s) { return User::operator new(s, AllocMarker); } @@ -1035,7 +1035,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// Produce a new ptrauth expression signing the given value using /// the same schema as is stored in one. @@ -1067,6 +1068,10 @@ class ConstantPtrAuth final : public Constant { return !getAddrDiscriminator()->isNullValue(); } + Constant *getDeactivationSymbol() const { + return cast(Op<4>().get()); + } + /// A constant value for the address discriminator which has special /// significance to ctors/dtors lowering. Regular address discrimination can't /// be applied for them since uses of llvm.global_{c|d}tors are disallowed @@ -1094,7 +1099,7 @@ class ConstantPtrAuth final : public Constant { template <> struct OperandTraits - : public FixedNumOperandTraits {}; + : public FixedNumOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPtrAuth, Constant) diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index d10b07ccd91c2..b6369c40235bb 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2534,6 +2534,29 @@ def int_hwasan_check_memaccess_shortgranules_fixedshadow : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i64_ty], [ImmArg>, ImmArg>]>; +// Returns a pointer which may be used to store a pointer at the address +// (first argument + second argument) with protection against use-after-free +// vulnerabilities. Stores via the pointer will cause the stored pointer to be +// blended with some combination of: +// - the field identifier (third argument) +// - the struct address (first argument), if the struct is not trivially +// copyable (fourth argument is true) +// before being stored. When loading from the pointer, the inverse operation +// is done on the loaded pointer after it is loaded. Specifically, when +// targeting AArch64 with pointer authentication enabled and when the fourth +// argument is true, the pointer is signed using the struct address before +// being stored, and authenticated after being loaded. Otherwise it is rotated +// left by 16 bits and a hash of the field identifier is subtracted before +// being stored, and the hash is added and the pointer is rotated right by +// 16 bits after being loaded. If the pointer is used otherwise than for +// loading or storing (e.g. its address escapes), it will cause all blending +// operations using the same field identifier to be disabled throughout the +// program. This is done using special linker relocations. +def int_protected_field_ptr : + DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, + llvm_metadata_ty, llvm_i1_ty], + [IntrNoMem, ImmArg>, ImmArg>]>; + // Xray intrinsics //===----------------------------------------------------------------------===// // Custom event logging for x-ray. diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h index bbd125fd38cf1..6acc55ee55289 100644 --- a/llvm/include/llvm/IR/LLVMContext.h +++ b/llvm/include/llvm/IR/LLVMContext.h @@ -96,6 +96,7 @@ class LLVMContext { OB_ptrauth = 7, // "ptrauth" OB_kcfi = 8, // "kcfi" OB_convergencectrl = 9, // "convergencectrl" + OB_deactivation_symbol = 10, // "deactivation-symbol" }; /// getMDKindID - Return a unique non-zero ID for the specified metadata kind. diff --git a/llvm/include/llvm/SandboxIR/Constant.h b/llvm/include/llvm/SandboxIR/Constant.h index 17f55e973cd76..5243a9476ac64 100644 --- a/llvm/include/llvm/SandboxIR/Constant.h +++ b/llvm/include/llvm/SandboxIR/Constant.h @@ -1096,7 +1096,8 @@ class ConstantPtrAuth final : public Constant { public: /// Return a pointer signed with the specified parameters. static ConstantPtrAuth *get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc); + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol); /// The pointer that is signed in this ptrauth signed pointer. Constant *getPointer() const; @@ -1111,6 +1112,8 @@ class ConstantPtrAuth final : public Constant { /// the only global-initializer user of the ptrauth signed pointer. Constant *getAddrDiscriminator() const; + Constant *getDeactivationSymbol() const; + /// Whether there is any non-null address discriminator. bool hasAddressDiscriminator() const { return cast(Val)->hasAddressDiscriminator(); diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td index e8b460aaf803b..3d8d20c6b2de7 100644 --- a/llvm/include/llvm/Target/Target.td +++ b/llvm/include/llvm/Target/Target.td @@ -682,6 +682,7 @@ class Instruction : InstructionEncoding { // If so, make sure to override // TargetInstrInfo::getInsertSubregLikeInputs. bit variadicOpsAreDefs = false; // Are variadic operands definitions? + bit supportsDeactivationSymbol = false; // Does the instruction have side effects that are not captured by any // operands of the instruction or other flags? diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index db064e1f41f02..847bedeb694d9 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -175,6 +175,8 @@ bool EliminateDuplicatePHINodes(BasicBlock *BB); bool EliminateDuplicatePHINodes(BasicBlock *BB, SmallPtrSetImpl &ToRemove); +bool shouldFoldLoadStoreWithPointerOperandThroughPhi(Value *Ptr); + /// This function is used to do simplification of a CFG. For example, it /// adjusts branches to branches to eliminate the extra hop, it eliminates /// unreachable basic blocks, and does other peephole optimization of the CFG. diff --git a/llvm/lib/Analysis/PtrUseVisitor.cpp b/llvm/lib/Analysis/PtrUseVisitor.cpp index 9c79546f491ef..b34f380b7db5c 100644 --- a/llvm/lib/Analysis/PtrUseVisitor.cpp +++ b/llvm/lib/Analysis/PtrUseVisitor.cpp @@ -22,7 +22,8 @@ void detail::PtrUseVisitorBase::enqueueUsers(Value &I) { if (VisitedUses.insert(&U).second) { UseToVisit NewU = { UseToVisit::UseAndIsOffsetKnownPair(&U, IsOffsetKnown), - Offset + Offset, + ProtectedField, }; Worklist.push_back(std::move(NewU)); } diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 960119bab0933..dfa014aa0bd7d 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -4226,11 +4226,12 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { } case lltok::kw_ptrauth: { // ValID ::= 'ptrauth' '(' ptr @foo ',' i32 - // (',' i64 (',' ptr addrdisc)? )? ')' + // (',' i64 (',' ptr addrdisc (',' ptr ds)? )? )? ')' Lex.Lex(); Constant *Ptr, *Key; - Constant *Disc = nullptr, *AddrDisc = nullptr; + Constant *Disc = nullptr, *AddrDisc = nullptr, + *DeactivationSymbol = nullptr; if (parseToken(lltok::lparen, "expected '(' in constant ptrauth expression") || @@ -4239,11 +4240,14 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { "expected comma in constant ptrauth expression") || parseGlobalTypeAndValue(Key)) return true; - // If present, parse the optional disc/addrdisc. - if (EatIfPresent(lltok::comma)) - if (parseGlobalTypeAndValue(Disc) || - (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc))) - return true; + // If present, parse the optional disc/addrdisc/ds. + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(Disc)) + return true; + if (EatIfPresent(lltok::comma) && parseGlobalTypeAndValue(AddrDisc)) + return true; + if (EatIfPresent(lltok::comma) && + parseGlobalTypeAndValue(DeactivationSymbol)) + return true; if (parseToken(lltok::rparen, "expected ')' in constant ptrauth expression")) return true; @@ -4274,7 +4278,16 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { AddrDisc = ConstantPointerNull::get(PointerType::get(Context, 0)); } - ID.ConstantVal = ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc); + if (DeactivationSymbol) { + if (!DeactivationSymbol->getType()->isPointerTy()) + return error( + ID.Loc, "constant ptrauth deactivation symbol must be a pointer"); + } else { + DeactivationSymbol = ConstantPointerNull::get(PointerType::get(Context, 0)); + } + + ID.ConstantVal = + ConstantPtrAuth::get(Ptr, KeyC, DiscC, AddrDisc, DeactivationSymbol); ID.Kind = ValID::t_Constant; return false; } diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 40e755902b724..c09c3b4f7d38c 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1611,7 +1611,13 @@ Expected BitcodeReader::materializeValue(unsigned StartValID, if (!Disc) return error("ptrauth disc operand must be ConstantInt"); - C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3]); + auto *DeactivationSymbol = + ConstOps.size() > 4 ? ConstOps[4] + : ConstantPointerNull::get(cast( + ConstOps[3]->getType())); + + C = ConstantPtrAuth::get(ConstOps[0], Key, Disc, ConstOps[3], + DeactivationSymbol); break; } case BitcodeConstant::NoCFIOpcode: { @@ -3811,6 +3817,16 @@ Error BitcodeReader::parseConstants() { (unsigned)Record[2], (unsigned)Record[3]}); break; } + case bitc::CST_CODE_PTRAUTH2: { + if (Record.size() < 4) + return error("Invalid ptrauth record"); + // Ptr, Key, Disc, AddrDisc, DeactivationSymbol + V = BitcodeConstant::create( + Alloc, CurTy, BitcodeConstant::ConstantPtrAuthOpcode, + {(unsigned)Record[0], (unsigned)Record[1], (unsigned)Record[2], + (unsigned)Record[3], (unsigned)Record[4]}); + break; + } } assert(V->getType() == getTypeByID(CurTyID) && "Incorrect result type ID"); diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp index 0af70f333f864..af86219c495dd 100644 --- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -195,6 +195,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB, assert(Info.CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); } + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + Info.DeactivationSymbol = cast(Bundle->Inputs[0]); + } + Info.CB = &CB; Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees); Info.CallConv = CallConv; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index f8afb42bf5535..ba5ad836263c6 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2861,6 +2861,9 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { } } + if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_deactivation_symbol)) + MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get()); + return true; } diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 359677027f52f..5592430d1ea73 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -38,8 +38,10 @@ void MachineIRBuilder::setMF(MachineFunction &MF) { //------------------------------------------------------------------------------ MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) { - return BuildMI(getMF(), {getDL(), getPCSections(), getMMRAMetadata()}, - getTII().get(Opcode)); + return BuildMI( + getMF(), + {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()}, + getTII().get(Opcode)); } MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 2409e601ceb46..8eca107385b84 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -120,7 +120,7 @@ MachineInstr::MachineInstr(MachineFunction &MF, const MCInstrDesc &TID, MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) : MCID(&MI.getDesc()), NumOperands(0), Flags(0), AsmPrinterFlags(0), Info(MI.Info), DbgLoc(MI.getDebugLoc()), DebugInstrNum(0), - Opcode(MI.getOpcode()) { + Opcode(MI.getOpcode()), DeactivationSymbol(MI.getDeactivationSymbol()) { assert(DbgLoc.hasTrivialDestructor() && "Expected trivial destructor"); CapOperands = OperandCapacity::get(MI.getNumOperands()); @@ -728,6 +728,8 @@ bool MachineInstr::isIdenticalTo(const MachineInstr &Other, // Call instructions with different CFI types are not identical. if (isCall() && getCFIType() != Other.getCFIType()) return false; + if (getDeactivationSymbol() != Other.getDeactivationSymbol()) + return false; return true; } @@ -2009,6 +2011,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << ','; OS << " cfi-type " << CFIType; } + if (getDeactivationSymbol()) + OS << ", deactivation-symbol " << getDeactivationSymbol()->getName(); if (DebugInstrNum) { if (!FirstOp) diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 9dc1764b49e46..6d57eca420fb7 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -21,9 +21,11 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" @@ -36,6 +38,8 @@ #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h" +#include + using namespace llvm; /// Threshold to leave statically sized memory intrinsic calls. Calls of known @@ -441,6 +445,258 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const { return Changed; } +namespace { + +enum class PointerEncoding { + Rotate, + PACCopyable, + PACNonCopyable, +}; + +bool expandProtectedFieldPtr(Function &Intr) { + Module &M = *Intr.getParent(); + bool IsAArch64 = Triple(M.getTargetTriple()).isAArch64(); + + std::set NonPFPFields; + std::set LoadsStores; + + Type *Int8Ty = Type::getInt8Ty(M.getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); + PointerType *PtrTy = PointerType::get(M.getContext(), 0); + + Function *SignIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_sign, {}); + Function *AuthIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_auth, {}); + + auto *EmuFnTy = FunctionType::get(Int64Ty, {Int64Ty, Int64Ty}, false); + FunctionCallee EmuSignIntr = M.getOrInsertFunction("__emupac_pacda", EmuFnTy); + FunctionCallee EmuAuthIntr = M.getOrInsertFunction("__emupac_autda", EmuFnTy); + + auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { +#ifndef FORCE_EMUPAC + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(SignIntr, {Val, B.getInt32(2), Disc}, DSBundle); +#endif + return B.CreateCall(EmuSignIntr, {Val, Disc}, DSBundle); + }; + + auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { +#ifndef FORCE_EMUPAC + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(AuthIntr, {Val, B.getInt32(2), Disc}, DSBundle); +#endif + return B.CreateCall(EmuAuthIntr, {Val, Disc}, DSBundle); + }; + + for (User *U : Intr.users()) { + auto *Call = cast(U); + auto *FieldName = cast( + cast(Call->getArgOperand(2))->getMetadata()); + std::set VisitedPhis; + + std::function FindLoadsStores; + FindLoadsStores = [&](Instruction *I) { + for (Use &U : I->uses()) { + if (auto *LI = dyn_cast(U.getUser())) { + if (isa(LI->getType())) { + LoadsStores.insert(LI); + continue; + } + } + if (auto *SI = dyn_cast(U.getUser())) { + if (U.getOperandNo() == 1 && + isa(SI->getValueOperand()->getType())) { + LoadsStores.insert(SI); + continue; + } + } + if (auto *P = dyn_cast(U.getUser())) { + if (VisitedPhis.insert(P).second) + FindLoadsStores(P); + continue; + } + // Comparisons against null cannot be used to recover the original + // pointer so we allow them. + if (auto *CI = dyn_cast(U.getUser())) { + if (auto *Op = dyn_cast(CI->getOperand(0))) + if (Op->isNullValue()) + continue; + if (auto *Op = dyn_cast(CI->getOperand(1))) + if (Op->isNullValue()) + continue; + } + NonPFPFields.insert(FieldName); + } + }; + + FindLoadsStores(Call); + } + + for (Instruction *I : LoadsStores) { + std::set Offsets; + std::set Fields; + std::set VisitedPhis; + bool IsNonTriviallyCopyable = false; + + std::function FindFields; + FindFields = [&](Value *V) { + if (auto *Call = dyn_cast(V)) { + if (Call->getCalledOperand() == &Intr) { + Offsets.insert(Call->getArgOperand(1)); + auto *Field = cast( + cast(Call->getArgOperand(2))->getMetadata()); + Fields.insert(Field); + if (cast(Call->getArgOperand(3))->getZExtValue()) + IsNonTriviallyCopyable = true; + return; + } + } + if (auto *P = dyn_cast(V)) { + if (VisitedPhis.insert(P).second) + for (Value *V : P->incoming_values()) + FindFields(V); + return; + } + Fields.insert(nullptr); + }; + FindFields(isa(I) ? cast(I)->getPointerOperand() + : cast(I)->getPointerOperand()); + if (Fields.size() != 1 || Offsets.size() != 1) { + for (Metadata *Field : Fields) + if (Field) + NonPFPFields.insert(Field); + continue; + } + + std::string FieldName = cast(*Fields.begin())->getString().str(); + uint64_t FieldSignature = std::hash()(FieldName); + + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *DS = M.getNamedValue(DSName); + if (!DS) { + DS = new GlobalVariable(M, Int8Ty, false, + GlobalVariable::ExternalWeakLinkage, nullptr, + DSName); + DS->setVisibility(GlobalValue::HiddenVisibility); + } + OperandBundleDef DSBundle("deactivation-symbol", DS); + + PointerEncoding Encoding; + if (!IsAArch64) + Encoding = PointerEncoding::Rotate; + else if (IsNonTriviallyCopyable) + Encoding = PointerEncoding::PACNonCopyable; + else + Encoding = PointerEncoding::PACCopyable; + + if (auto *LI = dyn_cast(I)) { + auto *FieldAddr = LI->getPointerOperand(); + IRBuilder<> B(LI->getNextNode()); + auto *LIInt = cast(B.CreatePtrToInt(LI, B.getInt64Ty())); + Value *Auth; + switch (Encoding) { + case PointerEncoding::Rotate: + Auth = B.CreateAdd(LIInt, B.getInt64(FieldSignature & 0xff)); + Auth = B.CreateOr(B.CreateLShr(Auth, 16), B.CreateShl(Auth, 48)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Auth = CreateAuth(B, LIInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Auth = + CreateAuth(B, LIInt, B.getInt64(FieldSignature & 0xffff), DSBundle); + break; + } + LI->replaceAllUsesWith(B.CreateIntToPtr(Auth, B.getPtrTy())); + LIInt->setOperand(0, LI); + } else if (auto *SI = dyn_cast(I)) { + IRBuilder<> B(SI); + auto *FieldAddr = SI->getPointerOperand(); + auto *SIValInt = + B.CreatePtrToInt(SI->getValueOperand(), B.getInt64Ty()); + Value *Sign; + switch (Encoding) { + case PointerEncoding::Rotate: + Sign = + B.CreateOr(B.CreateLShr(SIValInt, 48), B.CreateShl(SIValInt, 16)); + Sign = B.CreateSub(Sign, B.getInt64(FieldSignature & 0xff)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Sign = CreateSign(B, SIValInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Sign = CreateSign(B, SIValInt, B.getInt64(FieldSignature & 0xffff), + DSBundle); + break; + } + SI->setOperand(0, B.CreateIntToPtr(Sign, B.getPtrTy())); + } + } + + for (User *U : llvm::make_early_inc_range(Intr.users())) { + auto *Call = cast(U); + auto *Struct = Call->getArgOperand(0); + auto *Offset = Call->getArgOperand(1); + + IRBuilder<> B(Call); + if (cast(Offset)->getZExtValue() == 0) + Call->replaceAllUsesWith(Struct); + else + Call->replaceAllUsesWith(B.CreateGEP(B.getInt8Ty(), Struct, {Offset})); + Call->eraseFromParent(); + } + + if (!NonPFPFields.empty()) { + Constant *Nop = + ConstantExpr::getIntToPtr(ConstantInt::get(Int64Ty, 0xd503201f), PtrTy); + std::set LocalNonPFPFieldNames; + for (auto *Field : NonPFPFields) + LocalNonPFPFieldNames.insert(cast(Field)->getString().str()); + for (auto &FieldName : LocalNonPFPFieldNames) { + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *OldDS = M.getNamedValue(DSName); + GlobalValue *DS = GlobalAlias::create( + Int8Ty, 0, GlobalValue::ExternalLinkage, DSName, Nop, &M); + DS->setVisibility(GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } + } + return true; +} + +} + bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { bool Changed = false; for (Function &F : M) { @@ -572,6 +828,9 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { return lowerUnaryVectorIntrinsicAsLoop(M, CI); }); break; + case Intrinsic::protected_field_ptr: + Changed |= expandProtectedFieldPtr(F); + break; } } return Changed; diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 5182e4124f548..8528543211223 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -15,10 +15,12 @@ #include "InstrEmitter.h" #include "SDNodeDbgValue.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -61,6 +63,8 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses, unsigned N = Node->getNumOperands(); while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue) --N; + if (N && Node->getOperand(N - 1).getOpcode() == ISD::DEACTIVATION_SYMBOL) + --N; // Ignore deactivation symbol if it exists. if (N && Node->getOperand(N - 1).getValueType() == MVT::Other) --N; // Ignore chain if it exists. @@ -1219,15 +1223,23 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, } } - if (SDNode *GluedNode = Node->getGluedNode()) { - // FIXME: Possibly iterate over multiple glue nodes? - if (GluedNode->getOpcode() == - ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { - Register VReg = getVR(GluedNode->getOperand(0), VRBaseMap); - MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, - /*isImp=*/true); - MIB->addOperand(MO); - } + unsigned Op = Node->getNumOperands(); + if (Op != 0 && Node->getOperand(Op - 1)->getOpcode() == + ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) { + Register VReg = getVR(Node->getOperand(Op - 1)->getOperand(0), VRBaseMap); + MachineOperand MO = MachineOperand::CreateReg(VReg, /*isDef=*/false, + /*isImp=*/true); + MIB->addOperand(MO); + Op--; + } + + if (Op != 0 && + Node->getOperand(Op - 1)->getOpcode() == ISD::DEACTIVATION_SYMBOL) { + MI->setDeactivationSymbol( + *MF, const_cast( + cast(Node->getOperand(Op - 1)) + ->getGlobal())); + Op--; } // Run post-isel target hook to adjust this instruction if needed. @@ -1248,7 +1260,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, llvm_unreachable("This target-independent node should have been selected!"); case ISD::EntryToken: case ISD::MERGE_VALUES: - case ISD::TokenFactor: // fall thru + case ISD::TokenFactor: + case ISD::DEACTIVATION_SYMBOL: break; case ISD::CopyToReg: { Register DestReg = cast(Node->getOperand(1))->getReg(); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 7ce4eebf685e1..4981272c69d71 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1913,6 +1913,21 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, return SDValue(N, 0); } +SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) { + SDVTList VTs = getVTList(MVT::Untyped); + FoldingSetNodeID ID; + AddNodeIDNode(ID, ISD::DEACTIVATION_SYMBOL, VTs, {}); + ID.AddPointer(GV); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) + return SDValue(E, 0); + + auto *N = newSDNode(GV, VTs); + CSEMap.InsertNode(N, IP); + InsertNode(N); + return SDValue(N, 0); +} + SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; SDVTList VTs = getVTList(VT); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 6db2a5ffbfb84..727bec637bc79 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -45,6 +45,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/SwiftErrorValueTracking.h" @@ -5280,6 +5281,13 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Create the node. SDValue Result; + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + auto *Sym = Bundle->Inputs[0].get(); + SDValue SDSym = getValue(Sym); + SDSym = DAG.getDeactivationSymbol(cast(Sym)); + Ops.push_back(SDSym); + } + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { auto *Token = Bundle->Inputs[0].get(); SDValue ConvControlToken = getValue(Token); @@ -8928,6 +8936,11 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, ConvControlToken = getValue(Token); } + GlobalValue *DeactivationSymbol = nullptr; + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol)) { + DeactivationSymbol = cast(Bundle->Inputs[0].get()); + } + TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) @@ -8937,7 +8950,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, .setIsPreallocated( CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) .setCFIType(CFIType) - .setConvergenceControlToken(ConvControlToken); + .setConvergenceControlToken(ConvControlToken) + .setDeactivationSymbol(DeactivationSymbol); // Set the pointer authentication info if we have it. if (PAI) { @@ -9554,7 +9568,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { {LLVMContext::OB_deopt, LLVMContext::OB_funclet, LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi, - LLVMContext::OB_convergencectrl}) && + LLVMContext::OB_convergencectrl, + LLVMContext::OB_deactivation_symbol}) && "Cannot lower calls with arbitrary operand bundles!"); SDValue Callee = getValue(I.getCalledOperand()); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 63ee2d78cfa1b..45d9863835e10 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3258,6 +3258,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case ISD::LIFETIME_START: case ISD::LIFETIME_END: case ISD::PSEUDO_PROBE: + case ISD::DEACTIVATION_SYMBOL: NodeToMatch->setNodeId(-1); // Mark selected. return; case ISD::AssertSext: @@ -3334,7 +3335,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // These are the current input chain and glue for use when generating nodes. // Various Emit operations change these. For example, emitting a copytoreg // uses and updates these. - SDValue InputChain, InputGlue; + SDValue InputChain, InputGlue, DeactivationSymbol; // ChainNodesMatched - If a pattern matches nodes that have input/output // chains, the OPC_EmitMergeInputChains operation is emitted which indicates @@ -3487,6 +3488,15 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, InputGlue = N->getOperand(N->getNumOperands()-1); continue; + case OPC_CaptureDeactivationSymbol: + // If the current node has a deactivation symbol, capture it in + // DeactivationSymbol. + if (N->getNumOperands() != 0 && + N->getOperand(N->getNumOperands() - 1).getOpcode() == + ISD::DEACTIVATION_SYMBOL) + DeactivationSymbol = N->getOperand(N->getNumOperands()-1); + continue; + case OPC_MoveChild: { unsigned ChildNo = MatcherTable[MatcherIndex++]; if (ChildNo >= N.getNumOperands()) @@ -4169,6 +4179,8 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, // If this has chain/glue inputs, add them. if (EmitNodeInfo & OPFL_Chain) Ops.push_back(InputChain); + if (DeactivationSymbol.getNode() != nullptr) + Ops.push_back(DeactivationSymbol); if ((EmitNodeInfo & OPFL_GlueInput) && InputGlue.getNode() != nullptr) Ops.push_back(InputGlue); diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp index 7e0a1e2a8a06e..1ba47db017781 100644 --- a/llvm/lib/CodeGen/TargetInstrInfo.cpp +++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp @@ -1380,7 +1380,7 @@ void TargetInstrInfo::reassociateOps( const MCInstrDesc &MCID, Register DestReg) { return MachineInstrBuilder( MF, MF.CreateMachineInstr(MCID, MIMD.getDL(), /*NoImpl=*/true)) - .setPCSections(MIMD.getPCSections()) + .copyMIMetadata(MIMD) .addReg(DestReg, RegState::Define); }; diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index 79547b299a903..5efb321967008 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -1630,12 +1630,14 @@ static void WriteConstantInternal(raw_ostream &Out, const Constant *CV, if (const ConstantPtrAuth *CPA = dyn_cast(CV)) { Out << "ptrauth ("; - // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC]?]?) + // ptrauth (ptr CST, i32 KEY[, i64 DISC[, ptr ADDRDISC[, ptr DS]?]?]?) unsigned NumOpsToWrite = 2; if (!CPA->getOperand(2)->isNullValue()) NumOpsToWrite = 3; if (!CPA->getOperand(3)->isNullValue()) NumOpsToWrite = 4; + if (!CPA->getOperand(4)->isNullValue()) + NumOpsToWrite = 5; ListSeparator LS; for (unsigned i = 0, e = NumOpsToWrite; i != e; ++i) { diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp index fb659450bfeeb..007d36d19f373 100644 --- a/llvm/lib/IR/Constants.cpp +++ b/llvm/lib/IR/Constants.cpp @@ -2072,19 +2072,22 @@ Value *NoCFIValue::handleOperandChangeImpl(Value *From, Value *To) { // ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { - Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc}; + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { + Constant *ArgVec[] = {Ptr, Key, Disc, AddrDisc, DeactivationSymbol}; ConstantPtrAuthKeyType MapKey(ArgVec); LLVMContextImpl *pImpl = Ptr->getContext().pImpl; return pImpl->ConstantPtrAuths.getOrCreate(Ptr->getType(), MapKey); } ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { - return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator()); + return get(Pointer, getKey(), getDiscriminator(), getAddrDiscriminator(), + getDeactivationSymbol()); } ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) : Constant(Ptr->getType(), Value::ConstantPtrAuthVal, AllocMarker) { assert(Ptr->getType()->isPointerTy()); assert(Key->getBitWidth() == 32); @@ -2094,6 +2097,7 @@ ConstantPtrAuth::ConstantPtrAuth(Constant *Ptr, ConstantInt *Key, setOperand(1, Key); setOperand(2, Disc); setOperand(3, AddrDisc); + setOperand(4, DeactivationSymbol); } /// Remove the constant from the constant table. diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h index e5c9622e09927..bf9d8ab952271 100644 --- a/llvm/lib/IR/ConstantsContext.h +++ b/llvm/lib/IR/ConstantsContext.h @@ -545,7 +545,8 @@ struct ConstantPtrAuthKeyType { ConstantPtrAuth *create(TypeClass *Ty) const { return new ConstantPtrAuth(Operands[0], cast(Operands[1]), - cast(Operands[2]), Operands[3]); + cast(Operands[2]), Operands[3], + Operands[4]); } }; diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index f4b03e8cb8aa3..6190ebdac16d4 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -1687,7 +1687,9 @@ LLVMValueRef LLVMConstantPtrAuth(LLVMValueRef Ptr, LLVMValueRef Key, LLVMValueRef Disc, LLVMValueRef AddrDisc) { return wrap(ConstantPtrAuth::get( unwrap(Ptr), unwrap(Key), - unwrap(Disc), unwrap(AddrDisc))); + unwrap(Disc), unwrap(AddrDisc), + ConstantPointerNull::get( + cast(unwrap(AddrDisc)->getType())))); } /*-- Opcode mapping */ diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp index 447e5d92e0b99..9946fe6b9af13 100644 --- a/llvm/lib/IR/LLVMContext.cpp +++ b/llvm/lib/IR/LLVMContext.cpp @@ -53,6 +53,8 @@ static StringRef knownBundleName(unsigned BundleTagID) { return "kcfi"; case LLVMContext::OB_convergencectrl: return "convergencectrl"; + case LLVMContext::OB_deactivation_symbol: + return "deactivation-symbol"; default: llvm_unreachable("unknown bundle id"); } @@ -76,7 +78,7 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) { } for (unsigned BundleTagID = LLVMContext::OB_deopt; - BundleTagID <= LLVMContext::OB_convergencectrl; ++BundleTagID) { + BundleTagID <= LLVMContext::OB_deactivation_symbol; ++BundleTagID) { [[maybe_unused]] const auto *Entry = pImpl->getOrInsertBundleTag(knownBundleName(BundleTagID)); assert(Entry->second == BundleTagID && "operand bundle id drifted!"); diff --git a/llvm/lib/SandboxIR/Constant.cpp b/llvm/lib/SandboxIR/Constant.cpp index 3e13c935c4281..0a28cf9feeb4d 100644 --- a/llvm/lib/SandboxIR/Constant.cpp +++ b/llvm/lib/SandboxIR/Constant.cpp @@ -421,10 +421,12 @@ PointerType *NoCFIValue::getType() const { } ConstantPtrAuth *ConstantPtrAuth::get(Constant *Ptr, ConstantInt *Key, - ConstantInt *Disc, Constant *AddrDisc) { + ConstantInt *Disc, Constant *AddrDisc, + Constant *DeactivationSymbol) { auto *LLVMC = llvm::ConstantPtrAuth::get( cast(Ptr->Val), cast(Key->Val), - cast(Disc->Val), cast(AddrDisc->Val)); + cast(Disc->Val), cast(AddrDisc->Val), + cast(DeactivationSymbol->Val)); return cast(Ptr->getContext().getOrCreateConstant(LLVMC)); } @@ -448,6 +450,11 @@ Constant *ConstantPtrAuth::getAddrDiscriminator() const { cast(Val)->getAddrDiscriminator()); } +Constant *ConstantPtrAuth::getDeactivationSymbol() const { + return Ctx.getOrCreateConstant( + cast(Val)->getDeactivationSymbol()); +} + ConstantPtrAuth *ConstantPtrAuth::getWithSameSchema(Constant *Pointer) const { auto *LLVMC = cast(Val)->getWithSameSchema( cast(Pointer->Val)); diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt index 2754c97fce6c1..1b18eb7c6346b 100644 --- a/llvm/lib/Support/CMakeLists.txt +++ b/llvm/lib/Support/CMakeLists.txt @@ -368,3 +368,8 @@ if(LLVM_WITH_Z3) ${Z3_INCLUDE_DIR} ) endif() + +target_include_directories(LLVMSupport SYSTEM + PRIVATE + ${LLVM_THIRD_PARTY_DIR}/siphash/include + ) diff --git a/llvm/lib/Support/SipHash.cpp b/llvm/lib/Support/SipHash.cpp index 68545913a4f59..682e9231c776f 100644 --- a/llvm/lib/Support/SipHash.cpp +++ b/llvm/lib/Support/SipHash.cpp @@ -15,9 +15,9 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Endian.h" +#include "siphash/SipHash.h" #include using namespace llvm; @@ -25,135 +25,6 @@ using namespace support; #define DEBUG_TYPE "llvm-siphash" -// Lightly adapted from the SipHash reference C implementation: -// https://github.com/veorq/SipHash -// by Jean-Philippe Aumasson and Daniel J. Bernstein - -#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) - -#define SIPROUND \ - do { \ - v0 += v1; \ - v1 = ROTL(v1, 13); \ - v1 ^= v0; \ - v0 = ROTL(v0, 32); \ - v2 += v3; \ - v3 = ROTL(v3, 16); \ - v3 ^= v2; \ - v0 += v3; \ - v3 = ROTL(v3, 21); \ - v3 ^= v0; \ - v2 += v1; \ - v1 = ROTL(v1, 17); \ - v1 ^= v2; \ - v2 = ROTL(v2, 32); \ - } while (0) - -namespace { - -/// Computes a SipHash value -/// -/// \param in: pointer to input data (read-only) -/// \param inlen: input data length in bytes (any size_t value) -/// \param k: reference to the key data 16-byte array (read-only) -/// \returns output data, must be 8 or 16 bytes -/// -template -void siphash(const unsigned char *in, uint64_t inlen, - const unsigned char (&k)[16], unsigned char (&out)[outlen]) { - - const unsigned char *ni = (const unsigned char *)in; - const unsigned char *kk = (const unsigned char *)k; - - static_assert(outlen == 8 || outlen == 16, "result should be 8 or 16 bytes"); - - uint64_t v0 = UINT64_C(0x736f6d6570736575); - uint64_t v1 = UINT64_C(0x646f72616e646f6d); - uint64_t v2 = UINT64_C(0x6c7967656e657261); - uint64_t v3 = UINT64_C(0x7465646279746573); - uint64_t k0 = endian::read64le(kk); - uint64_t k1 = endian::read64le(kk + 8); - uint64_t m; - int i; - const unsigned char *end = ni + inlen - (inlen % sizeof(uint64_t)); - const int left = inlen & 7; - uint64_t b = ((uint64_t)inlen) << 56; - v3 ^= k1; - v2 ^= k0; - v1 ^= k1; - v0 ^= k0; - - if (outlen == 16) - v1 ^= 0xee; - - for (; ni != end; ni += 8) { - m = endian::read64le(ni); - v3 ^= m; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= m; - } - - switch (left) { - case 7: - b |= ((uint64_t)ni[6]) << 48; - LLVM_FALLTHROUGH; - case 6: - b |= ((uint64_t)ni[5]) << 40; - LLVM_FALLTHROUGH; - case 5: - b |= ((uint64_t)ni[4]) << 32; - LLVM_FALLTHROUGH; - case 4: - b |= ((uint64_t)ni[3]) << 24; - LLVM_FALLTHROUGH; - case 3: - b |= ((uint64_t)ni[2]) << 16; - LLVM_FALLTHROUGH; - case 2: - b |= ((uint64_t)ni[1]) << 8; - LLVM_FALLTHROUGH; - case 1: - b |= ((uint64_t)ni[0]); - break; - case 0: - break; - } - - v3 ^= b; - - for (i = 0; i < cROUNDS; ++i) - SIPROUND; - - v0 ^= b; - - if (outlen == 16) - v2 ^= 0xee; - else - v2 ^= 0xff; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - endian::write64le(out, b); - - if (outlen == 8) - return; - - v1 ^= 0xdd; - - for (i = 0; i < dROUNDS; ++i) - SIPROUND; - - b = v0 ^ v1 ^ v2 ^ v3; - endian::write64le(out + 8, b); -} - -} // end anonymous namespace - void llvm::getSipHash_2_4_64(ArrayRef In, const uint8_t (&K)[16], uint8_t (&Out)[8]) { siphash<2, 4>(In.data(), In.size(), K, Out); diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index e3ad085bc5a29..5ce64c9a3a091 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -54,6 +54,7 @@ #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCValue.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" @@ -69,15 +70,6 @@ using namespace llvm; -enum PtrauthCheckMode { Default, Unchecked, Poison, Trap }; -static cl::opt PtrauthAuthChecks( - "aarch64-ptrauth-auth-checks", cl::Hidden, - cl::values(clEnumValN(Unchecked, "none", "don't test for failure"), - clEnumValN(Poison, "poison", "poison on failure"), - clEnumValN(Trap, "trap", "trap on failure")), - cl::desc("Check pointer authentication auth/resign failures"), - cl::init(Default)); - #define DEBUG_TYPE "asm-printer" namespace { @@ -93,6 +85,7 @@ class AArch64AsmPrinter : public AsmPrinter { bool EnableImportCallOptimization = false; DenseMap>> SectionToImportedFunctionCalls; + unsigned PAuthIFuncNextUniqueID = 1; public: AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) @@ -200,6 +193,10 @@ class AArch64AsmPrinter : public AsmPrinter { // authenticating) void LowerLOADgotAUTH(const MachineInstr &MI); + const MCExpr *emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr); + /// tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst); @@ -2227,6 +2224,159 @@ void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, BRInst); } +static void emitAddress(MCStreamer &Streamer, MCRegister Reg, + const MCExpr *Expr, bool DSOLocal, + const MCSubtargetInfo &STI) { + MCValue Val; + if (!Expr->evaluateAsRelocatable(Val, nullptr)) + report_fatal_error("emitAddress could not evaluate"); + if (DSOLocal) { + Streamer.emitInstruction( + MCInstBuilder(AArch64::ADRP) + .addReg(Reg) + .addExpr(AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, + Streamer.getContext())), + STI); + Streamer.emitInstruction( + MCInstBuilder(AArch64::ADDXri) + .addReg(Reg) + .addReg(Reg) + .addExpr(AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, + Streamer.getContext())) + .addImm(0), + STI); + } else { + Streamer.emitInstruction(MCInstBuilder(AArch64::ADRP) + .addReg(Reg) + .addExpr(AArch64MCExpr::create( + Val.getSymA(), AArch64MCExpr::VK_GOT_PAGE, + Streamer.getContext())), + STI); + Streamer.emitInstruction(MCInstBuilder(AArch64::LDRXui) + .addReg(Reg) + .addReg(Reg) + .addExpr(AArch64MCExpr::create( + Val.getSymA(), AArch64MCExpr::VK_GOT_LO12, + Streamer.getContext())), + STI); + if (Val.getConstant()) + Streamer.emitInstruction(MCInstBuilder(AArch64::ADDXri) + .addReg(Reg) + .addReg(Reg) + .addImm(Val.getConstant()) + .addImm(0), + STI); + } +} + +static bool targetSupportsPAuthRelocation(const Triple &TT, + const MCExpr *Target, + const MCExpr *DSExpr) { + // No released version of glibc supports PAuth relocations. + if (TT.isOSGlibc()) + return false; + + // We emit PAuth constants as IRELATIVE relocations in cases where the + // constant cannot be represented as a PAuth relocation: + // 1) There is a deactivation symbol. + // 2) The signed value is not a symbol. + return !DSExpr && !isa(Target); +} + +static bool targetSupportsIRelativeRelocation(const Triple &TT) { + // IFUNCs are ELF-only. + if (!TT.isOSBinFormatELF()) + return false; + + // musl doesn't support IFUNCs. + if (TT.isMusl()) + return false; + + return true; +} + +const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative( + const MCExpr *Target, uint16_t Disc, AArch64PACKey::ID KeyID, + bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) { + const Triple &TT = TM.getTargetTriple(); + + // We only emit an IRELATIVE relocation if the target supports IRELATIVE and + // does not support the kind of PAuth relocation that we are trying to emit. + if (targetSupportsPAuthRelocation(TT, Target, DSExpr) || + !targetSupportsIRelativeRelocation(TT)) + return nullptr; + + // For now, only the DA key is supported. + if (KeyID != AArch64PACKey::DA) + return nullptr; + + std::unique_ptr STI( + TM.getTarget().createMCSubtargetInfo(TT.str(), "", "")); + assert(STI && "Unable to create subtarget info"); + + MCSymbol *Place = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(Place); + OutStreamer->pushSection(); + + OutStreamer->switchSection(OutStreamer->getContext().getELFSection( + ".text.startup", ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHF_EXECINSTR, + 0, "", true, PAuthIFuncNextUniqueID++, nullptr)); + + MCSymbol *IFuncSym = + OutStreamer->getContext().createLinkerPrivateSymbol("pauth_ifunc"); + OutStreamer->emitSymbolAttribute(IFuncSym, MCSA_ELF_TypeIndFunction); + OutStreamer->emitLabel(IFuncSym); + if (isa(Target)) { + OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi) + .addReg(AArch64::X0) + .addExpr(Target) + .addImm(0), + *STI); + } else { + emitAddress(*OutStreamer, AArch64::X0, Target, IsDSOLocal, *STI); + } + if (HasAddressDiversity) { + auto *PlacePlusDisc = MCBinaryExpr::createAdd( + MCSymbolRefExpr::create(Place, OutStreamer->getContext()), + MCConstantExpr::create(static_cast(Disc), + OutStreamer->getContext()), + OutStreamer->getContext()); + emitAddress(*OutStreamer, AArch64::X1, PlacePlusDisc, /*IsDSOLocal=*/true, + *STI); + } else { + emitMOVZ(AArch64::X1, Disc, 0); + } + + MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol(); + OutStreamer->emitLabel(PrePACInst); + + // We don't know the subtarget because this is being emitted for a global + // initializer. Because the performance of IFUNC resolvers is unimportant, we + // always call the EmuPAC runtime, which will end up using the PAC instruction + // if the target supports PAC. + MCSymbol *EmuPAC = + OutStreamer->getContext().getOrCreateSymbol("__emupac_pacda"); + const MCSymbolRefExpr *EmuPACRef = + MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext()); + OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef), + *STI); + + if (DSExpr) { + auto *PrePACInstExpr = + MCSymbolRefExpr::create(PrePACInst, OutStreamer->getContext()); + OutStreamer->emitRelocDirective(*PrePACInstExpr, "R_AARCH64_INST32", DSExpr, + SMLoc(), *STI); + } + + // We need a RET despite the above tail call because the deactivation symbol + // may replace it with a NOP. + OutStreamer->emitInstruction(MCInstBuilder(AArch64::RET).addReg(AArch64::LR), + *STI); + OutStreamer->popSection(); + + return MCSymbolRefExpr::create(IFuncSym, OutStreamer->getContext()); +} + const MCExpr * AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { MCContext &Ctx = OutContext; @@ -2238,22 +2388,26 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { auto *BaseGVB = dyn_cast(BaseGV); - // If we can't understand the referenced ConstantExpr, there's nothing - // else we can do: emit an error. - if (!BaseGVB) { - BaseGV->getContext().emitError( - "cannot resolve target base/addend of ptrauth constant"); - return nullptr; + const MCExpr *Sym; + if (BaseGVB) { + // If there is an addend, turn that into the appropriate MCExpr. + Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx); + if (Offset.sgt(0)) + Sym = MCBinaryExpr::createAdd( + Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx); + else if (Offset.slt(0)) + Sym = MCBinaryExpr::createSub( + Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx); + } else { + Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx); } - // If there is an addend, turn that into the appropriate MCExpr. - const MCExpr *Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx); - if (Offset.sgt(0)) - Sym = MCBinaryExpr::createAdd( - Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx); - else if (Offset.slt(0)) - Sym = MCBinaryExpr::createSub( - Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx); + const MCExpr *DSExpr = nullptr; + if (auto *DS = dyn_cast(CPA.getDeactivationSymbol())) { + if (isa(DS)) + return Sym; + DSExpr = MCSymbolRefExpr::create(getSymbol(DS), Ctx); + } uint64_t KeyID = CPA.getKey()->getZExtValue(); // We later rely on valid KeyID value in AArch64PACKeyIDToString call from @@ -2268,6 +2422,16 @@ AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) { report_fatal_error("AArch64 PAC Discriminator '" + Twine(Disc) + "' out of range [0, 0xFFFF]"); + // Check if we need to represent this with an IRELATIVE and emit it if so. + if (auto *IFuncSym = emitPAuthRelocationAsIRelative( + Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), + BaseGVB && BaseGVB->isDSOLocal(), DSExpr)) + return IFuncSym; + + if (DSExpr) + report_fatal_error("deactivation symbols unsupported in constant " + "expressions on this target"); + // Finally build the complete @AUTH expr. return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(), Ctx); @@ -2757,6 +2921,23 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { OutStreamer->emitLabel(LOHLabel); } + if (MI->getDeactivationSymbol()) { + if (isa(MI->getDeactivationSymbol())) { + // Just emit the nop directly. + EmitToStreamer(MCInstBuilder(AArch64::HINT).addImm(0)); + return; + } + MCSymbol *Dot = OutContext.createTempSymbol(); + OutStreamer->emitLabel(Dot); + const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Dot, OutContext); + + MCSymbol *DS = + OutContext.getOrCreateSymbol(MI->getDeactivationSymbol()->getName()); + const MCExpr *DSExpr = MCSymbolRefExpr::create(DS, OutContext); + OutStreamer->emitRelocDirective(*DeactDotExpr, "R_AARCH64_INST32", DSExpr, + SMLoc(), *TM.getMCSubtargetInfo()); + } + AArch64TargetStreamer *TS = static_cast(OutStreamer->getTargetStreamer()); // Do any manual lowerings. @@ -2866,7 +3047,7 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { return; } - case AArch64::AUT: + case AArch64::AUTx16x17: case AArch64::AUTPAC: emitPtrauthAuthResign(MI); return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 22083460b400a..2eafb8eae2082 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -361,7 +361,7 @@ class AArch64DAGToDAGISel : public SelectionDAGISel { bool tryIndexedLoad(SDNode *N); - void SelectPtrauthAuth(SDNode *N); + void SelectPtrauthAuthX16X17(SDNode *N); void SelectPtrauthResign(SDNode *N); bool trySelectStackSlotTagP(SDNode *N); @@ -1521,7 +1521,7 @@ extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) { AddrDisc); } -void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { +void AArch64DAGToDAGISel::SelectPtrauthAuthX16X17(SDNode *N) { SDLoc DL(N); // IntrinsicID is operand #0 SDValue Val = N->getOperand(1); @@ -1539,7 +1539,7 @@ void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) { AArch64::X16, Val, SDValue()); SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)}; - SDNode *AUT = CurDAG->getMachineNode(AArch64::AUT, DL, MVT::i64, Ops); + SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTx16x17, DL, MVT::i64, Ops); ReplaceNode(N, AUT); } @@ -5613,7 +5613,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { return; case Intrinsic::ptrauth_auth: - SelectPtrauthAuth(Node); + if (!Subtarget->isX16X17Safer(CurDAG->getMachineFunction())) + break; + SelectPtrauthAuthX16X17(Node); return; case Intrinsic::ptrauth_resign: diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 1c8e3afdfd718..793ab48b76903 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -9614,6 +9614,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, if (InGlue.getNode()) Ops.push_back(InGlue); + if (CLI.DeactivationSymbol) + Ops.push_back(DAG.getDeactivationSymbol(CLI.DeactivationSymbol)); + // If we're doing a tall call, use a TC_RETURN here rather than an // actual call instruction. if (IsTailCall) { diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 255cd0ec5840c..c26c4562a17fe 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -2403,6 +2403,7 @@ class BImm pattern> let Inst{25-0} = addr; let DecoderMethod = "DecodeUnconditionalBranch"; + let supportsDeactivationSymbol = true; } class BranchImm pattern> @@ -2460,6 +2461,7 @@ class SignAuthOneData opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = Rn; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthZero opcode_prefix, bits<2> opcode, string asm, @@ -2473,6 +2475,7 @@ class SignAuthZero opcode_prefix, bits<2> opcode, string asm, let Inst{11-10} = opcode; let Inst{9-5} = 0b11111; let Inst{4-0} = Rd; + let supportsDeactivationSymbol = true; } class SignAuthTwoOperand opc, string asm, diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 6c61e3a613f6f..98a35b3840771 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -66,6 +66,14 @@ def HasLOR : Predicate<"Subtarget->hasLOR()">, def HasPAuth : Predicate<"Subtarget->hasPAuth()">, AssemblerPredicateWithAll<(all_of FeaturePAuth), "pauth">; +// On most operating systems, the x16 and x17 registers are not special, so +// there is no benefit, and only a code size cost, to constraining PAC +// instructions to only using them. This predicate may be used to guard patterns +// that allow PAC instructions to be used with any register. +let RecomputePerFunction = 1 in { + def X16X17NotSafer : Predicate<"!Subtarget->isX16X17Safer(*MF)">; +} + def HasPAuthLR : Predicate<"Subtarget->hasPAuthLR()">, AssemblerPredicateWithAll<(all_of FeaturePAuthLR), "pauth-lr">; @@ -1820,7 +1828,9 @@ let Predicates = [HasPAuth] in { } defm PAC : SignAuth<0b000, 0b010, "pac", int_ptrauth_sign>; - defm AUT : SignAuth<0b001, 0b011, "aut", null_frag>; + let Predicates = [HasPAuth, X16X17NotSafer] in { + defm AUT : SignAuth<0b001, 0b011, "aut", int_ptrauth_auth>; + } def XPACI : ClearAuth<0, "xpaci">; def : Pat<(int_ptrauth_strip GPR64:$Rd, 0), (XPACI GPR64:$Rd)>; @@ -1912,10 +1922,11 @@ let Predicates = [HasPAuth] in { defm LDRAB : AuthLoad<1, "ldrab", simm10Scaled>; // AUT pseudo. - // This directly manipulates x16/x17, which are the only registers the OS - // guarantees are safe to use for sensitive operations. - def AUT : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc, GPR64noip:$AddrDisc), - []>, Sched<[WriteI, ReadI]> { + // This directly manipulates x16/x17, which are the only registers that + // certain OSs guarantee are safe to use for sensitive operations. + def AUTx16x17 : Pseudo<(outs), (ins i32imm:$Key, i64imm:$Disc, + GPR64noip:$AddrDisc), + []>, Sched<[WriteI, ReadI]> { let isCodeGenOnly = 1; let hasSideEffects = 1; let mayStore = 0; @@ -1926,8 +1937,8 @@ let Predicates = [HasPAuth] in { } // AUT and re-PAC a value, using different keys/data. - // This directly manipulates x16/x17, which are the only registers the OS - // guarantees are safe to use for sensitive operations. + // This directly manipulates x16/x17, which are the only registers that + // certain OSs guarantee are safe to use for sensitive operations. def AUTPAC : Pseudo<(outs), (ins i32imm:$AUTKey, i64imm:$AUTDisc, GPR64noip:$AUTAddrDisc, diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp index 7b4ded6322098..6044e84ab6640 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp @@ -102,6 +102,16 @@ static cl::opt UseScalarIncVL("sve-use-scalar-inc-vl", cl::init(false), cl::Hidden, cl::desc("Prefer add+cnt over addvl/inc/dec")); +cl::opt llvm::PtrauthAuthChecks( + "aarch64-ptrauth-auth-checks", cl::Hidden, + cl::values(clEnumValN(PtrauthCheckMode::Unchecked, "none", + "don't test for failure"), + clEnumValN(PtrauthCheckMode::Poison, "poison", + "poison on failure"), + clEnumValN(PtrauthCheckMode::Trap, "trap", "trap on failure")), + cl::desc("Check pointer authentication auth/resign failures"), + cl::init(PtrauthCheckMode::Default)); + unsigned AArch64Subtarget::getVectorInsertExtractBaseCost() const { if (OverrideVectorInsertExtractBaseCost.getNumOccurrences() > 0) return OverrideVectorInsertExtractBaseCost; @@ -663,6 +673,19 @@ AArch64Subtarget::getPtrAuthBlockAddressDiscriminatorIfEnabled( (Twine(ParentFn.getName()) + " blockaddress").str()); } +bool AArch64Subtarget::isX16X17Safer(const MachineFunction &MF) const { + // The Darwin kernel implements special protections for x16 and x17 so we + // should prefer to use those registers on that platform. + if (isTargetDarwin()) + return true; + // Traps are only implemented for the pseudo instructions, but are only + // necessary if FEAT_FPAC is not implemented. + if (hasFPAC()) + return false; + return MF.getFunction().hasFnAttribute("ptrauth-auth-traps") || + PtrauthAuthChecks == PtrauthCheckMode::Trap; +} + bool AArch64Subtarget::enableMachinePipeliner() const { return getSchedModel().hasInstrSchedModel(); } diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h index f5ffc72cae537..4e8f5f85146bd 100644 --- a/llvm/lib/Target/AArch64/AArch64Subtarget.h +++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h @@ -26,6 +26,7 @@ #include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" +#include "llvm/Support/CommandLine.h" #define GET_SUBTARGETINFO_HEADER #include "AArch64GenSubtargetInfo.inc" @@ -35,6 +36,9 @@ class GlobalValue; class StringRef; class Triple; +enum class PtrauthCheckMode { Default, Unchecked, Poison, Trap }; +extern cl::opt PtrauthAuthChecks; + class AArch64Subtarget final : public AArch64GenSubtargetInfo { public: enum ARMProcFamilyEnum : uint8_t { @@ -318,6 +322,10 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo { } } + /// Returns whether the operating system makes it safer to store sensitive + /// values in x16 and x17 as opposed to other registers. + bool isX16X17Safer(const MachineFunction &MF) const; + /// ParseSubtargetFeatures - Parses features string setting specified /// subtarget options. Definition of function is auto generated by tblgen. void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp index e4719b26cab52..7c99eb002876f 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp @@ -1367,6 +1367,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, } else if (Info.CFIType) { MIB->setCFIType(MF, Info.CFIType->getZExtValue()); } + MIB->setDeactivationSymbol(MF, Info.DeactivationSymbol); MIB.add(Info.Callee); diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp index 67a08e39fe879..6164f6e1e7d12 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -6735,7 +6735,7 @@ bool AArch64InstructionSelector::selectIntrinsic(MachineInstr &I, MIB.buildCopy({AArch64::X16}, {ValReg}); MIB.buildInstr(TargetOpcode::IMPLICIT_DEF, {AArch64::X17}, {}); - MIB.buildInstr(AArch64::AUT) + MIB.buildInstr(AArch64::AUTx16x17) .addImm(AUTKey) .addImm(AUTConstDiscC) .addUse(AUTAddrDisc) diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index a509edf160d32..e9dc50c6907d3 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -40,6 +40,8 @@ class AArch64ELFObjectWriter : public MCELFObjectTargetWriter { const MCFixup &Fixup, bool IsPCRel) const override; bool needsRelocateWithSymbol(const MCValue &Val, const MCSymbol &Sym, unsigned Type) const override; + void sortRelocs(const MCAssembler &Asm, + std::vector &Relocs) override; bool IsILP32; }; @@ -558,6 +560,17 @@ bool AArch64ELFObjectWriter::needsRelocateWithSymbol(const MCValue &Val, Val.getAccessVariant()); } +void AArch64ELFObjectWriter::sortRelocs( + const MCAssembler &Asm, std::vector &Relocs) { + // INST32 relocations should be applied last because they may overwrite the + // whole instruction and so should take precedence over other relocations that + // modify operands of the original instruction. + std::stable_partition(Relocs.begin(), Relocs.end(), + [](const ELFRelocationEntry &R) { + return R.Type != ELF::R_AARCH64_INST32; + }); +} + std::unique_ptr llvm::createAArch64ELFObjectWriter(uint8_t OSABI, bool IsILP32) { return std::make_unique(OSABI, IsILP32); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 12dd4cec85f59..58b98d8d93464 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -2946,9 +2946,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (NeedSign && isa(II->getArgOperand(4))) { auto *SignKey = cast(II->getArgOperand(3)); auto *SignDisc = cast(II->getArgOperand(4)); - auto *SignAddrDisc = ConstantPointerNull::get(Builder.getPtrTy()); + auto *Null = ConstantPointerNull::get(Builder.getPtrTy()); auto *NewCPA = ConstantPtrAuth::get(CPA->getPointer(), SignKey, - SignDisc, SignAddrDisc); + SignDisc, Null, Null); replaceInstUsesWith( *II, ConstantExpr::getPointerCast(NewCPA, II->getType())); return eraseInstFromFunction(*II); diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 80308bf92dbbc..4041c40de9cec 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -697,8 +697,7 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) { Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LoadInst *FirstLI = cast(PN.getIncomingValue(0)); - // Can't forward swifterror through a phi. - if (FirstLI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(FirstLI->getOperand(0))) return nullptr; // FIXME: This is overconservative; this transform is allowed in some cases @@ -737,8 +736,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LI->getPointerAddressSpace() != LoadAddrSpace) return nullptr; - // Can't forward swifterror through a phi. - if (LI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(LI->getOperand(0))) return nullptr; // We can't sink the load if the loaded value could be modified between diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 86be20c799a68..ca428f4f5b851 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -534,9 +534,9 @@ class Slice { public: Slice() = default; - Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) + Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable, Value *ProtectedField) : BeginOffset(BeginOffset), EndOffset(EndOffset), - UseAndIsSplittable(U, IsSplittable) {} + UseAndIsSplittable(U, IsSplittable), ProtectedField(ProtectedField) {} uint64_t beginOffset() const { return BeginOffset; } uint64_t endOffset() const { return EndOffset; } @@ -549,6 +549,8 @@ class Slice { bool isDead() const { return getUse() == nullptr; } void kill() { UseAndIsSplittable.setPointer(nullptr); } + Value *ProtectedField; + /// Support for ordering ranges. /// /// This provides an ordering over ranges such that start offsets are @@ -1075,7 +1077,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { EndOffset = AllocSize; } - AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); + AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable, ProtectedField)); } void visitBitCastInst(BitCastInst &BC) { @@ -4595,7 +4597,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PLoad << "\n"); @@ -4751,10 +4753,12 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { LLVMContext::MD_access_group}); // Now build a new slice for the alloca. + // ProtectedField==nullptr is a lie, but it doesn't matter because we + // already determined that all accesses are consistent. NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PStore->getOperandUse(PStore->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PStore << "\n"); @@ -5602,6 +5606,32 @@ SROA::runOnAlloca(AllocaInst &AI) { return {Changed, CFGChanged}; } + for (auto &P : AS.partitions()) { + std::optional ProtectedField; + // For now, we can't split if a field is accessed both via protected + // field and not. + for (Slice &S : P) { + if (auto *II = dyn_cast(S.getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S.ProtectedField; + if (*ProtectedField != S.ProtectedField) + return {Changed, CFGChanged}; + } + for (Slice *S : P.splitSliceTails()) { + if (auto *II = dyn_cast(S->getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S->ProtectedField; + if (*ProtectedField != S->ProtectedField) + return {Changed, CFGChanged}; + } + } + // Delete all the dead users of this alloca before splitting and rewriting it. for (Instruction *DeadUser : AS.getDeadUsers()) { // Free up everything used by this instruction. diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 02f1d08759129..80562255acae7 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -2148,6 +2148,22 @@ static bool replacingOperandWithVariableIsCheap(const Instruction *I, return !isa(I); } +bool llvm::shouldFoldLoadStoreWithPointerOperandThroughPhi(Value *Ptr) { + // swifterror pointers can only be used by a load or store; sinking a load + // or store would require introducing a select for the pointer operand, + // which isn't allowed for swifterror pointers. + if (Ptr->isSwiftError()) + return false; + + // Protected pointer field loads/stores should be paired with the intrinsic + // to avoid unnecessary address escapes. + if (auto *II = dyn_cast(Ptr)) + if (II->getIntrinsicID() == Intrinsic::protected_field_ptr) + return false; + + return true; +} + // All instructions in Insts belong to different blocks that all unconditionally // branch to a common successor. Analyze each instruction and return true if it // would be possible to sink them into their successor, creating one common @@ -2190,12 +2206,11 @@ static bool canSinkInstructions( if (!I->isSameOperationAs(I0, Instruction::CompareUsingIntersectedAttrs)) return false; - // swifterror pointers can only be used by a load or store; sinking a load - // or store would require introducing a select for the pointer operand, - // which isn't allowed for swifterror pointers. - if (isa(I) && I->getOperand(1)->isSwiftError()) + if (isa(I) && + !shouldFoldLoadStoreWithPointerOperandThroughPhi(I->getOperand(1))) return false; - if (isa(I) && I->getOperand(0)->isSwiftError()) + if (isa(I) && + !shouldFoldLoadStoreWithPointerOperandThroughPhi(I->getOperand(0))) return false; // Treat MMRAs conservatively. This pass can be quite aggressive and diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp index 5e50536a99206..320bef6c8f240 100644 --- a/llvm/lib/Transforms/Utils/ValueMapper.cpp +++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp @@ -526,8 +526,9 @@ Value *Mapper::mapValue(const Value *V) { if (isa(C)) return getVM()[V] = ConstantVector::get(Ops); if (isa(C)) - return getVM()[V] = ConstantPtrAuth::get(Ops[0], cast(Ops[1]), - cast(Ops[2]), Ops[3]); + return getVM()[V] = + ConstantPtrAuth::get(Ops[0], cast(Ops[1]), + cast(Ops[2]), Ops[3], Ops[4]); // If this is a no-operand constant, it must be because the type was remapped. if (isa(C)) return getVM()[V] = PoisonValue::get(NewTy); diff --git a/llvm/test/CodeGen/AArch64/ptrauth-call.ll b/llvm/test/CodeGen/AArch64/ptrauth-call.ll index bf35cf8fecbdb..7eb4cfca40f09 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-call.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-call.ll @@ -169,13 +169,23 @@ define i32 @test_tailcall_ib_var(ptr %arg0, ptr %arg1) #0 { define void @test_tailcall_omit_mov_x16_x16(ptr %objptr) #0 { ; CHECK-LABEL: test_tailcall_omit_mov_x16_x16: -; CHECK-NEXT: ldr x16, [x0] -; CHECK-NEXT: mov x17, x0 -; CHECK-NEXT: movk x17, #6503, lsl #48 -; CHECK-NEXT: autda x16, x17 -; CHECK-NEXT: ldr x1, [x16] -; CHECK-NEXT: movk x16, #54167, lsl #48 -; CHECK-NEXT: braa x1, x16 +; DARWIN-NEXT: ldr x16, [x0] +; DARWIN-NEXT: mov x17, x0 +; DARWIN-NEXT: movk x17, #6503, lsl #48 +; DARWIN-NEXT: autda x16, x17 +; DARWIN-NEXT: ldr x1, [x16] +; DARWIN-NEXT: movk x16, #54167, lsl #48 +; DARWIN-NEXT: braa x1, x16 +; ELF-NEXT: ldr x1, [x0] +; ELF-NEXT: mov x8, x0 +; ELF-NEXT: movk x8, #6503, lsl #48 +; ELF-NEXT: autda x1, x8 +; ELF-NEXT: ldr x2, [x1] +; FIXME: Get rid of the x16/x17 constraint on non-Darwin so we can eliminate +; this mov. +; ELF-NEXT: mov x16, x1 +; ELF-NEXT: movk x16, #54167, lsl #48 +; ELF-NEXT: braa x2, x16 %vtable.signed = load ptr, ptr %objptr, align 8 %objptr.int = ptrtoint ptr %objptr to i64 %vtable.discr = tail call i64 @llvm.ptrauth.blend(i64 %objptr.int, i64 6503) @@ -191,16 +201,27 @@ define void @test_tailcall_omit_mov_x16_x16(ptr %objptr) #0 { define i32 @test_call_omit_extra_moves(ptr %objptr) #0 { ; CHECK-LABEL: test_call_omit_extra_moves: ; DARWIN-NEXT: stp x29, x30, [sp, #-16]! -; ELF-NEXT: str x30, [sp, #-16]! -; CHECK-NEXT: ldr x16, [x0] -; CHECK-NEXT: mov x17, x0 -; CHECK-NEXT: movk x17, #6503, lsl #48 -; CHECK-NEXT: autda x16, x17 -; CHECK-NEXT: ldr x8, [x16] -; CHECK-NEXT: movk x16, #34646, lsl #48 -; CHECK-NEXT: blraa x8, x16 -; CHECK-NEXT: mov w0, #42 +; DARWIN-NEXT: ldr x16, [x0] +; DARWIN-NEXT: mov x17, x0 +; DARWIN-NEXT: movk x17, #6503, lsl #48 +; DARWIN-NEXT: autda x16, x17 +; DARWIN-NEXT: ldr x8, [x16] +; DARWIN-NEXT: movk x16, #34646, lsl #48 +; DARWIN-NEXT: blraa x8, x16 +; DARWIN-NEXT: mov w0, #42 ; DARWIN-NEXT: ldp x29, x30, [sp], #16 +; ELF-NEXT: str x30, [sp, #-16]! +; ELF-NEXT: ldr x8, [x0] +; ELF-NEXT: mov x9, x0 +; ELF-NEXT: movk x9, #6503, lsl #48 +; ELF-NEXT: autda x8, x9 +; ELF-NEXT: ldr x9, [x8] +; FIXME: Get rid of the x16/x17 constraint on non-Darwin so we can eliminate +; this mov. +; ELF-NEXT: mov x17, x8 +; ELF-NEXT: movk x17, #34646, lsl #48 +; ELF-NEXT: blraa x9, x17 +; ELF-NEXT: mov w0, #42 ; ELF-NEXT: ldr x30, [sp], #16 ; CHECK-NEXT: ret %vtable.signed = load ptr, ptr %objptr diff --git a/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll b/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll index d5340dcebad57..41d3ffe32c31e 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-fpac.ll @@ -1,17 +1,18 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,NOFPAC -; RUN: llc < %s -mtriple arm64e-apple-darwin -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,FPAC -; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,NOFPAC -; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,FPAC +; RUN: llc < %s -mtriple arm64e-apple-darwin -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,DARWIN,NOFPAC +; RUN: llc < %s -mtriple arm64e-apple-darwin -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL="L" --check-prefixes=ALL,DARWIN,FPAC,DARWIN-FPAC +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,ELF,NOFPAC +; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -mattr=+fpac -verify-machineinstrs | FileCheck %s -DL=".L" --check-prefixes=ALL,ELF,FPAC,ELF-FPAC target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_ia(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_ia: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autia x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autia x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autia x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 %arg1) ret i64 %tmp @@ -20,9 +21,10 @@ define i64 @test_auth_ia(i64 %arg, i64 %arg1) { define i64 @test_auth_ia_zero(i64 %arg) { ; ALL-LABEL: test_auth_ia_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autiza x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autiza x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autiza x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 0) ret i64 %tmp @@ -31,9 +33,10 @@ define i64 @test_auth_ia_zero(i64 %arg) { define i64 @test_auth_ib(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_ib: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autib x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autib x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autib x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 1, i64 %arg1) ret i64 %tmp @@ -42,9 +45,10 @@ define i64 @test_auth_ib(i64 %arg, i64 %arg1) { define i64 @test_auth_ib_zero(i64 %arg) { ; ALL-LABEL: test_auth_ib_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autizb x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autizb x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autizb x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 1, i64 0) ret i64 %tmp @@ -53,9 +57,10 @@ define i64 @test_auth_ib_zero(i64 %arg) { define i64 @test_auth_da(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_da: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autda x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autda x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autda x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 2, i64 %arg1) ret i64 %tmp @@ -64,9 +69,10 @@ define i64 @test_auth_da(i64 %arg, i64 %arg1) { define i64 @test_auth_da_zero(i64 %arg) { ; ALL-LABEL: test_auth_da_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdza x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdza x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdza x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 2, i64 0) ret i64 %tmp @@ -75,9 +81,10 @@ define i64 @test_auth_da_zero(i64 %arg) { define i64 @test_auth_db(i64 %arg, i64 %arg1) { ; ALL-LABEL: test_auth_db: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdb x16, x1 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdb x16, x1 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdb x0, x1 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 3, i64 %arg1) ret i64 %tmp @@ -86,9 +93,10 @@ define i64 @test_auth_db(i64 %arg, i64 %arg1) { define i64 @test_auth_db_zero(i64 %arg) { ; ALL-LABEL: test_auth_db_zero: ; ALL: %bb.0: -; ALL-NEXT: mov x16, x0 -; ALL-NEXT: autdzb x16 -; ALL-NEXT: mov x0, x16 +; DARWIN-NEXT: mov x16, x0 +; DARWIN-NEXT: autdzb x16 +; DARWIN-NEXT: mov x0, x16 +; ELF-NEXT: autdzb x0 ; ALL-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 3, i64 0) ret i64 %tmp @@ -362,12 +370,17 @@ define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { ; NOFPAC-NEXT: mov x0, x16 ; NOFPAC-NEXT: ret ; -; FPAC-LABEL: test_auth_trap_attribute: -; FPAC: %bb.0: -; FPAC-NEXT: mov x16, x0 -; FPAC-NEXT: autia x16, x1 -; FPAC-NEXT: mov x0, x16 -; FPAC-NEXT: ret +; DARWIN-FPAC-LABEL: test_auth_trap_attribute: +; DARWIN-FPAC: %bb.0: +; DARWIN-FPAC-NEXT: mov x16, x0 +; DARWIN-FPAC-NEXT: autia x16, x1 +; DARWIN-FPAC-NEXT: mov x0, x16 +; DARWIN-FPAC-NEXT: ret +; +; ELF-FPAC-LABEL: test_auth_trap_attribute: +; ELF-FPAC: %bb.0: +; ELF-FPAC-NEXT: autia x0, x1 +; ELF-FPAC-NEXT: ret %tmp = call i64 @llvm.ptrauth.auth(i64 %arg, i32 0, i64 %arg1) ret i64 %tmp } diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll index 74d2370c74c54..ab8ce04b4816d 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign-with-blend.ll @@ -33,21 +33,27 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_blend(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_blend: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: mov x17, x1 -; UNCHECKED-NEXT: movk x17, #65535, lsl #48 -; UNCHECKED-NEXT: autda x16, x17 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: mov x17, x1 +; UNCHECKED-DARWIN-NEXT: movk x17, #65535, lsl #48 +; UNCHECKED-DARWIN-NEXT: autda x16, x17 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: mov x8, x1 +; UNCHECKED-ELF-NEXT: movk x8, #65535, lsl #48 +; UNCHECKED-ELF-NEXT: autda x0, x8 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_blend: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: mov x17, x1 -; CHECKED-NEXT: movk x17, #65535, lsl #48 -; CHECKED-NEXT: autda x16, x17 -; CHECKED-NEXT: mov x0, x16 +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: mov x17, x1 +; CHECKED-DARWIN-NEXT: movk x17, #65535, lsl #48 +; CHECKED-DARWIN-NEXT: autda x16, x17 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: mov x8, x1 +; CHECKED-ELF-NEXT: movk x8, #65535, lsl #48 +; CHECKED-ELF-NEXT: autda x0, x8 ; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_blend: @@ -146,10 +152,10 @@ define i64 @test_resign_blend_and_const(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov x17, x16 ; CHECKED-NEXT: xpacd x17 ; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_1 +; CHECKED-NEXT: b.eq [[L]]auth_success_[[N2:[0-9]]] ; CHECKED-NEXT: mov x16, x17 ; CHECKED-NEXT: b [[L]]resign_end_1 -; CHECKED-NEXT: Lauth_success_1: +; CHECKED-NEXT: Lauth_success_[[N2]]: ; CHECKED-NEXT: mov x17, #56789 ; CHECKED-NEXT: pacdb x16, x17 ; CHECKED-NEXT: Lresign_end_1: @@ -232,10 +238,10 @@ define i64 @test_auth_too_large_discriminator(i64 %arg, i64 %arg1) { ; UNCHECKED-NEXT: mov w8, #65536 ; UNCHECKED-DARWIN-NEXT: bfi x1, x8, #48, #16 ; UNCHECKED-DARWIN-NEXT: mov x16, x0 -; UNCHECKED-ELF-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autda x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 ; UNCHECKED-ELF-NEXT: bfi x1, x8, #48, #16 -; UNCHECKED-NEXT: autda x16, x1 -; UNCHECKED-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autda x0, x1 ; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_too_large_discriminator: @@ -243,10 +249,10 @@ define i64 @test_auth_too_large_discriminator(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov w8, #65536 ; CHECKED-DARWIN-NEXT: bfi x1, x8, #48, #16 ; CHECKED-DARWIN-NEXT: mov x16, x0 -; CHECKED-ELF-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autda x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 ; CHECKED-ELF-NEXT: bfi x1, x8, #48, #16 -; CHECKED-NEXT: autda x16, x1 -; CHECKED-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autda x0, x1 ; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_too_large_discriminator: diff --git a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll index fdd5ae29f35ea..a034c586073b3 100644 --- a/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll +++ b/llvm/test/CodeGen/AArch64/ptrauth-intrinsic-auth-resign.ll @@ -1,13 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefixes=UNCHECKED,UNCHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL="L" --check-prefixes=UNCHECKED,UNCHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL="L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL="L" --check-prefixes=CHECKED,CHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL="L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL="L" --check-prefixes=CHECKED,CHECKED-DARWIN ; RUN: llc < %s -mtriple arm64e-apple-darwin -global-isel=0 -verify-machineinstrs \ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL="L" --check-prefix=TRAP @@ -15,14 +15,14 @@ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL="L" --check-prefix=TRAP ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefixes=UNCHECKED,UNCHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefix=UNCHECKED +; RUN: -aarch64-ptrauth-auth-checks=none | FileCheck %s -DL=".L" --check-prefixes=UNCHECKED,UNCHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL=".L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL=".L" --check-prefixes=CHECKED,CHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel -global-isel-abort=1 -verify-machineinstrs \ -; RUN: | FileCheck %s -DL=".L" --check-prefix=CHECKED +; RUN: | FileCheck %s -DL=".L" --check-prefixes=CHECKED,CHECKED-ELF ; RUN: llc < %s -mtriple aarch64-linux-gnu -mattr=+pauth -global-isel=0 -verify-machineinstrs \ ; RUN: -aarch64-ptrauth-auth-checks=trap | FileCheck %s -DL=".L" --check-prefix=TRAP @@ -33,18 +33,20 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define i64 @test_auth_ia(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_ia: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autia x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autia x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autia x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autia x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autia x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autia x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia: ; TRAP: %bb.0: @@ -64,18 +66,20 @@ define i64 @test_auth_ia(i64 %arg, i64 %arg1) { define i64 @test_auth_ia_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ia_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autiza x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autiza x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autiza x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autiza x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autiza x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autiza x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia_zero: ; TRAP: %bb.0: @@ -95,18 +99,20 @@ define i64 @test_auth_ia_zero(i64 %arg) { define i64 @test_auth_ib(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_ib: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autib x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autib x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autib x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ib: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autib x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autib x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autib x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ib: ; TRAP: %bb.0: @@ -126,18 +132,20 @@ define i64 @test_auth_ib(i64 %arg, i64 %arg1) { define i64 @test_auth_ib_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ib_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autizb x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autizb x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autizb x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ib_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autizb x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autizb x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autizb x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ib_zero: ; TRAP: %bb.0: @@ -157,18 +165,20 @@ define i64 @test_auth_ib_zero(i64 %arg) { define i64 @test_auth_da(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_da: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autda x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autda x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autda x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_da: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autda x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autda x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autda x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_da: ; TRAP: %bb.0: @@ -188,18 +198,20 @@ define i64 @test_auth_da(i64 %arg, i64 %arg1) { define i64 @test_auth_da_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_da_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdza x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdza x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdza x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_da_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdza x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdza x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdza x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_da_zero: ; TRAP: %bb.0: @@ -219,18 +231,20 @@ define i64 @test_auth_da_zero(i64 %arg) { define i64 @test_auth_db(i64 %arg, i64 %arg1) { ; UNCHECKED-LABEL: test_auth_db: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdb x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdb x16, x1 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdb x0, x1 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_db: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdb x16, x1 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdb x16, x1 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdb x0, x1 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_db: ; TRAP: %bb.0: @@ -250,18 +264,20 @@ define i64 @test_auth_db(i64 %arg, i64 %arg1) { define i64 @test_auth_db_zero(i64 %arg) { ; UNCHECKED-LABEL: test_auth_db_zero: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autdzb x16 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: autdzb x16 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: autdzb x0 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_db_zero: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autdzb x16 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: autdzb x16 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: autdzb x0 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_db_zero: ; TRAP: %bb.0: @@ -535,24 +551,24 @@ define i64 @test_resign_da_dzb(i64 %arg, i64 %arg1, i64 %arg2) { define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { ; UNCHECKED-LABEL: test_auth_trap_attribute: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: autia x16, x1 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-NEXT: mov x16, x0 +; UNCHECKED-NEXT: autia x16, x1 +; UNCHECKED-NEXT: mov x0, x16 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_trap_attribute: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: autia x16, x1 -; CHECKED-NEXT: mov x17, x16 -; CHECKED-NEXT: xpaci x17 -; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_6 -; CHECKED-NEXT: brk #0xc470 -; CHECKED-NEXT: Lauth_success_6: -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-NEXT: mov x16, x0 +; CHECKED-NEXT: autia x16, x1 +; CHECKED-NEXT: mov x17, x16 +; CHECKED-NEXT: xpaci x17 +; CHECKED-NEXT: cmp x16, x17 +; CHECKED-NEXT: b.eq [[L]]auth_success_6 +; CHECKED-NEXT: brk #0xc470 +; CHECKED-NEXT: Lauth_success_6: +; CHECKED-NEXT: mov x0, x16 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_trap_attribute: ; TRAP: %bb.0: @@ -572,20 +588,24 @@ define i64 @test_auth_trap_attribute(i64 %arg, i64 %arg1) "ptrauth-auth-traps" { define i64 @test_auth_ia_constdisc(i64 %arg) { ; UNCHECKED-LABEL: test_auth_ia_constdisc: -; UNCHECKED: %bb.0: -; UNCHECKED-NEXT: mov x16, x0 -; UNCHECKED-NEXT: mov x17, #256 -; UNCHECKED-NEXT: autia x16, x17 -; UNCHECKED-NEXT: mov x0, x16 -; UNCHECKED-NEXT: ret +; UNCHECKED: %bb.0: +; UNCHECKED-DARWIN-NEXT: mov x16, x0 +; UNCHECKED-DARWIN-NEXT: mov x17, #256 +; UNCHECKED-DARWIN-NEXT: autia x16, x17 +; UNCHECKED-DARWIN-NEXT: mov x0, x16 +; UNCHECKED-ELF-NEXT: mov w8, #256 +; UNCHECKED-ELF-NEXT: autia x0, x8 +; UNCHECKED-NEXT: ret ; ; CHECKED-LABEL: test_auth_ia_constdisc: -; CHECKED: %bb.0: -; CHECKED-NEXT: mov x16, x0 -; CHECKED-NEXT: mov x17, #256 -; CHECKED-NEXT: autia x16, x17 -; CHECKED-NEXT: mov x0, x16 -; CHECKED-NEXT: ret +; CHECKED: %bb.0: +; CHECKED-DARWIN-NEXT: mov x16, x0 +; CHECKED-DARWIN-NEXT: mov x17, #256 +; CHECKED-DARWIN-NEXT: autia x16, x17 +; CHECKED-DARWIN-NEXT: mov x0, x16 +; CHECKED-ELF-NEXT: mov w8, #256 +; CHECKED-ELF-NEXT: autia x0, x8 +; CHECKED-NEXT: ret ; ; TRAP-LABEL: test_auth_ia_constdisc: ; TRAP: %bb.0: @@ -621,10 +641,10 @@ define i64 @test_resign_da_constdisc(i64 %arg, i64 %arg1) { ; CHECKED-NEXT: mov x17, x16 ; CHECKED-NEXT: xpacd x17 ; CHECKED-NEXT: cmp x16, x17 -; CHECKED-NEXT: b.eq [[L]]auth_success_7 +; CHECKED-NEXT: b.eq [[L]]auth_success_[[N1:[0-9]]] ; CHECKED-NEXT: mov x16, x17 ; CHECKED-NEXT: b [[L]]resign_end_6 -; CHECKED-NEXT: Lauth_success_7: +; CHECKED-NEXT: Lauth_success_[[N1]]: ; CHECKED-NEXT: mov x17, #256 ; CHECKED-NEXT: pacda x16, x17 ; CHECKED-NEXT: Lresign_end_6: diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp index 088264e0429fd..bb1646bc34545 100644 --- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp +++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp @@ -1169,7 +1169,7 @@ define ptr @foo() { // Check get(), getKey(), getDiscriminator(), getAddrDiscriminator(). auto *NewPtrAuth = sandboxir::ConstantPtrAuth::get( &F, PtrAuth->getKey(), PtrAuth->getDiscriminator(), - PtrAuth->getAddrDiscriminator()); + PtrAuth->getAddrDiscriminator(), PtrAuth->getDeactivationSymbol()); EXPECT_EQ(NewPtrAuth, PtrAuth); // Check hasAddressDiscriminator(). EXPECT_EQ(PtrAuth->hasAddressDiscriminator(), diff --git a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp index 86ad41fa7ad50..90b58236060d2 100644 --- a/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp +++ b/llvm/unittests/Transforms/Utils/ValueMapperTest.cpp @@ -450,6 +450,10 @@ TEST(ValueMapperTest, mapValuePtrAuth) { PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage0"); std::unique_ptr Storage1 = std::make_unique( PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "Storage1"); + std::unique_ptr DS0 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS0"); + std::unique_ptr DS1 = std::make_unique( + PtrTy, false, GlobalValue::ExternalLinkage, nullptr, "DS1"); ConstantInt *ConstKey = ConstantInt::get(Int32Ty, 1); ConstantInt *ConstDisc = ConstantInt::get(Int64Ty, 1234); @@ -457,11 +461,12 @@ TEST(ValueMapperTest, mapValuePtrAuth) { ValueToValueMapTy VM; VM[Var0.get()] = Var1.get(); VM[Storage0.get()] = Storage1.get(); + VM[DS0.get()] = DS1.get(); - ConstantPtrAuth *Value = - ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, Storage0.get()); - ConstantPtrAuth *MappedValue = - ConstantPtrAuth::get(Var1.get(), ConstKey, ConstDisc, Storage1.get()); + ConstantPtrAuth *Value = ConstantPtrAuth::get(Var0.get(), ConstKey, ConstDisc, + Storage0.get(), DS0.get()); + ConstantPtrAuth *MappedValue = ConstantPtrAuth::get( + Var1.get(), ConstKey, ConstDisc, Storage1.get(), DS1.get()); EXPECT_EQ(ValueMapper(VM).mapValue(*Value), MappedValue); } diff --git a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp index 57997a6b0e4e0..9fc4022e55123 100644 --- a/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp +++ b/llvm/utils/TableGen/DAGISelMatcherEmitter.cpp @@ -954,6 +954,13 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, } } const EmitNodeMatcherCommon *EN = cast(N); + bool SupportsDeactivationSymbol = + EN->getInstruction().TheDef->getValueAsBit( + "supportsDeactivationSymbol"); + if (SupportsDeactivationSymbol) { + OS << "OPC_CaptureDeactivationSymbol,\n"; + OS.indent(FullIndexWidth + Indent); + } bool IsEmitNode = isa(EN); OS << (IsEmitNode ? "OPC_EmitNode" : "OPC_MorphNodeTo"); bool CompressVTs = EN->getNumVTs() < 3; @@ -1045,8 +1052,8 @@ unsigned MatcherTableEmitter::EmitMatcher(const Matcher *N, } else OS << '\n'; - return 4 + !CompressVTs + !CompressNodeInfo + NumTypeBytes + - NumOperandBytes + NumCoveredBytes; + return 4 + SupportsDeactivationSymbol + !CompressVTs + !CompressNodeInfo + + NumTypeBytes + NumOperandBytes + NumCoveredBytes; } case Matcher::CompleteMatch: { const CompleteMatchMatcher *CM = cast(N); diff --git a/llvm/utils/gn/build/toolchain/target_flags.gni b/llvm/utils/gn/build/toolchain/target_flags.gni index cdfab75ed8bcd..82323e66b3ae5 100644 --- a/llvm/utils/gn/build/toolchain/target_flags.gni +++ b/llvm/utils/gn/build/toolchain/target_flags.gni @@ -2,6 +2,12 @@ import("//llvm/triples.gni") import("//llvm/utils/gn/build/mac_sdk.gni") import("//llvm/utils/gn/build/toolchain/compiler.gni") +declare_args() { + stage2_pointer_field_protection = "none" + emupac_force = false + emupac_force_non_pac = false +} + # Flags in this file are passed both to the compiler that's building # compiler-rt at build time (via normal gn cflags/ldflags), as well as to the # compiler building compiler-rt test programs at test time (via @@ -63,3 +69,20 @@ if (current_cpu == "x86") { target_flags += [ "-m32" ] } } + +if (current_toolchain != host_toolchain && current_cpu == "arm64") { + target_flags += [ + "-march=armv8.3a", + ] +} +if (current_toolchain != host_toolchain) { + target_flags += [ + "-fexperimental-pointer-field-protection=$stage2_pointer_field_protection", + ] + target_ldflags += [ + "--rtlib=compiler-rt", + "--unwindlib=libunwind", + "-static-libgcc", + "-L" + rebase_path(root_build_dir) + "/lib", + ] +} diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn index d1048259bcd44..d1a97568a2f03 100644 --- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn +++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn @@ -1,5 +1,6 @@ import("//compiler-rt/target.gni") import("//llvm/utils/gn/build/buildflags.gni") +import("//llvm/utils/gn/build/toolchain/target_flags.gni") declare_args() { # Skip the atomic builtin (should normally be provided by a shared library). @@ -87,6 +88,7 @@ static_library("builtins") { cflags += [ "-fomit-frame-pointer" ] } cflags_c = [ "-std=c11" ] + cflags_cc = [ "-nostdinc++" ] } sources = [ @@ -500,6 +502,7 @@ static_library("builtins") { if (current_cpu == "arm64") { sources -= [ "fp_mode.c" ] sources += [ + "aarch64/emupac.cpp", "aarch64/fp_mode.c", "cpu_model/aarch64.c", ] @@ -508,6 +511,9 @@ static_library("builtins") { } } + if (emupac_force_non_pac) { + defines = [ "FORCE_NON_PAC" ] + } if (current_cpu == "avr") { sources += [ "avr/divmodhi4.S", @@ -601,6 +607,7 @@ static_library("builtins") { } deps = lse_targets + include_dirs = [ "//third-party/siphash/include" ] } # Currently unused but necessary to make sync_source_lists_from_cmake.py happy. diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn index b99676b52aea7..4d48c5ff41da0 100644 --- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn @@ -1,3 +1,5 @@ +import("//llvm/utils/gn/build/toolchain/target_flags.gni") + static_library("CodeGen") { output_name = "LLVMCodeGen" public_deps = [ @@ -254,4 +256,7 @@ static_library("CodeGen") { "WindowScheduler.cpp", "XRayInstrumentation.cpp", ] + if (emupac_force) { + defines = [ "FORCE_EMUPAC" ] + } } diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn index fe7ff6ef68f99..5461ed5246b85 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn @@ -31,6 +31,7 @@ static_library("Support") { include_dirs = [ "Unix", "Windows", + "//third-party/siphash/include", ] sources = [ "AArch64AttributeParser.cpp", diff --git a/llvm/utils/pfp-bazel b/llvm/utils/pfp-bazel new file mode 100755 index 0000000000000..ee9caff17ae8e --- /dev/null +++ b/llvm/utils/pfp-bazel @@ -0,0 +1,17 @@ +#!/bin/sh -xe + +shopt -s globstar + +tc="$(dirname $0)/../../ra" + +mtime_before="$(stat -c %Y $tc/bin/clang || true)" +ninja -C $tc clang +mtime_after="$(stat -c %Y $tc/bin/clang)" +if [ "$mtime_before" != "$mtime_after" ] ; then + rm -f $tc/stage2_*/**/*.o +fi +ninja -C $tc libcxx + +cflags=$(grep COMPILER_RT_TEST_COMPILER_CFLAGS= $tc/stage2_unix/toolchain.ninja | sed -e 's/\\\$ /:/g' | grep -o COMPILER_RT_TEST_COMPILER_CFLAGS='[^ ]*' | cut -d= -f2-) +cflags="$cflags:-Wno-unused-command-line-argument" +BAZEL_CXXOPTS=$cflags:-stdlib=libc++ BAZEL_LINKLIBS=$cflags:-Wl,-Bstatic:-lc++:-lc++abi:-Wl,-Bdynamic:-lm CC=clang PATH=$tc/bin:$PATH bazel "$@" diff --git a/llvm/utils/pfp-check-libcxx b/llvm/utils/pfp-check-libcxx new file mode 100755 index 0000000000000..dd9b944d71192 --- /dev/null +++ b/llvm/utils/pfp-check-libcxx @@ -0,0 +1,90 @@ +#!/bin/sh -xe + +shopt -s globstar + +tc="$(cd $(dirname $0)/../../ra && pwd)" + +mtime_before="$(stat -c %Y $tc/bin/clang || true)" +ninja -C $tc clang +mtime_after="$(stat -c %Y $tc/bin/clang)" +if [ "$mtime_before" != "$mtime_after" ] ; then + rm -f $tc/stage2_*/**/*.o +fi +ninja -C $tc libcxx + +cflags=$(grep COMPILER_RT_TEST_COMPILER_CFLAGS= $tc/stage2_unix/toolchain.ninja | sed -e 's/\\\$ /:/g' | grep -o COMPILER_RT_TEST_COMPILER_CFLAGS='[^ ]*' | sed -e 's/:/ /g' | cut -d= -f2-) +cflags="$cflags -Wno-unused-command-line-argument" + +mkdir -p $tc/check-libcxx +cat > $tc/check-libcxx/lit.site.cfg < $tc/check-libcxx/cmake-bridge.cfg < +Date: Mon, 3 Mar 2025 20:07:20 -0800 +Subject: [PATCH] PFP fixes + +--- + runtime/platform/allocation.h | 4 ++++ + runtime/vm/dart.cc | 5 ++--- + runtime/vm/isolate.cc | 5 ++++- + runtime/vm/isolate.h | 4 ++-- + runtime/vm/timeline.cc | 4 ++++ + 5 files changed, 16 insertions(+), 6 deletions(-) + +diff --git a/runtime/platform/allocation.h b/runtime/platform/allocation.h +index 26001ac54f2..5bc2a10960b 100644 +--- a/runtime/platform/allocation.h ++++ b/runtime/platform/allocation.h +@@ -40,6 +40,10 @@ class MallocAllocated { + public: + MallocAllocated() {} + ++ // Placement new. ++ void* operator new(size_t size, void *p) { return p; } ++ void* operator new[](size_t size, void *p) { return p; } ++ + // Intercept operator new to produce clearer error messages when we run out + // of memory. Don't do this when running under ASAN so it can continue to + // check malloc/new/new[] are paired with free/delete/delete[] respectively. +diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc +index 490549316ef..d78779a511a 100644 +--- a/runtime/vm/dart.cc ++++ b/runtime/vm/dart.cc +@@ -165,12 +165,12 @@ class DartInitializationState : public AllStatic { + std::atomic DartInitializationState::state_ = {kUnInitialized}; + std::atomic DartInitializationState::in_use_count_ = {0}; + +-#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + static void CheckOffsets() { + #if !defined(IS_SIMARM_HOST64) + // These offsets are embedded in precompiled instructions. We need the + // compiler and the runtime to agree. + bool ok = true; ++ (void)ok; + #define CHECK_OFFSET(expr, offset) \ + if ((expr) != (offset)) { \ + OS::PrintErr("%s got %" Pd ", %s expected %" Pd "\n", #expr, \ +@@ -226,7 +226,6 @@ static void CheckOffsets() { + } \ + } + #define CHECK_CONSTANT(Class, Name) CHECK_OFFSET(Class::Name, Class##_##Name); +-#endif // defined(DART_PRECOMPILED_RUNTIME) + + COMMON_OFFSETS_LIST(CHECK_FIELD, CHECK_ARRAY, CHECK_SIZEOF, + CHECK_ARRAY_SIZEOF, CHECK_PAYLOAD_SIZEOF, CHECK_RANGE, +@@ -258,8 +257,8 @@ static void CheckOffsets() { + #endif // defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + + char* Dart::DartInit(const Dart_InitializeParams* params) { +-#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + CheckOffsets(); ++#if defined(DART_PRECOMPILER) || defined(DART_PRECOMPILED_RUNTIME) + #elif defined(ARCH_IS_64_BIT) != defined(TARGET_ARCH_IS_64_BIT) + return Utils::StrDup( + "JIT cannot simulate target architecture with different word size than " +diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc +index 11cb004f448..c48623b8907 100644 +--- a/runtime/vm/isolate.cc ++++ b/runtime/vm/isolate.cc +@@ -448,6 +448,8 @@ IsolateGroup::~IsolateGroup() { + delete debugger_; + debugger_ = nullptr; + #endif ++ ++ delete object_store_; + } + + void IsolateGroup::RegisterIsolate(Isolate* isolate) { +@@ -1112,7 +1114,8 @@ bool Isolate::SendInternalLibMessage(Dart_Port main_port, + } + + void IsolateGroup::set_object_store(ObjectStore* object_store) { +- object_store_.reset(object_store); ++ delete object_store_; ++ object_store_ = object_store; + } + + class IsolateMessageHandler : public MessageHandler { +diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h +index 9f5d27fb404..5b501b3fc4f 100644 +--- a/runtime/vm/isolate.h ++++ b/runtime/vm/isolate.h +@@ -506,7 +506,7 @@ class IsolateGroup : public IntrusiveDListEntry { + void DropOriginalClassTable(); + + StoreBuffer* store_buffer() const { return store_buffer_.get(); } +- ObjectStore* object_store() const { return object_store_.get(); } ++ ObjectStore* object_store() const { return object_store_; } + Mutex* symbols_mutex() { return &symbols_mutex_; } + Mutex* type_canonicalization_mutex() { return &type_canonicalization_mutex_; } + Mutex* type_arguments_canonicalization_mutex() { +@@ -819,7 +819,7 @@ class IsolateGroup : public IntrusiveDListEntry { + // Accessed from generated code. + ClassTable* class_table_; + AcqRelAtomic cached_class_table_table_; +- std::unique_ptr object_store_; ++ ObjectStore* object_store_; + // End accessed from generated code. + + ClassTableAllocator class_table_allocator_; +diff --git a/runtime/vm/timeline.cc b/runtime/vm/timeline.cc +index 211f0e28741..ba07afac27c 100644 +--- a/runtime/vm/timeline.cc ++++ b/runtime/vm/timeline.cc +@@ -1595,6 +1595,10 @@ TimelineEventFixedBufferRecorder::TimelineEventFixedBufferRecorder( + if (memory_ == nullptr) { + OUT_OF_MEMORY(); + } ++ for (intptr_t i = 0; i != num_blocks_; ++i) { ++ new (reinterpret_cast(memory_->address()) + ++ i * sizeof(TimelineEventBlock)) TimelineEventBlock(i); ++ } + blocks_ = reinterpret_cast(memory_->address()); + } + +-- +2.48.1 + diff --git a/third-party/siphash/include/siphash/SipHash.h b/third-party/siphash/include/siphash/SipHash.h new file mode 100644 index 0000000000000..9653e9428b123 --- /dev/null +++ b/third-party/siphash/include/siphash/SipHash.h @@ -0,0 +1,161 @@ +//===--- SipHash.h - An implementation of SipHash -------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This is a header-only implementation of SipHash. It lacks library +// dependencies so it can be used from LLVM and compiler-rt. +// +//===----------------------------------------------------------------------===// + +#include +#include + +// Lightly adapted from the SipHash reference C implementation: +// https://github.com/veorq/SipHash +// by Jean-Philippe Aumasson and Daniel J. Bernstein + +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) + +#define U32TO8_LE(p, v) \ + (p)[0] = (uint8_t)((v)); \ + (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); \ + (p)[3] = (uint8_t)((v) >> 24); + +#define U64TO8_LE(p, v) \ + U32TO8_LE((p), (uint32_t)((v))); \ + U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); + +#define U8TO64_LE(p) \ + (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) + +#define SIPROUND \ + do { \ + v0 += v1; \ + v1 = ROTL(v1, 13); \ + v1 ^= v0; \ + v0 = ROTL(v0, 32); \ + v2 += v3; \ + v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; \ + v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; \ + v1 = ROTL(v1, 17); \ + v1 ^= v2; \ + v2 = ROTL(v2, 32); \ + } while (0) + +namespace { + +/// Computes a SipHash value +/// +/// \param in: pointer to input data (read-only) +/// \param inlen: input data length in bytes (any size_t value) +/// \param k: reference to the key data 16-byte array (read-only) +/// \returns output data, must be 8 or 16 bytes +/// +template +void siphash(const unsigned char *in, uint64_t inlen, + const unsigned char (&k)[16], unsigned char (&out)[outlen]) { + + const unsigned char *ni = (const unsigned char *)in; + const unsigned char *kk = (const unsigned char *)k; + + static_assert(outlen == 8 || outlen == 16, "result should be 8 or 16 bytes"); + + uint64_t v0 = UINT64_C(0x736f6d6570736575); + uint64_t v1 = UINT64_C(0x646f72616e646f6d); + uint64_t v2 = UINT64_C(0x6c7967656e657261); + uint64_t v3 = UINT64_C(0x7465646279746573); + uint64_t k0 = U8TO64_LE(kk); + uint64_t k1 = U8TO64_LE(kk + 8); + uint64_t m; + int i; + const unsigned char *end = ni + inlen - (inlen % sizeof(uint64_t)); + const int left = inlen & 7; + uint64_t b = ((uint64_t)inlen) << 56; + v3 ^= k1; + v2 ^= k0; + v1 ^= k1; + v0 ^= k0; + + if (outlen == 16) + v1 ^= 0xee; + + for (; ni != end; ni += 8) { + m = U8TO64_LE(ni); + v3 ^= m; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= m; + } + + switch (left) { + case 7: + b |= ((uint64_t)ni[6]) << 48; + /* FALLTHRU */ + case 6: + b |= ((uint64_t)ni[5]) << 40; + /* FALLTHRU */ + case 5: + b |= ((uint64_t)ni[4]) << 32; + /* FALLTHRU */ + case 4: + b |= ((uint64_t)ni[3]) << 24; + /* FALLTHRU */ + case 3: + b |= ((uint64_t)ni[2]) << 16; + /* FALLTHRU */ + case 2: + b |= ((uint64_t)ni[1]) << 8; + /* FALLTHRU */ + case 1: + b |= ((uint64_t)ni[0]); + /* FALLTHRU */ + break; + case 0: + break; + } + + v3 ^= b; + + for (i = 0; i < cROUNDS; ++i) + SIPROUND; + + v0 ^= b; + + if (outlen == 16) + v2 ^= 0xee; + else + v2 ^= 0xff; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out, b); + + if (outlen == 8) + return; + + v1 ^= 0xdd; + + for (i = 0; i < dROUNDS; ++i) + SIPROUND; + + b = v0 ^ v1 ^ v2 ^ v3; + U64TO8_LE(out + 8, b); +} + +} // end anonymous namespace