diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h index 2b9cd035623cc..f8c4863685e64 100644 --- a/clang/include/clang/AST/ASTContext.h +++ b/clang/include/clang/AST/ASTContext.h @@ -183,6 +183,12 @@ struct TypeInfoChars { } }; +struct PFPField { + CharUnits structOffset; + CharUnits offset; + FieldDecl *field; +}; + /// Holds long-lived AST nodes (such as types and decls) that can be /// referred to throughout the semantic analysis of a file. class ASTContext : public RefCountedBase { @@ -3703,6 +3709,22 @@ OPT_LIST(V) StringRef getCUIDHash() const; + bool isPFPStruct(const RecordDecl *rec) const; + void findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, bool IncludeVBases) const; + bool hasPFPFields(QualType ty) const; + bool isPFPField(const FieldDecl *field) const; + + /// Returns whether this record's PFP fields (if any) are trivially + /// relocatable (i.e. may be memcpy'd). This may also return true if the + /// record does not have any PFP fields, so it may be necessary for the caller + /// to check for PFP fields, e.g. by calling hasPFPFields(). + bool arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const; + + llvm::SetVector PFPFieldsWithEvaluatedOffset; + void recordMemberDataPointerEvaluation(const ValueDecl *VD); + void recordOffsetOfEvaluation(const OffsetOfExpr *E); + private: /// All OMPTraitInfo objects live in this collection, one per /// `pragma omp [begin] declare variant` directive. diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 0912a004549ae..7d5b1d987257a 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -2597,6 +2597,12 @@ def CountedByOrNull : DeclOrTypeAttr { let LangOpts = [COnly]; } +def NoPointerFieldProtection : DeclOrTypeAttr { + let Spellings = [Clang<"no_field_protection">]; + let Subjects = SubjectList<[Field], ErrorDiag>; + let Documentation = [Undocumented]; +} + def SizedBy : DeclOrTypeAttr { let Spellings = [Clang<"sized_by">]; let Subjects = SubjectList<[Field], ErrorDiag>; diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index 14bff8a68846d..e0de99aa683f8 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -265,6 +265,9 @@ FEATURE(shadow_call_stack, FEATURE(tls, PP.getTargetInfo().isTLSSupported()) FEATURE(underlying_type, LangOpts.CPlusPlus) FEATURE(experimental_library, LangOpts.ExperimentalLibrary) +FEATURE(pointer_field_protection, + LangOpts.getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) // C11 features supported by other languages as extensions. EXTENSION(c_alignas, true) diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index 72321c204ce96..ad209c8b8be02 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -455,6 +455,9 @@ LANGOPT(RelativeCXXABIVTables, 1, 0, NotCompatible, LANGOPT(OmitVTableRTTI, 1, 0, NotCompatible, "Use an ABI-incompatible v-table layout that omits the RTTI component") +ENUM_LANGOPT(PointerFieldProtection, PointerFieldProtectionKind, 2, PointerFieldProtectionKind::None, NotCompatible, + "Encode struct pointer fields to protect against UAF vulnerabilities") + LANGOPT(VScaleMin, 32, 0, NotCompatible, "Minimum vscale value") LANGOPT(VScaleMax, 32, 0, NotCompatible, "Maximum vscale value") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 4c642c9e10c91..d856c6546b30a 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -379,6 +379,17 @@ class LangOptionsBase { BKey }; + enum class PointerFieldProtectionKind { + /// Pointer field protection disabled + None, + /// Pointer field protection enabled, allocator does not tag heap + /// allocations. + Untagged, + /// Pointer field protection enabled, allocator is expected to tag heap + /// allocations. + Tagged, + }; + enum class ThreadModelKind { /// POSIX Threads. POSIX, diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 77379f1130149..e47be19aae727 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -3033,6 +3033,12 @@ defm experimental_omit_vtable_rtti : BoolFOption<"experimental-omit-vtable-rtti" NegFlag, BothFlags<[], [CC1Option], " the RTTI component from virtual tables">>; +def experimental_pointer_field_protection_EQ : Joined<["-"], "fexperimental-pointer-field-protection=">, Group, + Visibility<[ClangOption, CC1Option]>, + Values<"none,untagged,tagged">, NormalizedValuesScope<"LangOptions::PointerFieldProtectionKind">, + NormalizedValues<["None", "Untagged", "Tagged"]>, + MarshallingInfoEnum, "None">; + def fcxx_abi_EQ : Joined<["-"], "fc++-abi=">, Group, Visibility<[ClangOption, CC1Option]>, HelpText<"C++ ABI to use. This will override the target C++ ABI.">; diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp index 679812adcdf12..d978d5559d057 100644 --- a/clang/lib/AST/ASTContext.cpp +++ b/clang/lib/AST/ASTContext.cpp @@ -15162,3 +15162,97 @@ bool ASTContext::useAbbreviatedThunkName(GlobalDecl VirtualMethodDecl, ThunksToBeAbbreviated[VirtualMethodDecl] = std::move(SimplifiedThunkNames); return Result; } + +bool ASTContext::arePFPFieldsTriviallyRelocatable(const RecordDecl *RD) const { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::Tagged) + return !isa(RD) || + cast(RD)->hasTrivialDestructor(); + return true; +} + +bool ASTContext::isPFPStruct(const RecordDecl *rec) const { + if (getLangOpts().getPointerFieldProtection() != + LangOptions::PointerFieldProtectionKind::None) + if (auto *cxxRec = dyn_cast(rec)) + return !cxxRec->isStandardLayout(); + return false; +} + +void ASTContext::findPFPFields(QualType Ty, CharUnits Offset, + std::vector &Fields, + bool IncludeVBases) const { + if (auto *AT = getAsConstantArrayType(Ty)) { + if (auto *ElemDecl = AT->getElementType()->getAsCXXRecordDecl()) { + const ASTRecordLayout &ElemRL = getASTRecordLayout(ElemDecl); + for (unsigned i = 0; i != AT->getSize(); ++i) { + findPFPFields(AT->getElementType(), Offset + i * ElemRL.getSize(), + Fields, true); + } + } + } + auto *Decl = Ty->getAsCXXRecordDecl(); + if (!Decl) + return; + const ASTRecordLayout &RL = getASTRecordLayout(Decl); + for (FieldDecl *field : Decl->fields()) { + CharUnits fieldOffset = + Offset + toCharUnitsFromBits(RL.getFieldOffset(field->getFieldIndex())); + if (isPFPField(field)) + Fields.push_back({Offset, fieldOffset, field}); + findPFPFields(field->getType(), fieldOffset, Fields, true); + } + for (auto &Base : Decl->bases()) { + if (Base.isVirtual()) + continue; + CharUnits BaseOffset = + Offset + RL.getBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + if (IncludeVBases) { + for (auto &Base : Decl->vbases()) { + CharUnits BaseOffset = + Offset + RL.getVBaseClassOffset(Base.getType()->getAsCXXRecordDecl()); + findPFPFields(Base.getType(), BaseOffset, Fields, false); + } + } +} + +bool ASTContext::hasPFPFields(QualType ty) const { + std::vector pfpFields; + findPFPFields(ty, CharUnits::Zero(), pfpFields, true); + return !pfpFields.empty(); +} + +bool ASTContext::isPFPField(const FieldDecl *field) const { + if (!isPFPStruct(field->getParent())) + return false; + return field->getType()->isPointerType() && + !field->hasAttr(); +} + +void ASTContext::recordMemberDataPointerEvaluation(const ValueDecl *VD) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None) + return; + auto *FD = dyn_cast(VD); + if (!FD) + FD = cast(cast(VD)->chain().back()); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} + +void ASTContext::recordOffsetOfEvaluation(const OffsetOfExpr *E) { + if (getLangOpts().getPointerFieldProtection() == + LangOptions::PointerFieldProtectionKind::None || + E->getNumComponents() == 0) + return; + OffsetOfNode Comp = E->getComponent(E->getNumComponents() - 1); + if (Comp.getKind() != OffsetOfNode::Field) + return; + FieldDecl *FD = Comp.getField(); + if (!isPFPField(FD)) + return; + PFPFieldsWithEvaluatedOffset.insert(FD); +} diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index e23e84368516a..38d1267fee7d1 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -15001,6 +15001,7 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( } bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) { + Info.Ctx.recordOffsetOfEvaluation(OOE); CharUnits Result; unsigned n = OOE->getNumComponents(); if (n == 0) diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp index 818d2139628e3..820080a613396 100644 --- a/clang/lib/AST/TypePrinter.cpp +++ b/clang/lib/AST/TypePrinter.cpp @@ -2119,6 +2119,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T, case attr::ExtVectorType: OS << "ext_vector_type"; break; + case attr::NoPointerFieldProtection: + OS << "no_field_protection"; + break; } OS << "))"; } diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 48c91eb4a5b4f..9821b3f889aec 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -4494,6 +4494,19 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, EmitArgCheck(TCK_Load, Src, E->getArg(1), 1); auto *I = Builder.CreateMemMove(Dest, Src, SizeVal, false); addInstToNewSourceAtom(I, nullptr); + if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_trivially_relocate) { + std::vector PFPFields; + getContext().findPFPFields(E->getArg(0)->getType()->getPointeeType(), + CharUnits::Zero(), PFPFields, true); + for (auto &Field : PFPFields) { + if (getContext().arePFPFieldsTriviallyRelocatable( + Field.field->getParent())) + continue; + auto DestFieldPtr = EmitAddressOfPFPField(Dest, Field); + auto SrcFieldPtr = EmitAddressOfPFPField(Src, Field); + Builder.CreateStore(Builder.CreateLoad(SrcFieldPtr), DestFieldPtr); + } + } return RValue::get(Dest, *this); } case Builtin::BImemset: diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp index c8c3d6b20c496..d8237273088be 100644 --- a/clang/lib/CodeGen/CGCall.cpp +++ b/clang/lib/CodeGen/CGCall.cpp @@ -1318,7 +1318,8 @@ static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, llvm::Type *Ty, /// This safely handles the case when the src type is smaller than the /// destination type; in this situation the values of bits which not /// present in the src are undefined. -static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, +static llvm::Value *CreateCoercedLoad(Address Src, QualType SrcFETy, + llvm::Type *Ty, CodeGenFunction &CGF) { llvm::Type *SrcTy = Src.getElementType(); @@ -1326,6 +1327,57 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, if (SrcTy == Ty) return CGF.Builder.CreateLoad(Src); + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct in registers has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + CGF.getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto LoadCoercedField = [&](CharUnits Offset, + llvm::Type *FieldType) -> llvm::Value * { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *FieldVal = CGF.Builder.CreateLoad(fieldAddr); + if (isa(FieldType)) + FieldVal = CGF.Builder.CreatePtrToInt(FieldVal, FieldType); + PFPFields.erase(PFPFields.begin()); + return FieldVal; + } + auto FieldAddr = CGF.Builder + .CreateConstInBoundsByteGEP( + Src.withElementType(CGF.Int8Ty), Offset) + .withElementType(FieldType); + return CGF.Builder.CreateLoad(FieldAddr); + }; + if (isa(Ty) || isa(Ty)) { + auto Addr = CGF.EmitAddressOfPFPField(Src, PFPFields[0]); + llvm::Value *Val = CGF.Builder.CreateLoad(Addr); + if (isa(Ty)) + Val = CGF.Builder.CreatePtrToInt(Val, Ty); + return Val; + } + if (auto *AT = dyn_cast(Ty)) { + auto *ET = AT->getElementType(); + CharUnits wordSize = CGF.getContext().toCharUnitsFromBits( + CGF.CGM.getDataLayout().getTypeSizeInBits(ET)); + CharUnits Offset = CharUnits::Zero(); + llvm::Value *Val = llvm::PoisonValue::get(AT); + for (unsigned i = 0; i != AT->getNumElements(); ++i, Offset += wordSize) + Val = CGF.Builder.CreateInsertValue(Val, LoadCoercedField(Offset, ET), i); + return Val; + } + auto *ST = cast(Ty); + llvm::Value *Val = llvm::PoisonValue::get(ST); + auto *SL = CGF.CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + Val = CGF.Builder.CreateInsertValue( + Val, LoadCoercedField(Offset, ST->getElementType(i)), i); + } + return Val; + } + llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); if (llvm::StructType *SrcSTy = dyn_cast(SrcTy)) { @@ -1397,7 +1449,9 @@ static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, return CGF.Builder.CreateLoad(Tmp); } -void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, +void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, + QualType SrcFETy, + Address Dst, llvm::TypeSize DstSize, bool DstIsVolatile) { if (!DstSize) @@ -1418,6 +1472,52 @@ void CodeGenFunction::CreateCoercedStore(llvm::Value *Src, Address Dst, } } + // Coercion directly through memory does not work if the structure has pointer + // field protection because the struct passed by value has a different bit + // pattern to the struct in memory, so we must read the elements one by one + // and use them to form the coerced structure. + std::vector PFPFields; + getContext().findPFPFields(SrcFETy, CharUnits::Zero(), PFPFields, true); + if (!PFPFields.empty()) { + auto StoreCoercedField = [&](CharUnits Offset, llvm::Value *FieldVal) { + if (!PFPFields.empty() && PFPFields[0].offset == Offset) { + auto fieldAddr = EmitAddressOfPFPField(Dst, PFPFields[0]); + if (isa(FieldVal->getType())) + FieldVal = Builder.CreateIntToPtr(FieldVal, VoidPtrTy); + Builder.CreateStore(FieldVal, fieldAddr); + PFPFields.erase(PFPFields.begin()); + } else { + auto fieldAddr = + Builder + .CreateConstInBoundsByteGEP(Dst.withElementType(Int8Ty), Offset) + .withElementType(FieldVal->getType()); + Builder.CreateStore(FieldVal, fieldAddr); + } + }; + + if (isa(SrcTy) || isa(SrcTy)) { + if (isa(SrcTy)) + Src = Builder.CreateIntToPtr(Src, VoidPtrTy); + auto Addr = EmitAddressOfPFPField(Dst, PFPFields[0]); + Builder.CreateStore(Src, Addr); + } else if (auto *at = dyn_cast(SrcTy)) { + auto *et = at->getElementType(); + CharUnits wordSize = getContext().toCharUnitsFromBits( + CGM.getDataLayout().getTypeSizeInBits(et)); + CharUnits Offset = CharUnits::Zero(); + for (unsigned i = 0; i != at->getNumElements(); ++i, Offset += wordSize) + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } else { + auto *ST = cast(SrcTy); + auto *SL = CGM.getDataLayout().getStructLayout(ST); + for (unsigned i = 0; i != ST->getNumElements(); ++i) { + CharUnits Offset = CharUnits::fromQuantity(SL->getElementOffset(i)); + StoreCoercedField(Offset, Builder.CreateExtractValue(Src, i)); + } + } + return; + } + if (SrcSize.isScalable() || SrcSize <= DstSize) { if (SrcTy->isIntegerTy() && Dst.getElementType()->isPointerTy() && SrcSize == CGM.getDataLayout().getTypeAllocSize(Dst.getElementType())) { @@ -3409,7 +3509,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, auto AI = Fn->getArg(FirstIRArg); AI->setName(Arg->getName() + ".coerce"); CreateCoercedStore( - AI, Ptr, + AI, Ty, Ptr, llvm::TypeSize::getFixed( getContext().getTypeSizeInChars(Ty).getQuantity() - ArgI.getDirectOffset()), @@ -4042,7 +4142,7 @@ void CodeGenFunction::EmitFunctionEpilog( // If the value is offset in memory, apply the offset now. Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); - RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); + RV = CreateCoercedLoad(V, RetTy, RetAI.getCoerceToType(), *this); } // In ARC, end functions that return a retainable type with a call @@ -4092,6 +4192,7 @@ void CodeGenFunction::EmitFunctionEpilog( auto eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + RetTy, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -5607,7 +5708,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // In the simple case, just pass the coerced loaded value. assert(NumIRArgs == 1); llvm::Value *Load = - CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); + CreateCoercedLoad(Src, I->Ty, ArgInfo.getCoerceToType(), *this); if (CallInfo.isCmseNSCall()) { // For certain parameter types, clear padding bits, as they may reveal @@ -5669,6 +5770,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, Address eltAddr = Builder.CreateStructGEP(addr, i); llvm::Value *elt = CreateCoercedLoad( eltAddr, + I->Ty, unpaddedStruct ? unpaddedStruct->getElementType(unpaddedIndex++) : unpaddedCoercionType, *this); @@ -6169,7 +6271,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, // If the value is offset in memory, apply the offset now. Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI); CreateCoercedStore( - CI, StorePtr, + CI, RetTy, StorePtr, llvm::TypeSize::getFixed(DestSize - RetAI.getDirectOffset()), DestIsVolatile); } diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp index 4a465e6526da0..4e602935bf60b 100644 --- a/clang/lib/CodeGen/CGClass.cpp +++ b/clang/lib/CodeGen/CGClass.cpp @@ -591,12 +591,20 @@ static void EmitBaseInitializer(CodeGenFunction &CGF, isBaseVirtual); } -static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) { +static bool isMemcpyEquivalentSpecialMember(CodeGenModule &CGM, + const CXXMethodDecl *D) { auto *CD = dyn_cast(D); if (!(CD && CD->isCopyOrMoveConstructor()) && !D->isCopyAssignmentOperator() && !D->isMoveAssignmentOperator()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(D->getParent()) && + CGM.getContext().hasPFPFields( + QualType(D->getParent()->getTypeForDecl(), 0))) + return false; + // We can emit a memcpy for a trivial copy or move constructor/assignment. if (D->isTrivial() && !D->getParent()->mayInsertExtraPadding()) return true; @@ -662,7 +670,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF, QualType BaseElementTy = CGF.getContext().getBaseElementType(Array); CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); if (BaseElementTy.isPODType(CGF.getContext()) || - (CE && isMemcpyEquivalentSpecialMember(CE->getConstructor()))) { + (CE && isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor()))) { unsigned SrcArgIndex = CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args); llvm::Value *SrcPtr @@ -929,6 +937,11 @@ namespace { if (PointerAuthQualifier Q = F->getType().getPointerAuth(); Q && Q.isAddressDiscriminated()) return false; + // Non-trivially-copyable fields with pointer field protection need to be + // copied one by one. + if (!CGF.getContext().arePFPFieldsTriviallyRelocatable(ClassDecl) && + CGF.getContext().isPFPField(F)) + return false; return true; } @@ -1066,7 +1079,8 @@ namespace { CXXConstructExpr *CE = dyn_cast(MemberInit->getInit()); // Bail out on non-memcpyable, not-trivially-copyable members. - if (!(CE && isMemcpyEquivalentSpecialMember(CE->getConstructor())) && + if (!(CE && + isMemcpyEquivalentSpecialMember(CGF.CGM, CE->getConstructor())) && !(FieldType.isTriviallyCopyableType(CGF.getContext()) || FieldType->isReferenceType())) return false; @@ -1177,7 +1191,7 @@ namespace { return nullptr; } else if (CXXMemberCallExpr *MCE = dyn_cast(S)) { CXXMethodDecl *MD = dyn_cast(MCE->getCalleeDecl()); - if (!(MD && isMemcpyEquivalentSpecialMember(MD))) + if (!(MD && isMemcpyEquivalentSpecialMember(CGF.CGM, MD))) return nullptr; MemberExpr *IOA = dyn_cast(MCE->getImplicitObjectArgument()); if (!IOA) @@ -2153,7 +2167,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D, // If this is a trivial constructor, emit a memcpy now before we lose // the alignment information on the argument. // FIXME: It would be better to preserve alignment information into CallArg. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor"); const Expr *Arg = E->getArg(0); @@ -2212,6 +2226,22 @@ void CodeGenFunction::EmitCXXConstructorCall( EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This, getContext().getRecordType(ClassDecl), CharUnits::Zero()); + // When initializing an object that has pointer field protection and whose + // fields are not trivially relocatable we must initialize any pointer fields + // to a valid signed pointer (any pointer value will do, but we just use null + // pointers). This is because if the object is subsequently copied, its copy + // constructor will need to read and authenticate any pointer fields in order + // to copy the object to a new address, which will fail if the pointers are + // uninitialized. + if (!getContext().arePFPFieldsTriviallyRelocatable(D->getParent())) { + std::vector PFPFields; + getContext().findPFPFields(QualType(ClassDecl->getTypeForDecl(), 0), + CharUnits::Zero(), PFPFields, Type != Ctor_Base); + for (auto &Field : PFPFields) + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), + EmitAddressOfPFPField(This, Field)); + } + if (D->isTrivial() && D->isDefaultConstructor()) { assert(Args.size() == 1 && "trivial default ctor with args"); return; @@ -2220,7 +2250,7 @@ void CodeGenFunction::EmitCXXConstructorCall( // If this is a trivial constructor, just emit what's needed. If this is a // union copy constructor, we must emit a memcpy, because the AST does not // model that copy. - if (isMemcpyEquivalentSpecialMember(D)) { + if (isMemcpyEquivalentSpecialMember(CGM, D)) { assert(Args.size() == 2 && "unexpected argcount for trivial ctor"); QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType(); Address Src = makeNaturalAddressForPointer( @@ -2998,7 +3028,15 @@ void CodeGenFunction::EmitForwardingCallToLambda( QualType resultType = FPT->getReturnType(); ReturnValueSlot returnSlot; if (!resultType->isVoidType() && - calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect && + (calleeFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect || + // With pointer field protection, we need to set up the return slot when + // returning an object with trivial ABI to avoid the memcpy that would + // otherwise be generated by the call to EmitReturnOfRValue() below, as + // that may corrupt the pointer signature. It doesn't hurt to do this all + // the time as it results in slightly simpler codegen. + (resultType->isRecordType() && + resultType->getAsCXXRecordDecl() + ->hasTrivialCopyConstructorForCall())) && !hasScalarEvaluationKind(calleeFnInfo->getReturnType())) returnSlot = ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(), diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index 85c768807572f..081db0da49b2e 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -5087,10 +5087,16 @@ static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, return emitAddrOfZeroSizeField(CGF, base, field, IsInBounds); const RecordDecl *rec = field->getParent(); - unsigned idx = CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); + if (CGF.getContext().isPFPField(field)) { + const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(rec); + auto Offset = CGF.getContext().toCharUnitsFromBits( + RL.getFieldOffset(field->getFieldIndex())); + return CGF.EmitAddressOfPFPField(base, field, Offset); + } + if (!IsInBounds) return CGF.Builder.CreateConstGEP2_32(base, 0, idx, field->getName()); diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp index cad6731173700..a8189643337e9 100644 --- a/clang/lib/CodeGen/CGExprAgg.cpp +++ b/clang/lib/CodeGen/CGExprAgg.cpp @@ -138,7 +138,7 @@ class AggExprEmitter : public StmtVisitor { if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E)) { CGF.CreateCoercedStore( - Result, Dest.getAddress(), + Result, E->getType(), Dest.getAddress(), llvm::TypeSize::getFixed( Dest.getPreferredSize(CGF.getContext(), E->getType()) .getQuantity()), diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp index 359e30cb8f5cd..9333e1897ae53 100644 --- a/clang/lib/CodeGen/CGExprCXX.cpp +++ b/clang/lib/CodeGen/CGExprCXX.cpp @@ -260,6 +260,16 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr( (MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()) && !MD->getParent()->mayInsertExtraPadding(); + // Assignment operators for objects with non-trivially-relocatable PFP fields + // aren't trivial, we need to auth and sign the fields one by one. + if (TrivialAssignment && + !CGM.getContext().arePFPFieldsTriviallyRelocatable(MD->getParent()) && + CGM.getContext().hasPFPFields( + QualType(MD->getParent()->getTypeForDecl(), 0))) { + TrivialForCodegen = false; + TrivialAssignment = false; + } + // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment // operator before the LHS. CallArgList RtlArgStorage; diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp index 715bd392f59f7..b345ddccb557d 100644 --- a/clang/lib/CodeGen/CGExprConstant.cpp +++ b/clang/lib/CodeGen/CGExprConstant.cpp @@ -904,6 +904,29 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD, if (!EltInit) return false; + if (CGM.getContext().isPFPField(*Field)) { + llvm::ConstantInt *Disc; + llvm::Constant *AddrDisc; + if (CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) { + uint64_t FieldSignature = + std::hash()(CGM.getPFPFieldName(*Field)) & 0xffff; + Disc = llvm::ConstantInt::get(CGM.Int64Ty, FieldSignature); + AddrDisc = llvm::ConstantPointerNull::get(CGM.VoidPtrTy); + } else if (Emitter.isAbstract()) { + return false; + } else { + Disc = llvm::ConstantInt::get( + CGM.Int64Ty, (-(Layout.getFieldOffset(FieldNo) / 8)) & 0xffff); + AddrDisc = Emitter.getCurrentAddrPrivate(); + } + EltInit = llvm::ConstantPtrAuth::get( + EltInit, llvm::ConstantInt::get(CGM.Int32Ty, 2), Disc, AddrDisc, + CGM.getPFPDeactivationSymbol(*Field)); + if (!CGM.getContext().arePFPFieldsTriviallyRelocatable(RD)) + Emitter.registerCurrentAddrPrivate(EltInit, + cast(AddrDisc)); + } + if (ZeroInitPadding) { if (!DoZeroInitPadding(Layout, FieldNo, **Field, AllowOverwrite, SizeSoFar, ZeroFieldSize)) @@ -1654,7 +1677,20 @@ ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value, llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) { initializeNonAbstract(D.getType().getAddressSpace()); - return markIfFailed(tryEmitPrivateForVarInit(D)); + llvm::Constant *Init = tryEmitPrivateForVarInit(D); + + // If a placeholder address was needed for a TLS variable, implying that the + // initializer's value depends on its address, then the object may not be + // initialized in .tdata because the initializer will be memcpy'd to the + // thread's TLS. Instead the initialization must be done in code. + if (!PlaceholderAddresses.empty() && D.getTLSKind() != VarDecl::TLS_None) { + for (auto &entry : PlaceholderAddresses) + entry.second->eraseFromParent(); + PlaceholderAddresses.clear(); + Init = nullptr; + } + + return markIfFailed(Init); } llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E, @@ -2609,6 +2645,7 @@ CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) { return getCXXABI().EmitMemberFunctionPointer(method); // Otherwise, a member data pointer. + getContext().recordMemberDataPointerEvaluation(decl); uint64_t fieldOffset = getContext().getFieldOffset(decl); CharUnits chars = getContext().toCharUnitsFromBits((int64_t) fieldOffset); return getCXXABI().EmitMemberDataPointer(type, chars); diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 776a646ceb32f..c023296be8b7d 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -2219,6 +2219,31 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType, CGF.EmitBlock(contBB); } +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const PFPField &Field) { + return EmitAddressOfPFPField( + Builder.CreateConstInBoundsByteGEP(RecordPtr.withElementType(Int8Ty), + Field.structOffset), + Field.field, Field.offset - Field.structOffset); +} + +Address CodeGenFunction::EmitAddressOfPFPField(Address RecordPtr, + const FieldDecl *Field, + CharUnits Offset) { + return Address( + EmitRuntimeCall( + CGM.getIntrinsic(llvm::Intrinsic::protected_field_ptr), + {RecordPtr.getBasePointer(), Builder.getInt64(Offset.getQuantity()), + llvm::MetadataAsValue::get( + getLLVMContext(), + llvm::MDString::get(getLLVMContext(), + CGM.getPFPFieldName(Field))), + getContext().arePFPFieldsTriviallyRelocatable(Field->getParent()) + ? Builder.getFalse() + : Builder.getTrue()}), + VoidPtrTy, RecordPtr.getAlignment().alignmentAtOffset(Offset)); +} + void CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Ignore empty classes in C++. @@ -2281,13 +2306,22 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) { // Get and call the appropriate llvm.memcpy overload. Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false); - return; + } else { + // Otherwise, just memset the whole thing to zero. This is legal + // because in LLVM, all default initializers (other than the ones we just + // handled above, and the case handled below) are guaranteed to have a bit + // pattern of all zeros. + Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); } - // Otherwise, just memset the whole thing to zero. This is legal - // because in LLVM, all default initializers (other than the ones we just - // handled above) are guaranteed to have a bit pattern of all zeros. - Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false); + // With the pointer field protection feature, null pointers do not have a bit + // pattern of zero in memory, so we must initialize them separately. + std::vector PFPFields; + getContext().findPFPFields(Ty, CharUnits::Zero(), PFPFields, true); + for (auto &Field : PFPFields) { + auto addr = EmitAddressOfPFPField(DestPtr, Field); + Builder.CreateStore(llvm::ConstantPointerNull::get(VoidPtrTy), addr); + } } llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) { diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h index 6c32c98cec011..a5be27fc057a3 100644 --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -5020,8 +5020,8 @@ class CodeGenFunction : public CodeGenTypeCache { /// Create a store to \arg DstPtr from \arg Src, truncating the stored value /// to at most \arg DstSize bytes. - void CreateCoercedStore(llvm::Value *Src, Address Dst, llvm::TypeSize DstSize, - bool DstIsVolatile); + void CreateCoercedStore(llvm::Value *Src, QualType SrcFETy, Address Dst, + llvm::TypeSize DstSize, bool DstIsVolatile); /// EmitExtendGCLifetime - Given a pointer to an Objective-C object, /// make sure it survives garbage collection until this point. @@ -5509,6 +5509,10 @@ class CodeGenFunction : public CodeGenTypeCache { void EmitRISCVMultiVersionResolver(llvm::Function *Resolver, ArrayRef Options); + Address EmitAddressOfPFPField(Address RecordPtr, const PFPField &Field); + Address EmitAddressOfPFPField(Address RecordPtr, const FieldDecl *Field, + CharUnits Offset); + private: QualType getVarArgType(const Expr *Arg); diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index c8866f15745c2..b359a6abbb151 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -962,6 +962,7 @@ void CodeGenModule::Release() { applyGlobalValReplacements(); applyReplacements(); emitMultiVersionFunctions(); + emitPFPFieldsWithEvaluatedOffset(); if (Context.getLangOpts().IncrementalExtensions && GlobalTopLevelStmtBlockInFlight.first) { @@ -4564,6 +4565,35 @@ void CodeGenModule::emitMultiVersionFunctions() { emitMultiVersionFunctions(); } +llvm::GlobalValue *CodeGenModule::getPFPDeactivationSymbol(FieldDecl *FD) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *DS = TheModule.getNamedValue(DSName); + if (!DS) { + DS = new llvm::GlobalVariable(TheModule, Int8Ty, false, + llvm::GlobalVariable::ExternalWeakLinkage, + nullptr, DSName); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + } + return DS; +} + +void CodeGenModule::emitPFPFieldsWithEvaluatedOffset() { + llvm::Constant *Nop = llvm::ConstantExpr::getIntToPtr( + llvm::ConstantInt::get(Int64Ty, 0xd503201f), VoidPtrTy); + for (auto *FD : getContext().PFPFieldsWithEvaluatedOffset) { + std::string DSName = "__pfp_ds_" + getPFPFieldName(FD); + llvm::GlobalValue *OldDS = TheModule.getNamedValue(DSName); + llvm::GlobalValue *DS = llvm::GlobalAlias::create( + Int8Ty, 0, llvm::GlobalValue::ExternalLinkage, DSName, Nop, &TheModule); + DS->setVisibility(llvm::GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } +} + static void replaceDeclarationWith(llvm::GlobalValue *Old, llvm::Constant *New) { assert(cast(Old)->isDeclaration() && "Not a declaration"); @@ -8124,3 +8154,12 @@ void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) { NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx); } + +std::string CodeGenModule::getPFPFieldName(const FieldDecl *FD) { + std::string OutName; + llvm::raw_string_ostream Out(OutName); + getCXXABI().getMangleContext().mangleCanonicalTypeName( + QualType(FD->getParent()->getTypeForDecl(), 0), Out, false); + Out << "." << FD->getName(); + return OutName; +} diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h index cb013feb769fc..26ac025fe7a4a 100644 --- a/clang/lib/CodeGen/CodeGenModule.h +++ b/clang/lib/CodeGen/CodeGenModule.h @@ -1824,6 +1824,9 @@ class CodeGenModule : public CodeGenTypeCache { return PAlign; } + std::string getPFPFieldName(const FieldDecl *FD); + llvm::GlobalValue *getPFPDeactivationSymbol(FieldDecl *FD); + private: bool shouldDropDLLAttribute(const Decl *D, const llvm::GlobalValue *GV) const; @@ -2018,6 +2021,10 @@ class CodeGenModule : public CodeGenTypeCache { llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map, StringRef Suffix); + + /// Emit deactivation symbols for any PFP fields whose offset is taken with + /// offsetof. + void emitPFPFieldsWithEvaluatedOffset(); }; } // end namespace CodeGen diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp index f4a99467010af..cb09bc18a999d 100644 --- a/clang/lib/CodeGen/ItaniumCXXABI.cpp +++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp @@ -1240,6 +1240,7 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP, return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM); } + getContext().recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = getContext().toCharUnitsFromBits(getContext().getFieldOffset(MPD)); return EmitMemberDataPointer(MPT, ThisAdjustment + FieldOffset); diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp index a181559834296..300fa450c5247 100644 --- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp +++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp @@ -2913,6 +2913,7 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP, // the class in which it was declared, and convert from there if necessary. // For indirect field decls, get the outermost anonymous field and use the // parent class. + Ctx.recordMemberDataPointerEvaluation(MPD); CharUnits FieldOffset = Ctx.toCharUnitsFromBits(Ctx.getFieldOffset(MPD)); const FieldDecl *FD = dyn_cast(MPD); if (!FD) diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index b76163afc8aa4..f8584f0a11d31 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -7629,6 +7629,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, Twine("-funique-source-file-identifier=") + Input.getBaseInput())); } + if (!IsCudaDevice) + Args.AddLastArg(CmdArgs, + options::OPT_experimental_pointer_field_protection_EQ); + // Setup statistics file output. SmallString<128> StatsFile = getStatsFileName(Args, Output, Input, D); if (!StatsFile.empty()) { diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index 7ebb53318702c..3856b07209f60 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -6581,6 +6581,10 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL)); } +static void handleNoPFPAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { + D->addAttr(NoPointerFieldProtectionAttr::Create(S.Context, AL)); +} + static void handleCountedByAttrField(Sema &S, Decl *D, const ParsedAttr &AL) { auto *FD = dyn_cast(D); assert(FD); @@ -7603,6 +7607,10 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, handleCountedByAttrField(S, D, AL); break; + case ParsedAttr::AT_NoPointerFieldProtection: + handleNoPFPAttrField(S, D, AL); + break; + // Microsoft attributes: case ParsedAttr::AT_LayoutVersion: handleLayoutVersion(S, D, AL); diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp index 1d8687e4bf1c1..e6291f2e08e81 100644 --- a/clang/lib/Sema/SemaTypeTraits.cpp +++ b/clang/lib/Sema/SemaTypeTraits.cpp @@ -235,6 +235,22 @@ static bool IsEligibleForReplacement(Sema &SemaRef, const CXXRecordDecl *D) { return !D->hasDeletedDestructor(); } +static bool IsImplementationDefinedNonRelocatable(Sema &SemaRef, + const CXXRecordDecl *D) { + // FIXME: Should also check for polymorphic union members here if PAuth ABI is + // enabled. + + // FIXME: PFP should not affect trivial relocatability except in cases where a + // PFP field is a member of a union, instead it should affect the + // implementation of std::trivially_relocate. See: + // https://discourse.llvm.org/t/rfc-structure-protection-a-family-of-uaf-mitigation-techniques/85555/16?u=pcc + if (!SemaRef.Context.arePFPFieldsTriviallyRelocatable(D) && + SemaRef.Context.hasPFPFields(QualType(D->getTypeForDecl(), 0))) + return true; + + return false; +} + ASTContext::CXXRecordDeclRelocationInfo Sema::CheckCXX2CRelocatableAndReplaceable(const CXXRecordDecl *D) { ASTContext::CXXRecordDeclRelocationInfo Info{false, false}; @@ -276,6 +292,11 @@ Sema::CheckCXX2CRelocatableAndReplaceable(const CXXRecordDecl *D) { if (!IsEligibleForTrivialRelocation(*this, D)) return false; + // if it does not meet implementation-defined criteria for inhibiting + // trivially-relocatable, + if (IsImplementationDefinedNonRelocatable(*this, D)) + return false; + // has the trivially_relocatable_if_eligible class-property-specifier, if (D->hasAttr()) return true; diff --git a/clang/test/CodeGen/pfp-attribute-disable.cpp b/clang/test/CodeGen/pfp-attribute-disable.cpp new file mode 100644 index 0000000000000..a8efb08b9a7ee --- /dev/null +++ b/clang/test/CodeGen/pfp-attribute-disable.cpp @@ -0,0 +1,33 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + + +struct S { + int* ptr1; + __attribute__((no_field_protection)) int* ptr2; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers_without_no_field_protection +int* load_pointers_without_no_field_protection(S *t) { + return t->ptr1; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: load_pointers_with_no_field_protection +int* load_pointers_with_no_field_protection(S *t) { + return t->ptr2; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_without_no_field_protection +void store_pointers_without_no_field_protection(S *t, int *input) { + t->ptr1 = input; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + +// CHECK-LABEL: store_pointers_with_no_field_protection +void store_pointers_with_no_field_protection(S *t, int *input) { + t->ptr2 = input; +} +// CHECK-NOT: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-load-store.cpp b/clang/test/CodeGen/pfp-load-store.cpp new file mode 100644 index 0000000000000..b5f6a11574270 --- /dev/null +++ b/clang/test/CodeGen/pfp-load-store.cpp @@ -0,0 +1,40 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -O1 -o - %s | FileCheck %s + +int val; + +struct Pointer { + int* ptr; +private: + int private_data; +}; + +struct ArrayType { + int* array[3]; +private: + int private_data; +}; + +struct Array { + ArrayType array; +private: + int private_data; +}; + +struct Struct { + Pointer ptr; +}; + +// CHECK-LABEL: test_pointer +Pointer test_pointer(Pointer t) { + t.ptr = &val; + return t; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} + + + +// CHECK-LABEL: test_struct +int* test_struct(Struct *t) { + return (t->ptr).ptr; +} +// CHECK: call {{.*}} @llvm.protected.field.ptr{{.*}} diff --git a/clang/test/CodeGen/pfp-memcpy.cpp b/clang/test/CodeGen/pfp-memcpy.cpp new file mode 100644 index 0000000000000..3e92db6110ae3 --- /dev/null +++ b/clang/test/CodeGen/pfp-memcpy.cpp @@ -0,0 +1,19 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct ClassWithTrivialCopy { + ClassWithTrivialCopy(); + ~ClassWithTrivialCopy(); + void *a; +private: + void *c; +}; + +// Make sure we don't emit memcpy for operator= and constuctors. +void make_trivial_copy(ClassWithTrivialCopy *s1, ClassWithTrivialCopy *s2) { + *s1 = *s2; + ClassWithTrivialCopy s3(*s2); +} + +// CHECK-LABEL: define{{.*}} void @_Z17make_trivial_copyP20ClassWithTrivialCopyS0_ +// CHECK-NOT: memcpy +// CHECK: ret void \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-null-init.cpp b/clang/test/CodeGen/pfp-null-init.cpp new file mode 100644 index 0000000000000..94c1224f1be13 --- /dev/null +++ b/clang/test/CodeGen/pfp-null-init.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s + +struct S { + void *p; +private: + int private_data; +}; + +// CHECK-LABEL: null_init +void null_init() { + S s{}; +} + +// Check that the constructor was applied +// CHECK: call void @llvm.memset.{{.*}} +// CHECK: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.p", i1 false) \ No newline at end of file diff --git a/clang/test/CodeGen/pfp-struct-gep.cpp b/clang/test/CodeGen/pfp-struct-gep.cpp new file mode 100644 index 0000000000000..964545efa9f4b --- /dev/null +++ b/clang/test/CodeGen/pfp-struct-gep.cpp @@ -0,0 +1,25 @@ +// RUN: %clang_cc1 -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-NOPFP +// RUN: %clang_cc1 -fexperimental-pointer-field-protection=tagged -emit-llvm -o - %s | FileCheck %s -check-prefix=CHECK-PFP + + +struct S { + int* ptr; +private: + int private_data; +}; // Not Standard-layout, mixed access + +// CHECK-LABEL: load_pointers +int* load_pointers(S *t) { + return t->ptr; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + +// CHECK-LABEL: store_pointers +void store_pointers(S* t, int* p) { + t->ptr = p; +} +// CHECK-PFP: call {{.*}} @llvm.protected.field.ptr({{.*}}, i64 0, metadata !"_ZTS1S.ptr", i1 false) +// CHECK-NOPFP: getelementptr + + diff --git a/clang/test/CodeGenCXX/trivial_abi.cpp b/clang/test/CodeGenCXX/trivial_abi.cpp index b8cc0d1cc6528..e96f647749a79 100644 --- a/clang/test/CodeGenCXX/trivial_abi.cpp +++ b/clang/test/CodeGenCXX/trivial_abi.cpp @@ -283,11 +283,9 @@ static_assert(sizeof(S) == 8 && sizeof(S2) == 8, ""); // CHECK: define{{.*}} @"_ZN3$_08__invokeEv"() // CHECK: %[[RETVAL:.*]] = alloca %[[STRUCT_SMALL]], align 8 -// CHECK: %[[COERCE:.*]] = alloca %[[STRUCT_SMALL]], align 8 // CHECK: %[[CALL:.*]] = call{{.*}} @"_ZNK3$_0clEv" -// CHECK: %[[COERCEDIVE:.*]] = getelementptr{{.*}} %[[COERCE]] +// CHECK: %[[RETVALDIVE:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[COERCEVALIP:.*]] = inttoptr{{.*}} %[[CALL]] -// CHECK: call {{.*}}memcpy{{.*}} %[[RETVAL]]{{.*}} %[[COERCE]] // CHECK: %[[COERCEDIVE1:.*]] = getelementptr{{.*}} %[[RETVAL]] // CHECK: %[[TMP:.*]] = load{{.*}} %[[COERCEDIVE1]] // CHECK: %[[COERCEVALPI:.*]] = ptrtoint{{.*}} %[[TMP]] diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index c8e6d28584623..0247877d2c0c7 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -375,6 +375,7 @@ set(files __flat_set/flat_set.h __flat_set/ra_iterator.h __flat_set/utils.h + __force_nonstandard_layout __format/buffer.h __format/concepts.h __format/container_adaptor.h diff --git a/libcxx/include/__config b/libcxx/include/__config index d940461c30234..ddc15e56234d9 100644 --- a/libcxx/include/__config +++ b/libcxx/include/__config @@ -1207,6 +1207,12 @@ typedef __char32_t char32_t; # define _LIBCPP_HAS_EXPLICIT_THIS_PARAMETER 0 # endif +# if __has_feature(pointer_field_protection) +# define _LIBCPP_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCPP_NO_PFP +# endif + #endif // __cplusplus #endif // _LIBCPP___CONFIG diff --git a/libcxx/include/__force_nonstandard_layout b/libcxx/include/__force_nonstandard_layout new file mode 100644 index 0000000000000..6cc748f073b2a --- /dev/null +++ b/libcxx/include/__force_nonstandard_layout @@ -0,0 +1,43 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___FORCE_NONSTANDARD_LAYOUT +#define _LIBCPP___FORCE_NONSTANDARD_LAYOUT + +#include <__config> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// Force a class to be non-standard layout by giving it two bases with the same +// type. This is useful when structure protection is enabled because structure +// protection cannot be applied to standard layout classes. We may use this in +// cases where the standard does not specify whether a standard library class is +// standard layout. See C++2a [class]p7: +// A class S is a standard-layout class if it: +// -- has at most one base class subobject of any given type +_LIBCPP_DIAGNOSTIC_PUSH +_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Winaccessible-base") +class __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout_base2 : __force_nonstandard_layout_base1 {}; +class __force_nonstandard_layout : __force_nonstandard_layout_base1, __force_nonstandard_layout_base2 {}; +_LIBCPP_DIAGNOSTIC_POP + +#if __has_feature(pointer_field_protection) +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT : __force_nonstandard_layout +#else +# define _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT +#endif + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___FORCE_NONSTANDARD_LAYOUT diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h index dc112ebfd0faa..2e2d93662461a 100644 --- a/libcxx/include/__functional/function.h +++ b/libcxx/include/__functional/function.h @@ -14,6 +14,7 @@ #include <__config> #include <__cstddef/nullptr_t.h> #include <__exception/exception.h> +#include <__force_nonstandard_layout> #include <__functional/binary_function.h> #include <__functional/invoke.h> #include <__functional/unary_function.h> @@ -427,7 +428,7 @@ template class __policy_func; template -class __policy_func<_Rp(_ArgTypes...)> { +class __policy_func<_Rp(_ArgTypes...)> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { // Inline storage for small objects. __policy_storage __buf_; diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h index 0cbd995105671..e51989874beb4 100644 --- a/libcxx/include/__memory/shared_ptr.h +++ b/libcxx/include/__memory/shared_ptr.h @@ -16,6 +16,7 @@ #include <__cstddef/nullptr_t.h> #include <__cstddef/ptrdiff_t.h> #include <__exception/exception.h> +#include <__force_nonstandard_layout> #include <__functional/binary_function.h> #include <__functional/operations.h> #include <__functional/reference_wrapper.h> @@ -304,7 +305,7 @@ using __shared_ptr_nullptr_deleter_ctor_reqs _LIBCPP_NODEBUG = #endif template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI shared_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI shared_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { struct __nullptr_sfinae_tag {}; public: @@ -1204,7 +1205,7 @@ inline _LIBCPP_HIDE_FROM_ABI _Dp* get_deleter(const shared_ptr<_Tp>& __p) _NOEXC #endif // _LIBCPP_HAS_RTTI template -class _LIBCPP_SHARED_PTR_TRIVIAL_ABI weak_ptr { +class _LIBCPP_SHARED_PTR_TRIVIAL_ABI weak_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: #if _LIBCPP_STD_VER >= 17 typedef remove_extent_t<_Tp> element_type; diff --git a/libcxx/include/__memory/unique_ptr.h b/libcxx/include/__memory/unique_ptr.h index eff24546cdc01..e895b23598f2d 100644 --- a/libcxx/include/__memory/unique_ptr.h +++ b/libcxx/include/__memory/unique_ptr.h @@ -17,6 +17,7 @@ #include <__config> #include <__cstddef/nullptr_t.h> #include <__cstddef/size_t.h> +#include <__force_nonstandard_layout> #include <__functional/hash.h> #include <__functional/operations.h> #include <__memory/allocator_traits.h> // __pointer @@ -127,7 +128,7 @@ struct __unique_ptr_deleter_sfinae<_Deleter&> { #endif template > -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI unique_ptr { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI unique_ptr _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; @@ -396,7 +397,7 @@ struct __unique_ptr_array_bounds_stored { }; template -class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI unique_ptr<_Tp[], _Dp> { +class _LIBCPP_UNIQUE_PTR_TRIVIAL_ABI unique_ptr<_Tp[], _Dp> _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: typedef _Tp element_type; typedef _Dp deleter_type; diff --git a/libcxx/include/__tree b/libcxx/include/__tree index 9d4ba3ad0639c..9c0db56fe1f92 100644 --- a/libcxx/include/__tree +++ b/libcxx/include/__tree @@ -13,6 +13,7 @@ #include <__algorithm/min.h> #include <__assert> #include <__config> +#include <__force_nonstandard_layout> #include <__fwd/map.h> #include <__fwd/pair.h> #include <__fwd/set.h> @@ -792,7 +793,7 @@ _LIBCPP_DIAGNOSE_WARNING(!__is_invocable_v<_Compare const&, _Tp const&, _Tp cons int __diagnose_non_const_comparator(); template -class __tree { +class __tree _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: using value_type = __get_node_value_type_t<_Tp>; typedef _Compare value_compare; diff --git a/libcxx/include/__type_traits/is_trivially_relocatable.h b/libcxx/include/__type_traits/is_trivially_relocatable.h index 9b0e240de55f4..dc8b0e23360fd 100644 --- a/libcxx/include/__type_traits/is_trivially_relocatable.h +++ b/libcxx/include/__type_traits/is_trivially_relocatable.h @@ -34,10 +34,13 @@ template struct __libcpp_is_trivially_relocatable : is_trivially_copyable<_Tp> {}; #endif +// __trivially_relocatable on libc++'s builtin types does not currently return the right answer with PFP. +#if !__has_feature(pointer_field_protection) template struct __libcpp_is_trivially_relocatable<_Tp, __enable_if_t::value> > : true_type {}; +#endif _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 4e0d76fbbe3de..840527eefc4db 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -21,6 +21,7 @@ #include <__assert> #include <__config> #include <__debug_utils/sanitizers.h> +#include <__force_nonstandard_layout> #include <__format/enable_insertable.h> #include <__fwd/vector.h> #include <__iterator/advance.h> @@ -85,7 +86,7 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD template */> -class vector { +class vector _LIBCPP_MAYBE_FORCE_NONSTANDARD_LAYOUT { public: // // Types diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index 45b9c72a05b82..420ff19248e6c 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -2368,6 +2368,9 @@ module std [system] { module undef_macros { textual header "__undef_macros" } + module force_nonstandard_layout { + header "__force_nonstandard_layout" + } // This module needs to appear after __tree to work around issues with modules in Objective-C++ mode. module coroutine { diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo index 24aaabf0a87df..a1ee4155bcb3d 100644 --- a/libcxx/include/typeinfo +++ b/libcxx/include/typeinfo @@ -300,7 +300,7 @@ class _LIBCPP_EXPORTED_FROM_ABI _LIBCPP_TYPE_INFO_VTABLE_POINTER_AUTH type_info protected: typedef __type_info_implementations::__impl __impl; - __impl::__type_name_t __type_name; + _LIBCPP_NO_PFP __impl::__type_name_t __type_name; _LIBCPP_HIDE_FROM_ABI explicit type_info(const char* __n) : __type_name(__impl::__string_to_type_name(__n)) {} diff --git a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp index f125cc9adc491..d84914999b8fa 100644 --- a/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp +++ b/libcxx/test/libcxx/gdb/gdb_pretty_printer_test.sh.cpp @@ -256,9 +256,12 @@ void unique_ptr_test() { ComparePrettyPrintToRegex(std::move(forty_two), R"(std::unique_ptr containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::unique_ptr this_is_null; ComparePrettyPrintToChars(std::move(this_is_null), R"(std::unique_ptr is nullptr)"); +#endif } void bitset_test() { @@ -476,10 +479,13 @@ void vector_test() { "std::vector of length " "3, capacity 3 = {5, 6, 7}"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::vector> test3({7, 8}); ComparePrettyPrintToChars(std::move(test3), "std::vector of length " "2, capacity 2 = {7, 8}"); +#endif } void set_iterator_test() { @@ -650,8 +656,11 @@ void shared_ptr_test() { test0, R"(std::shared_ptr count [3\?], weak [0\?]( \(libc\+\+ missing debug info\))? containing = {__ptr_ = 0x[a-f0-9]+})"); +#if !__has_feature(pointer_field_protection) + // GDB doesn't know how to read PFP fields correctly yet. std::shared_ptr test3; ComparePrettyPrintToChars(test3, "std::shared_ptr is nullptr"); +#endif } void streampos_test() { diff --git a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp index 8066925f2900a..3b9d7e1776a73 100644 --- a/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp +++ b/libcxx/test/libcxx/type_traits/is_trivially_relocatable.compile.pass.cpp @@ -28,6 +28,12 @@ # include #endif +#if __has_feature(pointer_field_protection) +constexpr bool pfp_disabled = false; +#else +constexpr bool pfp_disabled = true; +#endif + static_assert(std::__libcpp_is_trivially_relocatable::value, ""); static_assert(std::__libcpp_is_trivially_relocatable::value, ""); static_assert(std::__libcpp_is_trivially_relocatable::value, ""); @@ -70,8 +76,8 @@ static_assert(!std::__libcpp_is_trivially_relocatable::val // ---------------------- // __split_buffer -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); // standard library types @@ -84,7 +90,7 @@ static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable, 1> >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable, 1> >::value == pfp_disabled, ""); // basic_string #if !__has_feature(address_sanitizer) || !_LIBCPP_INSTRUMENTED_WITH_ASAN @@ -99,17 +105,17 @@ struct NotTriviallyRelocatableCharTraits : constexpr_char_traits { }; static_assert(std::__libcpp_is_trivially_relocatable< - std::basic_string, std::allocator > >::value, + std::basic_string, std::allocator > >::value == pfp_disabled, ""); static_assert(std::__libcpp_is_trivially_relocatable< - std::basic_string, std::allocator > >::value, + std::basic_string, std::allocator > >::value == pfp_disabled, ""); static_assert(std::__libcpp_is_trivially_relocatable< - std::basic_string, std::allocator > >::value, + std::basic_string, std::allocator > >::value == pfp_disabled, ""); static_assert( std::__libcpp_is_trivially_relocatable< - std::basic_string, std::allocator > >::value, + std::basic_string, std::allocator > >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable< std::basic_string, test_allocator > >::value, @@ -121,21 +127,21 @@ static_assert( #endif // deque -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); // exception_ptr #ifndef _LIBCPP_ABI_MICROSOFT // FIXME: Is this also the case on windows? -static_assert(std::__libcpp_is_trivially_relocatable::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable::value == pfp_disabled, ""); #endif // expected #if TEST_STD_VER >= 23 -static_assert(std::__libcpp_is_trivially_relocatable >::value); -static_assert(std::__libcpp_is_trivially_relocatable, int>>::value); -static_assert(std::__libcpp_is_trivially_relocatable>>::value); -static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr>>::value); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled); +static_assert(std::__libcpp_is_trivially_relocatable, int>>::value == pfp_disabled); +static_assert(std::__libcpp_is_trivially_relocatable>>::value == pfp_disabled); +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr>>::value == pfp_disabled); static_assert(!std::__libcpp_is_trivially_relocatable>::value); static_assert(!std::__libcpp_is_trivially_relocatable>::value); @@ -145,42 +151,42 @@ static_assert( // locale #ifndef TEST_HAS_NO_LOCALIZATION -static_assert(std::__libcpp_is_trivially_relocatable::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable::value == pfp_disabled, ""); #endif // optional #if TEST_STD_VER >= 17 static_assert(std::__libcpp_is_trivially_relocatable>::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable>::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable>>::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable>>::value == pfp_disabled, ""); #endif // TEST_STD_VER >= 17 // pair -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value == pfp_disabled, ""); // shared_ptr -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); // tuple #if TEST_STD_VER >= 11 static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable > >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable > >::value == pfp_disabled, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value == pfp_disabled, ""); #endif // TEST_STD_VER >= 11 @@ -205,9 +211,9 @@ struct NotTriviallyRelocatablePointer { void operator()(T*); }; -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, @@ -221,23 +227,23 @@ static_assert(!std::__libcpp_is_trivially_relocatable= 17 static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable > >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable > >::value == pfp_disabled, ""); static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); static_assert(!std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value, +static_assert(std::__libcpp_is_trivially_relocatable, std::unique_ptr > >::value == pfp_disabled, ""); #endif // TEST_STD_VER >= 17 // vector -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); static_assert(!std::__libcpp_is_trivially_relocatable > >::value, ""); // weak_ptr -static_assert(std::__libcpp_is_trivially_relocatable >::value, ""); +static_assert(std::__libcpp_is_trivially_relocatable >::value == pfp_disabled, ""); // TODO: Mark all the trivially relocatable STL types as such diff --git a/libcxxabi/include/__cxxabi_config.h b/libcxxabi/include/__cxxabi_config.h index 759445dac91f9..00e1357cdab85 100644 --- a/libcxxabi/include/__cxxabi_config.h +++ b/libcxxabi/include/__cxxabi_config.h @@ -109,4 +109,14 @@ # define _LIBCXXABI_NOEXCEPT noexcept #endif +#if defined(_LIBCXXABI_COMPILER_CLANG) +# if __has_feature(pointer_field_protection) +# define _LIBCXXABI_NO_PFP [[clang::no_field_protection]] +# else +# define _LIBCXXABI_NO_PFP +# endif +#else +# define _LIBCXXABI_NO_PFP +#endif + #endif // ____CXXABI_CONFIG_H diff --git a/libcxxabi/src/private_typeinfo.h b/libcxxabi/src/private_typeinfo.h index 328a02edef5c1..a3bc0bffd41bc 100644 --- a/libcxxabi/src/private_typeinfo.h +++ b/libcxxabi/src/private_typeinfo.h @@ -145,7 +145,7 @@ class _LIBCXXABI_TYPE_VIS __class_type_info : public __shim_type_info { // Has one non-virtual public base class at offset zero class _LIBCXXABI_TYPE_VIS __si_class_type_info : public __class_type_info { public: - const __class_type_info *__base_type; + _LIBCXXABI_NO_PFP const __class_type_info *__base_type; _LIBCXXABI_HIDDEN virtual ~__si_class_type_info(); @@ -204,7 +204,7 @@ class _LIBCXXABI_TYPE_VIS __vmi_class_type_info : public __class_type_info { class _LIBCXXABI_TYPE_VIS __pbase_type_info : public __shim_type_info { public: unsigned int __flags; - const __shim_type_info *__pointee; + _LIBCXXABI_NO_PFP const __shim_type_info *__pointee; enum __masks { __const_mask = 0x1, @@ -245,7 +245,7 @@ class _LIBCXXABI_TYPE_VIS __pointer_type_info : public __pbase_type_info { class _LIBCXXABI_TYPE_VIS __pointer_to_member_type_info : public __pbase_type_info { public: - const __class_type_info *__context; + _LIBCXXABI_NO_PFP const __class_type_info *__context; _LIBCXXABI_HIDDEN virtual ~__pointer_to_member_type_info(); _LIBCXXABI_HIDDEN virtual bool can_catch(const __shim_type_info *, diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h index 0858d8aee2186..1e1d39372fec5 100644 --- a/llvm/include/llvm/Analysis/PtrUseVisitor.h +++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h @@ -134,6 +134,7 @@ class PtrUseVisitorBase { UseAndIsOffsetKnownPair UseAndIsOffsetKnown; APInt Offset; + Value *ProtectedField; }; /// The worklist of to-visit uses. @@ -158,6 +159,8 @@ class PtrUseVisitorBase { /// The constant offset of the use if that is known. APInt Offset; + Value *ProtectedField; + /// @} /// Note that the constructor is protected because this class must be a base @@ -230,6 +233,7 @@ class PtrUseVisitor : protected InstVisitor, IntegerType *IntIdxTy = cast(DL.getIndexType(I.getType())); IsOffsetKnown = true; Offset = APInt(IntIdxTy->getBitWidth(), 0); + ProtectedField = nullptr; PI.reset(); // Enqueue the uses of this pointer. @@ -242,6 +246,7 @@ class PtrUseVisitor : protected InstVisitor, IsOffsetKnown = ToVisit.UseAndIsOffsetKnown.getInt(); if (IsOffsetKnown) Offset = std::move(ToVisit.Offset); + ProtectedField = ToVisit.ProtectedField; Instruction *I = cast(U->getUser()); static_cast(this)->visit(I); @@ -300,6 +305,16 @@ class PtrUseVisitor : protected InstVisitor, case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: return; // No-op intrinsics. + + case Intrinsic::protected_field_ptr: { + auto *OffsetConst = dyn_cast(II.getArgOperand(1)); + if (!IsOffsetKnown || !OffsetConst) + return Base::visitIntrinsicInst(II); + Offset += OffsetConst->getValue().sextOrTrunc(Offset.getBitWidth()); + ProtectedField = II.getArgOperand(2); + enqueueUsers(II); + break; + } } } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index bd6f94ac1286c..faec1946f8494 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2569,6 +2569,29 @@ def int_hwasan_check_memaccess_shortgranules_fixedshadow : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty, llvm_i64_ty], [ImmArg>, ImmArg>]>; +// Returns a pointer which may be used to store a pointer at the address +// (first argument + second argument) with protection against use-after-free +// vulnerabilities. Stores via the pointer will cause the stored pointer to be +// blended with some combination of: +// - the field identifier (third argument) +// - the struct address (first argument), if the struct is not trivially +// copyable (fourth argument is true) +// before being stored. When loading from the pointer, the inverse operation +// is done on the loaded pointer after it is loaded. Specifically, when +// targeting AArch64 with pointer authentication enabled and when the fourth +// argument is true, the pointer is signed using the struct address before +// being stored, and authenticated after being loaded. Otherwise it is rotated +// left by 16 bits and a hash of the field identifier is subtracted before +// being stored, and the hash is added and the pointer is rotated right by +// 16 bits after being loaded. If the pointer is used otherwise than for +// loading or storing (e.g. its address escapes), it will cause all blending +// operations using the same field identifier to be disabled throughout the +// program. This is done using special linker relocations. +def int_protected_field_ptr : + DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty, + llvm_metadata_ty, llvm_i1_ty], + [IntrNoMem, ImmArg>, ImmArg>]>; + // Xray intrinsics //===----------------------------------------------------------------------===// // Custom event logging for x-ray. diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h index df146458b4e6f..3bab0efe4fdf7 100644 --- a/llvm/include/llvm/Transforms/Utils/Local.h +++ b/llvm/include/llvm/Transforms/Utils/Local.h @@ -183,6 +183,8 @@ LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB); LLVM_ABI bool EliminateDuplicatePHINodes(BasicBlock *BB, SmallPtrSetImpl &ToRemove); +bool shouldFoldLoadStoreWithPointerOperandThroughPhi(const Value *Ptr); + /// This function is used to do simplification of a CFG. For example, it /// adjusts branches to branches to eliminate the extra hop, it eliminates /// unreachable basic blocks, and does other peephole optimization of the CFG. diff --git a/llvm/lib/Analysis/PtrUseVisitor.cpp b/llvm/lib/Analysis/PtrUseVisitor.cpp index 9c79546f491ef..b34f380b7db5c 100644 --- a/llvm/lib/Analysis/PtrUseVisitor.cpp +++ b/llvm/lib/Analysis/PtrUseVisitor.cpp @@ -22,7 +22,8 @@ void detail::PtrUseVisitorBase::enqueueUsers(Value &I) { if (VisitedUses.insert(&U).second) { UseToVisit NewU = { UseToVisit::UseAndIsOffsetKnownPair(&U, IsOffsetKnown), - Offset + Offset, + ProtectedField, }; Worklist.push_back(std::move(NewU)); } diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp index 265a32cf4d127..223af69d8e5c5 100644 --- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -21,9 +21,11 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/Function.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" @@ -36,6 +38,8 @@ #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" #include "llvm/Transforms/Utils/LowerVectorIntrinsics.h" +#include + using namespace llvm; /// Threshold to leave statically sized memory intrinsic calls. Calls of known @@ -455,6 +459,254 @@ bool PreISelIntrinsicLowering::expandMemIntrinsicUses( return Changed; } +namespace { + +enum class PointerEncoding { + Rotate, + PACCopyable, + PACNonCopyable, +}; + +bool expandProtectedFieldPtr(Function &Intr) { + Module &M = *Intr.getParent(); + bool IsAArch64 = Triple(M.getTargetTriple()).isAArch64(); + + std::set NonPFPFields; + std::set LoadsStores; + + Type *Int8Ty = Type::getInt8Ty(M.getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); + PointerType *PtrTy = PointerType::get(M.getContext(), 0); + + Function *SignIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_sign, {}); + Function *AuthIntr = + Intrinsic::getOrInsertDeclaration(&M, Intrinsic::ptrauth_auth, {}); + + auto *EmuFnTy = FunctionType::get(Int64Ty, {Int64Ty, Int64Ty}, false); + FunctionCallee EmuSignIntr = M.getOrInsertFunction("__emupac_pacda", EmuFnTy); + FunctionCallee EmuAuthIntr = M.getOrInsertFunction("__emupac_autda", EmuFnTy); + + auto CreateSign = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(SignIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuSignIntr, {Val, Disc}, DSBundle); + }; + + auto CreateAuth = [&](IRBuilder<> &B, Value *Val, Value *Disc, + OperandBundleDef DSBundle) { + Function *F = B.GetInsertBlock()->getParent(); + Attribute FSAttr = F->getFnAttribute("target-features"); + if (FSAttr.isValid() && FSAttr.getValueAsString().contains("+pauth")) + return B.CreateCall(AuthIntr, {Val, B.getInt32(2), Disc}, DSBundle); + return B.CreateCall(EmuAuthIntr, {Val, Disc}, DSBundle); + }; + + for (User *U : Intr.users()) { + auto *Call = cast(U); + auto *FieldName = cast( + cast(Call->getArgOperand(2))->getMetadata()); + std::set VisitedPhis; + + std::function FindLoadsStores; + FindLoadsStores = [&](Instruction *I) { + for (Use &U : I->uses()) { + if (auto *LI = dyn_cast(U.getUser())) { + if (isa(LI->getType())) { + LoadsStores.insert(LI); + continue; + } + } + if (auto *SI = dyn_cast(U.getUser())) { + if (U.getOperandNo() == 1 && + isa(SI->getValueOperand()->getType())) { + LoadsStores.insert(SI); + continue; + } + } + if (auto *P = dyn_cast(U.getUser())) { + if (VisitedPhis.insert(P).second) + FindLoadsStores(P); + continue; + } + // Comparisons against null cannot be used to recover the original + // pointer so we allow them. + if (auto *CI = dyn_cast(U.getUser())) { + if (auto *Op = dyn_cast(CI->getOperand(0))) + if (Op->isNullValue()) + continue; + if (auto *Op = dyn_cast(CI->getOperand(1))) + if (Op->isNullValue()) + continue; + } + NonPFPFields.insert(FieldName); + } + }; + + FindLoadsStores(Call); + } + + for (Instruction *I : LoadsStores) { + std::set Offsets; + std::set Fields; + std::set VisitedPhis; + bool IsNonTriviallyCopyable = false; + + std::function FindFields; + FindFields = [&](Value *V) { + if (auto *Call = dyn_cast(V)) { + if (Call->getCalledOperand() == &Intr) { + Offsets.insert(Call->getArgOperand(1)); + auto *Field = cast( + cast(Call->getArgOperand(2))->getMetadata()); + Fields.insert(Field); + if (cast(Call->getArgOperand(3))->getZExtValue()) + IsNonTriviallyCopyable = true; + return; + } + } + if (auto *P = dyn_cast(V)) { + if (VisitedPhis.insert(P).second) + for (Value *V : P->incoming_values()) + FindFields(V); + return; + } + Fields.insert(nullptr); + }; + FindFields(isa(I) ? cast(I)->getPointerOperand() + : cast(I)->getPointerOperand()); + if (Fields.size() != 1 || Offsets.size() != 1) { + for (Metadata *Field : Fields) + if (Field) + NonPFPFields.insert(Field); + continue; + } + + std::string FieldName = cast(*Fields.begin())->getString().str(); + uint64_t FieldSignature = std::hash()(FieldName); + + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *DS = M.getNamedValue(DSName); + if (!DS) { + DS = new GlobalVariable(M, Int8Ty, false, + GlobalVariable::ExternalWeakLinkage, nullptr, + DSName); + DS->setVisibility(GlobalValue::HiddenVisibility); + } + OperandBundleDef DSBundle("deactivation-symbol", DS); + + PointerEncoding Encoding; + if (!IsAArch64) + Encoding = PointerEncoding::Rotate; + else if (IsNonTriviallyCopyable) + Encoding = PointerEncoding::PACNonCopyable; + else + Encoding = PointerEncoding::PACCopyable; + + if (auto *LI = dyn_cast(I)) { + auto *FieldAddr = LI->getPointerOperand(); + IRBuilder<> B(LI->getNextNode()); + auto *LIInt = cast(B.CreatePtrToInt(LI, B.getInt64Ty())); + Value *Auth; + switch (Encoding) { + case PointerEncoding::Rotate: + Auth = B.CreateAdd(LIInt, B.getInt64(FieldSignature & 0xff)); + Auth = B.CreateOr(B.CreateLShr(Auth, 16), B.CreateShl(Auth, 48)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Auth = CreateAuth(B, LIInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Auth = + CreateAuth(B, LIInt, B.getInt64(FieldSignature & 0xffff), DSBundle); + break; + } + LI->replaceAllUsesWith(B.CreateIntToPtr(Auth, B.getPtrTy())); + LIInt->setOperand(0, LI); + } else if (auto *SI = dyn_cast(I)) { + IRBuilder<> B(SI); + auto *FieldAddr = SI->getPointerOperand(); + auto *SIValInt = + B.CreatePtrToInt(SI->getValueOperand(), B.getInt64Ty()); + Value *Sign; + switch (Encoding) { + case PointerEncoding::Rotate: + Sign = + B.CreateOr(B.CreateLShr(SIValInt, 48), B.CreateShl(SIValInt, 16)); + Sign = B.CreateSub(Sign, B.getInt64(FieldSignature & 0xff)); + break; + case PointerEncoding::PACNonCopyable: { + Value *Struct; + if (auto *Call = dyn_cast(FieldAddr)) + Struct = Call->getArgOperand(0); + else if (cast(*Offsets.begin())->getZExtValue() == 0) + Struct = FieldAddr; + else + Struct = B.CreateGEP(B.getInt8Ty(), FieldAddr, + {B.CreateNeg(*Offsets.begin())}); + auto *StructInt = B.CreatePtrToInt(Struct, B.getInt64Ty()); + Sign = CreateSign(B, SIValInt, StructInt, DSBundle); + break; + } + case PointerEncoding::PACCopyable: + Sign = CreateSign(B, SIValInt, B.getInt64(FieldSignature & 0xffff), + DSBundle); + break; + } + SI->setOperand(0, B.CreateIntToPtr(Sign, B.getPtrTy())); + } + } + + for (User *U : llvm::make_early_inc_range(Intr.users())) { + auto *Call = cast(U); + auto *Struct = Call->getArgOperand(0); + auto *Offset = Call->getArgOperand(1); + + IRBuilder<> B(Call); + if (cast(Offset)->getZExtValue() == 0) + Call->replaceAllUsesWith(Struct); + else + Call->replaceAllUsesWith(B.CreateGEP(B.getInt8Ty(), Struct, {Offset})); + Call->eraseFromParent(); + } + + if (!NonPFPFields.empty()) { + Constant *Nop = + ConstantExpr::getIntToPtr(ConstantInt::get(Int64Ty, 0xd503201f), PtrTy); + std::set LocalNonPFPFieldNames; + for (auto *Field : NonPFPFields) + LocalNonPFPFieldNames.insert(cast(Field)->getString().str()); + for (auto &FieldName : LocalNonPFPFieldNames) { + std::string DSName = "__pfp_ds_" + FieldName; + GlobalValue *OldDS = M.getNamedValue(DSName); + GlobalValue *DS = GlobalAlias::create( + Int8Ty, 0, GlobalValue::ExternalLinkage, DSName, Nop, &M); + DS->setVisibility(GlobalValue::HiddenVisibility); + if (OldDS) { + DS->takeName(OldDS); + OldDS->replaceAllUsesWith(DS); + OldDS->eraseFromParent(); + } + } + } + return true; +} + +} + bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { // Map unique constants to globals. DenseMap CMap; @@ -591,6 +843,9 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const { return lowerUnaryVectorIntrinsicAsLoop(M, CI); }); break; + case Intrinsic::protected_field_ptr: + Changed |= expandProtectedFieldPtr(F); + break; } } return Changed; diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp index 6477141ab095f..f156883d9dcee 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp @@ -697,8 +697,7 @@ static bool isSafeAndProfitableToSinkLoad(LoadInst *L) { Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LoadInst *FirstLI = cast(PN.getIncomingValue(0)); - // Can't forward swifterror through a phi. - if (FirstLI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(FirstLI->getOperand(0))) return nullptr; // FIXME: This is overconservative; this transform is allowed in some cases @@ -737,8 +736,7 @@ Instruction *InstCombinerImpl::foldPHIArgLoadIntoPHI(PHINode &PN) { LI->getPointerAddressSpace() != LoadAddrSpace) return nullptr; - // Can't forward swifterror through a phi. - if (LI->getOperand(0)->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(LI->getOperand(0))) return nullptr; // We can't sink the load if the loaded value could be modified between diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 70b4552190a4e..774b5a4ada863 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -534,9 +534,9 @@ class Slice { public: Slice() = default; - Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) + Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable, Value *ProtectedField) : BeginOffset(BeginOffset), EndOffset(EndOffset), - UseAndIsSplittable(U, IsSplittable) {} + UseAndIsSplittable(U, IsSplittable), ProtectedField(ProtectedField) {} uint64_t beginOffset() const { return BeginOffset; } uint64_t endOffset() const { return EndOffset; } @@ -549,6 +549,8 @@ class Slice { bool isDead() const { return getUse() == nullptr; } void kill() { UseAndIsSplittable.setPointer(nullptr); } + Value *ProtectedField; + /// Support for ordering ranges. /// /// This provides an ordering over ranges such that start offsets are @@ -1075,7 +1077,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor { EndOffset = AllocSize; } - AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); + AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable, ProtectedField)); } void visitBitCastInst(BitCastInst &BC) { @@ -4693,7 +4695,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PLoad << "\n"); @@ -4849,10 +4851,12 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { LLVMContext::MD_access_group}); // Now build a new slice for the alloca. + // ProtectedField==nullptr is a lie, but it doesn't matter because we + // already determined that all accesses are consistent. NewSlices.push_back( Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, &PStore->getOperandUse(PStore->getPointerOperandIndex()), - /*IsSplittable*/ false)); + /*IsSplittable*/ false, nullptr)); LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset() << ", " << NewSlices.back().endOffset() << "): " << *PStore << "\n"); @@ -5715,6 +5719,32 @@ SROA::runOnAlloca(AllocaInst &AI) { return {Changed, CFGChanged}; } + for (auto &P : AS.partitions()) { + std::optional ProtectedField; + // For now, we can't split if a field is accessed both via protected + // field and not. + for (Slice &S : P) { + if (auto *II = dyn_cast(S.getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S.ProtectedField; + if (*ProtectedField != S.ProtectedField) + return {Changed, CFGChanged}; + } + for (Slice *S : P.splitSliceTails()) { + if (auto *II = dyn_cast(S->getUse()->getUser())) + if (II->getIntrinsicID() == Intrinsic::lifetime_start || + II->getIntrinsicID() == Intrinsic::lifetime_end) + continue; + if (!ProtectedField) + ProtectedField = S->ProtectedField; + if (*ProtectedField != S->ProtectedField) + return {Changed, CFGChanged}; + } + } + // Delete all the dead users of this alloca before splitting and rewriting it. for (Instruction *DeadUser : AS.getDeadUsers()) { // Free up everything used by this instruction. diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp index f5208d50c6aae..d5e2252de5e93 100644 --- a/llvm/lib/Transforms/Utils/Local.cpp +++ b/llvm/lib/Transforms/Utils/Local.cpp @@ -4212,10 +4212,7 @@ bool llvm::canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx) { if (Op->getType()->isMetadataTy()) return false; - // swifterror pointers can only be used by a load, store, or as a swifterror - // argument; swifterror pointers are not allowed to be used in select or phi - // instructions. - if (Op->isSwiftError()) + if (!shouldFoldLoadStoreWithPointerOperandThroughPhi(Op)) return false; // Early exit. diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index a75f29000ca18..2601a58bb8b24 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -2129,6 +2129,22 @@ static bool replacingOperandWithVariableIsCheap(const Instruction *I, return !isa(I); } +bool llvm::shouldFoldLoadStoreWithPointerOperandThroughPhi(const Value *Ptr) { + // swifterror pointers can only be used by a load or store; sinking a load + // or store would require introducing a select for the pointer operand, + // which isn't allowed for swifterror pointers. + if (Ptr->isSwiftError()) + return false; + + // Protected pointer field loads/stores should be paired with the intrinsic + // to avoid unnecessary address escapes. + if (auto *II = dyn_cast(Ptr)) + if (II->getIntrinsicID() == Intrinsic::protected_field_ptr) + return false; + + return true; +} + // All instructions in Insts belong to different blocks that all unconditionally // branch to a common successor. Analyze each instruction and return true if it // would be possible to sink them into their successor, creating one common diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn index 6051674a790e8..de1d1b1f7f3ca 100644 --- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn +++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn @@ -1032,6 +1032,7 @@ if (current_toolchain == default_toolchain) { "__flat_set/flat_set.h", "__flat_set/ra_iterator.h", "__flat_set/utils.h", + "__force_nonstandard_layout", "__format/buffer.h", "__format/concepts.h", "__format/container_adaptor.h",