diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h index 1ae463f845c81..4bdeb51873787 100644 --- a/compiler-rt/lib/hwasan/hwasan.h +++ b/compiler-rt/lib/hwasan/hwasan.h @@ -136,6 +136,21 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, StackTrace *stack); void hwasan_free(void *ptr, StackTrace *stack); +void *hwasan_new(uptr size, StackTrace *stack); +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack); +void *hwasan_new_array(uptr size, StackTrace *stack); +void *hwasan_new_array_aligned(uptr size, uptr alignment, StackTrace *stack); +void hwasan_delete(void *ptr, StackTrace *stack); +void hwasan_delete_aligned(void *ptr, uptr alignment, StackTrace *stack); +void hwasan_delete_sized(void *ptr, uptr size, StackTrace *stack); +void hwasan_delete_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack); +void hwasan_delete_array(void *ptr, StackTrace *stack); +void hwasan_delete_array_aligned(void *ptr, uptr alignment, StackTrace *stack); +void hwasan_delete_array_sized(void *ptr, uptr size, StackTrace *stack); +void hwasan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack); + void InstallAtExitHandler(); #define GET_MALLOC_STACK_TRACE \ diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.cpp b/compiler-rt/lib/hwasan/hwasan_allocator.cpp index 75dbb336e3445..ec47be27f0e7e 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.cpp +++ b/compiler-rt/lib/hwasan/hwasan_allocator.cpp @@ -128,6 +128,12 @@ inline __lsan::ChunkTag Metadata::GetLsanTag() const { return static_cast<__lsan::ChunkTag>(lsan_tag); } +inline void Metadata::SetAllocType(AllocType type) { alloc_type = type; } + +inline AllocType Metadata::GetAllocType() const { + return static_cast(alloc_type); +} + uptr GetAliasRegionStart() { #if defined(HWASAN_ALIASING_MODE) constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); @@ -181,7 +187,7 @@ static uptr TaggedSize(uptr size) { } static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, - bool zeroise) { + AllocType alloc_type, bool zeroise) { // Keep this consistent with LSAN and ASAN behavior. if (UNLIKELY(orig_size == 0)) orig_size = 1; @@ -259,6 +265,7 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored : __lsan::kDirectlyLeaked); #endif + meta->SetAllocType(alloc_type); meta->SetAllocated(StackDepotPut(*stack), orig_size); RunMallocHooks(user_ptr, orig_size); return user_ptr; @@ -285,7 +292,7 @@ static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr, return false; } -static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { +static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr, AllocType) { CHECK(tagged_ptr); void *untagged_ptr = UntagPtr(tagged_ptr); @@ -379,15 +386,15 @@ static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old, void *untagged_ptr_old = UntagPtr(tagged_ptr_old); if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old)) return nullptr; - void *tagged_ptr_new = - HwasanAllocate(stack, new_size, alignment, false /*zeroise*/); + void *tagged_ptr_new = HwasanAllocate(stack, new_size, alignment, FROM_MALLOC, + false /*zeroise*/); if (tagged_ptr_old && tagged_ptr_new) { Metadata *meta = reinterpret_cast(allocator.GetMetaData(untagged_ptr_old)); void *untagged_ptr_new = UntagPtr(tagged_ptr_new); internal_memcpy(untagged_ptr_new, untagged_ptr_old, Min(new_size, static_cast(meta->GetRequestedSize()))); - HwasanDeallocate(stack, tagged_ptr_old); + HwasanDeallocate(stack, tagged_ptr_old, FROM_MALLOC); } return tagged_ptr_new; } @@ -398,7 +405,7 @@ static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) { return nullptr; ReportCallocOverflow(nmemb, size, stack); } - return HwasanAllocate(stack, nmemb * size, sizeof(u64), true); + return HwasanAllocate(stack, nmemb * size, sizeof(u64), FROM_MALLOC, true); } HwasanChunkView FindHeapChunkByAddress(uptr address) { @@ -449,7 +456,8 @@ static uptr AllocationSizeFast(const void *p) { } void *hwasan_malloc(uptr size, StackTrace *stack) { - return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, sizeof(u64), FROM_MALLOC, false)); } void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { @@ -458,9 +466,10 @@ void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) { if (!ptr) - return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, sizeof(u64), FROM_MALLOC, false)); if (size == 0) { - HwasanDeallocate(stack, ptr); + HwasanDeallocate(stack, ptr, FROM_MALLOC); return nullptr; } return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64))); @@ -478,7 +487,7 @@ void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) { void *hwasan_valloc(uptr size, StackTrace *stack) { return SetErrnoOnNull( - HwasanAllocate(stack, size, GetPageSizeCached(), false)); + HwasanAllocate(stack, size, GetPageSizeCached(), FROM_MALLOC, false)); } void *hwasan_pvalloc(uptr size, StackTrace *stack) { @@ -491,7 +500,8 @@ void *hwasan_pvalloc(uptr size, StackTrace *stack) { } // pvalloc(0) should allocate one page. size = size ? RoundUpTo(size, PageSize) : PageSize; - return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, PageSize, FROM_MALLOC, false)); } void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { @@ -501,7 +511,8 @@ void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) { return nullptr; ReportInvalidAlignedAllocAlignment(size, alignment, stack); } - return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, alignment, FROM_MALLOC, false)); } void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { @@ -511,7 +522,8 @@ void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) { return nullptr; ReportInvalidAllocationAlignment(alignment, stack); } - return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false)); + return SetErrnoOnNull( + HwasanAllocate(stack, size, alignment, FROM_MALLOC, false)); } int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, @@ -521,7 +533,7 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, return errno_EINVAL; ReportInvalidPosixMemalignAlignment(alignment, stack); } - void *ptr = HwasanAllocate(stack, size, alignment, false); + void *ptr = HwasanAllocate(stack, size, alignment, FROM_MALLOC, false); if (UNLIKELY(!ptr)) // OOM error is already taken care of by HwasanAllocate. return errno_ENOMEM; @@ -531,7 +543,95 @@ int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size, } void hwasan_free(void *ptr, StackTrace *stack) { - return HwasanDeallocate(stack, ptr); + HwasanDeallocate(stack, ptr, FROM_MALLOC); +} + +namespace { + +void *hwasan_new(uptr size, StackTrace *stack, bool array) { + return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), + array ? FROM_NEW_BR : FROM_NEW, true)); +} + +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack, + bool array) { + if (UNLIKELY(alignment == 0 || !IsPowerOfTwo(alignment))) { + errno = errno_EINVAL; + if (AllocatorMayReturnNull()) + return nullptr; + ReportInvalidAllocationAlignment(alignment, stack); + } + return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, + array ? FROM_NEW_BR : FROM_NEW, false)); +} + +void hwasan_delete(void *ptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_aligned(void *ptr, uptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_sized(void *ptr, uptr, StackTrace *stack, bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +void hwasan_delete_sized_aligned(void *ptr, uptr, uptr, StackTrace *stack, + bool array) { + HwasanDeallocate(stack, ptr, array ? FROM_NEW_BR : FROM_NEW); +} + +} // namespace + +void *hwasan_new(uptr size, StackTrace *stack) { + return hwasan_new(size, stack, /*array=*/false); +} + +void *hwasan_new_aligned(uptr size, uptr alignment, StackTrace *stack) { + return hwasan_new_aligned(size, alignment, stack, /*array=*/false); +} + +void *hwasan_new_array(uptr size, StackTrace *stack) { + return hwasan_new(size, stack, /*array=*/true); +} + +void *hwasan_new_array_aligned(uptr size, uptr alignment, StackTrace *stack) { + return hwasan_new_aligned(size, alignment, stack, /*array=*/true); +} + +void hwasan_delete(void *ptr, StackTrace *stack) { + hwasan_delete(ptr, stack, /*array=*/false); +} + +void hwasan_delete_aligned(void *ptr, uptr alignment, StackTrace *stack) { + hwasan_delete_aligned(ptr, alignment, stack, /*array=*/false); +} + +void hwasan_delete_sized(void *ptr, uptr size, StackTrace *stack) { + hwasan_delete_sized(ptr, size, stack, /*array=*/false); +} + +void hwasan_delete_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack) { + hwasan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/false); +} + +void hwasan_delete_array(void *ptr, StackTrace *stack) { + hwasan_delete(ptr, stack, /*array=*/true); +} + +void hwasan_delete_array_aligned(void *ptr, uptr alignment, StackTrace *stack) { + hwasan_delete_aligned(ptr, alignment, stack, /*array=*/true); +} + +void hwasan_delete_array_sized(void *ptr, uptr size, StackTrace *stack) { + hwasan_delete_sized(ptr, size, stack, /*array=*/true); +} + +void hwasan_delete_array_sized_aligned(void *ptr, uptr size, uptr alignment, + StackTrace *stack) { + hwasan_delete_sized_aligned(ptr, size, alignment, stack, /*array=*/true); } } // namespace __hwasan diff --git a/compiler-rt/lib/hwasan/hwasan_allocator.h b/compiler-rt/lib/hwasan/hwasan_allocator.h index 2ada2a0b1851a..6c28ffed591e6 100644 --- a/compiler-rt/lib/hwasan/hwasan_allocator.h +++ b/compiler-rt/lib/hwasan/hwasan_allocator.h @@ -31,13 +31,20 @@ namespace __hwasan { +enum AllocType { + FROM_MALLOC = 1, // Memory block came from malloc, calloc, realloc, etc. + FROM_NEW = 2, // Memory block came from operator new. + FROM_NEW_BR = 3 // Memory block came from operator new [ ] +}; + struct Metadata { private: atomic_uint64_t alloc_context_id; u32 requested_size_low; u16 requested_size_high; atomic_uint8_t chunk_state; - u8 lsan_tag; + u8 lsan_tag : 2; + u8 alloc_type : 2; public: inline void SetAllocated(u32 stack, u64 size); @@ -49,6 +56,8 @@ struct Metadata { inline u32 GetAllocThreadId() const; inline void SetLsanTag(__lsan::ChunkTag tag); inline __lsan::ChunkTag GetLsanTag() const; + inline void SetAllocType(AllocType type); + inline AllocType GetAllocType() const; }; static_assert(sizeof(Metadata) == 16); diff --git a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp index 232eb0eb6da67..62226216e73d1 100644 --- a/compiler-rt/lib/hwasan/hwasan_new_delete.cpp +++ b/compiler-rt/lib/hwasan/hwasan_new_delete.cpp @@ -22,75 +22,77 @@ #if HWASAN_REPLACE_OPERATORS_NEW_AND_DELETE // TODO(alekseys): throw std::bad_alloc instead of dying on OOM. -# define OPERATOR_NEW_BODY \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_malloc(size, &stack); \ - if (UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ +# define OPERATOR_NEW_BODY \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_new(size, &stack); \ + if (UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res # define OPERATOR_NEW_BODY_NOTHROW \ GET_MALLOC_STACK_TRACE; \ - return hwasan_malloc(size, &stack) -# define OPERATOR_NEW_BODY_ARRAY \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_malloc(size, &stack); \ - if (UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ + return hwasan_new(size, &stack) +# define OPERATOR_NEW_BODY_ARRAY \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_new_array(size, &stack); \ + if (UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res # define OPERATOR_NEW_BODY_ARRAY_NOTHROW \ GET_MALLOC_STACK_TRACE; \ - return hwasan_malloc(size, &stack) -# define OPERATOR_NEW_BODY_ALIGN \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_memalign(static_cast(align), size, &stack); \ - if (UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ + return hwasan_new_array(size, &stack) +# define OPERATOR_NEW_BODY_ALIGN \ + GET_MALLOC_STACK_TRACE; \ + void *res = hwasan_new_aligned(size, static_cast(align), &stack); \ + if (UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res # define OPERATOR_NEW_BODY_ALIGN_NOTHROW \ GET_MALLOC_STACK_TRACE; \ - return hwasan_memalign(static_cast(align), size, &stack) -# define OPERATOR_NEW_BODY_ALIGN_ARRAY \ - GET_MALLOC_STACK_TRACE; \ - void *res = hwasan_memalign(static_cast(align), size, &stack); \ - if (UNLIKELY(!res)) \ - ReportOutOfMemory(size, &stack); \ + return hwasan_new_aligned(size, static_cast(align), &stack) +# define OPERATOR_NEW_BODY_ALIGN_ARRAY \ + GET_MALLOC_STACK_TRACE; \ + void *res = \ + hwasan_new_array_aligned(size, static_cast(align), &stack); \ + if (UNLIKELY(!res)) \ + ReportOutOfMemory(size, &stack); \ return res # define OPERATOR_NEW_BODY_ALIGN_ARRAY_NOTHROW \ GET_MALLOC_STACK_TRACE; \ - return hwasan_memalign(static_cast(align), size, &stack) + return hwasan_new_array_aligned(size, static_cast(align), &stack) # define OPERATOR_DELETE_BODY \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete(ptr, &stack) # define OPERATOR_DELETE_BODY_ARRAY \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_array(ptr, &stack) # define OPERATOR_DELETE_BODY_ALIGN \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_aligned(ptr, static_cast(align), &stack) # define OPERATOR_DELETE_BODY_ALIGN_ARRAY \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_array_aligned(ptr, static_cast(align), &stack) # define OPERATOR_DELETE_BODY_SIZE \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_sized(ptr, size, &stack) # define OPERATOR_DELETE_BODY_SIZE_ARRAY \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_array_sized(ptr, size, &stack) # define OPERATOR_DELETE_BODY_SIZE_ALIGN \ GET_MALLOC_STACK_TRACE; \ if (ptr) \ - hwasan_free(ptr, &stack) -# define OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY \ - GET_MALLOC_STACK_TRACE; \ - if (ptr) \ - hwasan_free(ptr, &stack) + hwasan_delete_sized_aligned(ptr, size, static_cast(align), &stack) +# define OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY \ + GET_MALLOC_STACK_TRACE; \ + if (ptr) \ + hwasan_delete_array_sized_aligned(ptr, size, static_cast(align), \ + &stack) #elif defined(__ANDROID__) @@ -151,11 +153,11 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( OPERATOR_DELETE_BODY_ARRAY; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t) NOEXCEPT { + void *ptr, size_t size) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t) NOEXCEPT { + void *ptr, size_t size) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ARRAY; } @@ -189,31 +191,33 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( OPERATOR_DELETE_BODY_ALIGN; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, std::align_val_t) NOEXCEPT { + void *ptr, std::align_val_t align) NOEXCEPT { OPERATOR_DELETE_BODY_ALIGN_ARRAY; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { + void *ptr, std::align_val_t align, std::nothrow_t const &) NOEXCEPT { OPERATOR_DELETE_BODY_ALIGN; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, std::align_val_t, std::nothrow_t const &) NOEXCEPT { + void *ptr, std::align_val_t align, std::nothrow_t const &) NOEXCEPT { OPERATOR_DELETE_BODY_ALIGN_ARRAY; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t, std::align_val_t) NOEXCEPT { + void *ptr, size_t size, std::align_val_t align) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ALIGN; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t, std::align_val_t) NOEXCEPT { + void *ptr, size_t size, std::align_val_t align) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( - void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT { + void *ptr, size_t size, std::align_val_t align, + std::nothrow_t const &) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ALIGN; } INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( - void *ptr, size_t, std::align_val_t, std::nothrow_t const &) NOEXCEPT { + void *ptr, size_t size, std::align_val_t align, + std::nothrow_t const &) NOEXCEPT { OPERATOR_DELETE_BODY_SIZE_ALIGN_ARRAY; }