Skip to content

Commit 4817a6d

Browse files
authored
Allocate empty group on heap only for std::allocator (#254)
* Allocate empty group on heap only for `std::allocator` * Add missing assert,
1 parent feb600c commit 4817a6d

File tree

2 files changed

+51
-15
lines changed

2 files changed

+51
-15
lines changed

parallel_hashmap/phmap.h

Lines changed: 50 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -329,11 +329,16 @@ static_assert(kDeleted == -2,
329329
// A single block of empty control bytes for tables without any slots allocated.
330330
// This enables removing a branch in the hot path of find().
331331
// --------------------------------------------------------------------------
332+
template <class std_alloc_t>
332333
inline ctrl_t* EmptyGroup() {
333-
alignas(16) static constexpr ctrl_t empty_group[] = {
334-
kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
335-
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
336-
return const_cast<ctrl_t*>(empty_group);
334+
PHMAP_IF_CONSTEXPR (std_alloc_t::value) {
335+
alignas(16) static constexpr ctrl_t empty_group[] = {
336+
kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty,
337+
kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty};
338+
339+
return const_cast<ctrl_t*>(empty_group);
340+
}
341+
return nullptr;
337342
}
338343

339344
// --------------------------------------------------------------------------
@@ -869,6 +874,8 @@ class raw_hash_set
869874
template <class K>
870875
using key_arg = typename KeyArgImpl::template type<K, key_type>;
871876

877+
using std_alloc_t = std::is_same<typename std::decay<Alloc>::type, phmap::priv::Allocator<value_type>>;
878+
872879
private:
873880
// Give an early error when key_type is not hashable/eq.
874881
auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k));
@@ -989,6 +996,11 @@ class raw_hash_set
989996
iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) {}
990997

991998
void skip_empty_or_deleted() {
999+
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
1000+
// ctrl_ could be nullptr
1001+
if (!ctrl_)
1002+
return;
1003+
}
9921004
while (IsEmptyOrDeleted(*ctrl_)) {
9931005
// ctrl is not necessarily aligned to Group::kWidth. It is also likely
9941006
// to read past the space for ctrl bytes and into slots. This is ok
@@ -1057,7 +1069,7 @@ class raw_hash_set
10571069
explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(),
10581070
const key_equal& eq = key_equal(),
10591071
const allocator_type& alloc = allocator_type())
1060-
: ctrl_(EmptyGroup()), settings_(0, hashfn, eq, alloc) {
1072+
: ctrl_(EmptyGroup<std_alloc_t>()), settings_(0, hashfn, eq, alloc) {
10611073
if (bucket_cnt) {
10621074
size_t new_capacity = NormalizeCapacity(bucket_cnt);
10631075
reset_growth_left(new_capacity);
@@ -1180,7 +1192,7 @@ class raw_hash_set
11801192
std::is_nothrow_copy_constructible<hasher>::value&&
11811193
std::is_nothrow_copy_constructible<key_equal>::value&&
11821194
std::is_nothrow_copy_constructible<allocator_type>::value)
1183-
: ctrl_(phmap::exchange(that.ctrl_, EmptyGroup())),
1195+
: ctrl_(phmap::exchange(that.ctrl_, EmptyGroup<std_alloc_t>())),
11841196
slots_(phmap::exchange(that.slots_, nullptr)),
11851197
size_(phmap::exchange(that.size_, 0)),
11861198
capacity_(phmap::exchange(that.capacity_, 0)),
@@ -1194,7 +1206,7 @@ class raw_hash_set
11941206
}
11951207

11961208
raw_hash_set(raw_hash_set&& that, const allocator_type& a)
1197-
: ctrl_(EmptyGroup()),
1209+
: ctrl_(EmptyGroup<std_alloc_t>()),
11981210
slots_(nullptr),
11991211
size_(0),
12001212
capacity_(0),
@@ -1615,6 +1627,7 @@ class raw_hash_set
16151627
// This overload is necessary because otherwise erase<K>(const K&) would be
16161628
// a better match if non-const iterator is passed as an argument.
16171629
iterator erase(iterator it) {
1630+
assert(it != end());
16181631
auto res = it;
16191632
++res;
16201633
_erase(it);
@@ -1738,7 +1751,8 @@ class raw_hash_set
17381751

17391752
template <class K = key_type>
17401753
void prefetch(const key_arg<K>& key) const {
1741-
prefetch_hash(this->hash(key));
1754+
PHMAP_IF_CONSTEXPR (std_alloc_t::value)
1755+
prefetch_hash(this->hash(key));
17421756
}
17431757

17441758
// The API of find() has two extensions.
@@ -1848,6 +1862,11 @@ class raw_hash_set
18481862

18491863
template <class K = key_type>
18501864
bool find_impl(const key_arg<K>& key, size_t hashval, size_t& offset) {
1865+
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
1866+
// ctrl_ could be nullptr
1867+
if (!ctrl_)
1868+
return false;
1869+
}
18511870
auto seq = probe(hashval);
18521871
while (true) {
18531872
Group g{ ctrl_ + seq.offset() };
@@ -2025,7 +2044,7 @@ class raw_hash_set
20252044
// Unpoison before returning the memory to the allocator.
20262045
SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_);
20272046
Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize());
2028-
ctrl_ = EmptyGroup();
2047+
ctrl_ = EmptyGroup<std_alloc_t>();
20292048
slots_ = nullptr;
20302049
size_ = 0;
20312050
capacity_ = 0;
@@ -2135,6 +2154,11 @@ class raw_hash_set
21352154
}
21362155

21372156
bool has_element(const value_type& elem, size_t hashval) const {
2157+
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
2158+
// ctrl_ could be nullptr
2159+
if (!ctrl_)
2160+
return false;
2161+
}
21382162
auto seq = probe(hashval);
21392163
while (true) {
21402164
Group g{ctrl_ + seq.offset()};
@@ -2197,6 +2221,11 @@ class raw_hash_set
21972221
protected:
21982222
template <class K>
21992223
size_t _find_key(const K& key, size_t hashval) {
2224+
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
2225+
// ctrl_ could be nullptr
2226+
if (!ctrl_)
2227+
return (size_t)-1;
2228+
}
22002229
auto seq = probe(hashval);
22012230
while (true) {
22022231
Group g{ctrl_ + seq.offset()};
@@ -2221,7 +2250,12 @@ class raw_hash_set
22212250
}
22222251

22232252
size_t prepare_insert(size_t hashval) PHMAP_ATTRIBUTE_NOINLINE {
2224-
auto target = find_first_non_full(hashval);
2253+
PHMAP_IF_CONSTEXPR (!std_alloc_t::value) {
2254+
// ctrl_ could be nullptr
2255+
if (!ctrl_)
2256+
rehash_and_grow_if_necessary();
2257+
}
2258+
FindInfo target = find_first_non_full(hashval);
22252259
if (PHMAP_PREDICT_FALSE(growth_left() == 0 &&
22262260
!IsDeleted(ctrl_[target.offset]))) {
22272261
rehash_and_grow_if_necessary();
@@ -2335,10 +2369,10 @@ class raw_hash_set
23352369
// TODO(alkis): Investigate removing some of these fields:
23362370
// - ctrl/slots can be derived from each other
23372371
// - size can be moved into the slot array
2338-
ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t]
2339-
slot_type* slots_ = nullptr; // [capacity * slot_type]
2340-
size_t size_ = 0; // number of full slots
2341-
size_t capacity_ = 0; // total number of slots
2372+
ctrl_t* ctrl_ = EmptyGroup<std_alloc_t>(); // [(capacity + 1) * ctrl_t]
2373+
slot_type* slots_ = nullptr; // [capacity * slot_type]
2374+
size_t size_ = 0; // number of full slots
2375+
size_t capacity_ = 0; // total number of slots
23422376
HashtablezInfoHandle infoz_;
23432377
std::tuple<size_t /* growth_left */, hasher, key_equal, allocator_type>
23442378
settings_{0, hasher{}, key_equal{}, allocator_type{}};
@@ -4576,6 +4610,8 @@ struct HashtableDebugAccess<Set, typename std::enable_if<has_member_type_raw_has
45764610

45774611
static size_t GetNumProbes(const Set& set,
45784612
const typename Set::key_type& key) {
4613+
if (!set.ctrl_)
4614+
return 0;
45794615
size_t num_probes = 0;
45804616
size_t hashval = set.hash(key);
45814617
auto seq = set.probe(hashval);

tests/raw_hash_set_test.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ TEST(BitMask, LeadingTrailing) {
162162
}
163163

164164
TEST(Group, EmptyGroup) {
165-
for (h2_t h = 0; h != 128; ++h) EXPECT_FALSE(Group{EmptyGroup()}.Match(h));
165+
for (h2_t h = 0; h != 128; ++h) EXPECT_FALSE(Group{EmptyGroup<std::true_type>()}.Match(h));
166166
}
167167

168168
TEST(Group, Match) {

0 commit comments

Comments
 (0)