@@ -30,13 +30,11 @@ template <typename T>
30
30
struct Element
31
31
{
32
32
using ElementType = T;
33
- CacheAlignedAndPaddedObject< std::atomic<std::uint_fast8_t > > state{ElementState::READY_FOR_PUSH};
33
+ std::atomic<std::uint_fast8_t > state{ElementState::READY_FOR_PUSH};
34
34
T *value_ptr;
35
35
typename std::aligned_storage<sizeof (T), alignof (T)>::type storage;
36
36
37
- Element () : value_ptr(nullptr )
38
- {
39
- }
37
+ Element () = default ;
40
38
41
39
Element (const Element &) = delete ;
42
40
Element (Element &&) = delete ;
@@ -46,7 +44,7 @@ struct Element
46
44
47
45
~Element ()
48
46
{
49
- if (value_ptr )
47
+ if (state. load (std::memory_order_acquire) == ElementState::READY_FOR_POP )
50
48
value_ptr->~T ();
51
49
}
52
50
};
@@ -80,6 +78,14 @@ struct CountInt64CompatibilityCheck
80
78
{
81
79
static_assert (Count <= static_cast <std::size_t >(std::numeric_limits<std::int64_t >::max()), " Count exceeds the maximum. Count should fit in a std::int64_t." );
82
80
};
81
+
82
+ struct DummyAtomicSizeT
83
+ {
84
+ std::size_t fetch_add (const std::size_t , const std::memory_order)
85
+ {
86
+ return 0 ;
87
+ }
88
+ };
83
89
} // namespace Private
84
90
85
91
template <typename ProducerTypeTraits, typename ConsumerTypeTraits, typename T, std::size_t Count>
@@ -113,14 +119,14 @@ struct MultiProducerTypeTraits
113
119
114
120
while (true )
115
121
{
116
- const std::size_t local_end = end.fetch_add (1 , std::memory_order_acquire);
117
- const std:: size_t element_index = local_end & BaseType::COUNT_MASK ;
122
+ const std::size_t element_index = end.fetch_add (1 , std::memory_order_acquire) & BaseType::COUNT_MASK ;
123
+ auto &element = BaseType::elements[element_index] ;
118
124
119
- auto expected_element_state = ElementState::READY_FOR_PUSH;
120
- if (std::atomic_compare_exchange_strong (&BaseType::elements[element_index] .state , &expected_element_state, ElementState::IN_PROGRESS))
125
+ std:: uint_fast8_t expected_element_state = ElementState::READY_FOR_PUSH;
126
+ if (std::atomic_compare_exchange_strong (&element .state , &expected_element_state, std::uint_fast8_t ( ElementState::IN_PROGRESS) ))
121
127
{
122
- BaseType::elements[element_index] .value_ptr = new (&BaseType::elements[element_index] .storage ) ElementType (std::forward<Args>(args)...);
123
- BaseType::elements[element_index] .state .store (ElementState::READY_FOR_POP, std::memory_order_release);
128
+ element .value_ptr = new (&element .storage ) ElementType (std::forward<Args>(args)...);
129
+ element .state .store (ElementState::READY_FOR_POP, std::memory_order_release);
124
130
BaseType::pop_task_count.fetch_add (1 , std::memory_order_release);
125
131
return true ;
126
132
}
@@ -157,16 +163,16 @@ struct MultiConsumerTypeTraits
157
163
158
164
while (true )
159
165
{
160
- const std::size_t local_begin = begin.fetch_add (1 , std::memory_order_acquire);
161
- const std:: size_t element_index = local_begin & BaseType::COUNT_MASK ;
166
+ const std::size_t element_index = begin.fetch_add (1 , std::memory_order_acquire) & BaseType::COUNT_MASK ;
167
+ auto &element = BaseType::elements[element_index] ;
162
168
163
- auto expected_element_state = ElementState::READY_FOR_POP;
164
- if (std::atomic_compare_exchange_strong (&BaseType::elements[element_index] .state , &expected_element_state, ElementState::IN_PROGRESS))
169
+ std:: uint_fast8_t expected_element_state = ElementState::READY_FOR_POP;
170
+ if (std::atomic_compare_exchange_strong (&element .state , &expected_element_state, std::uint_fast8_t ( ElementState::IN_PROGRESS) ))
165
171
{
166
- OptionalType<ElementType> result{std::move (*BaseType::elements[element_index] .value_ptr )};
167
- BaseType::elements[element_index] .value_ptr ->~ElementType ();
172
+ OptionalType<ElementType> result{std::move (*element .value_ptr )};
173
+ element .value_ptr ->~ElementType ();
168
174
169
- BaseType::elements[element_index] .state .store (ElementState::READY_FOR_PUSH, std::memory_order_release);
175
+ element .state .store (ElementState::READY_FOR_PUSH, std::memory_order_release);
170
176
BaseType::push_task_count.fetch_add (1 , std::memory_order_release);
171
177
return result;
172
178
}
@@ -190,50 +196,39 @@ struct SingleProducerTypeTraits
190
196
private:
191
197
struct State
192
198
{
193
- std::size_t pushed_task_count{0 };
194
- std::size_t local_push_task_count{BaseType::COUNT};
195
199
std::size_t end{0 };
196
200
};
197
201
198
- CacheAlignedAndPaddedObject<State> state{} ;
202
+ CacheAlignedAndPaddedObject<State> state;
199
203
200
204
public:
201
205
using ElementType = typename BaseType::ElementType;
202
206
203
207
template <typename ... Args>
204
208
bool push (Args &&... args)
205
209
{
206
- if (state.local_push_task_count == state.pushed_task_count )
207
- {
208
- const auto stack_local_push_task_count = BaseType::push_task_count.load (std::memory_order_acquire);
209
- if (state.local_push_task_count == stack_local_push_task_count)
210
- return false ;
211
- state.local_push_task_count = stack_local_push_task_count;
212
- }
210
+ const std::size_t element_index = state.end & BaseType::COUNT_MASK;
211
+ auto &element = BaseType::elements[element_index];
213
212
214
- state.pushed_task_count ++;
215
-
216
- while (true )
213
+ if (element.state .load (std::memory_order_acquire) == ElementState::READY_FOR_PUSH)
217
214
{
218
- const std::size_t element_index = (state.end ++) & BaseType::COUNT_MASK;
219
-
220
- if (BaseType::elements[element_index].state .load (std::memory_order_acquire) == ElementState::READY_FOR_PUSH)
221
- {
222
- BaseType::elements[element_index].value_ptr = new (&BaseType::elements[element_index].storage ) ElementType (std::forward<Args>(args)...);
223
- BaseType::elements[element_index].state .store (ElementState::READY_FOR_POP, std::memory_order_release);
224
- BaseType::pop_task_count.fetch_add (1 , std::memory_order_release);
215
+ element.value_ptr = new (&element.storage ) ElementType (std::forward<Args>(args)...);
216
+ element.state .store (ElementState::READY_FOR_POP, std::memory_order_release);
217
+ BaseType::pop_task_count.fetch_add (1 , std::memory_order_release);
225
218
226
- return true ;
227
- }
219
+ state. end ++ ;
220
+ return true ;
228
221
}
222
+ else
223
+ return false ;
229
224
}
230
225
};
231
226
232
227
template <typename BaseType>
233
228
struct SharedState : BaseType
234
229
{
235
230
protected:
236
- CacheAlignedAndPaddedObject<std:: atomic_size_t > push_task_count{BaseType::COUNT };
231
+ Private::DummyAtomicSizeT push_task_count{};
237
232
};
238
233
};
239
234
@@ -245,51 +240,40 @@ struct SingleConsumerTypeTraits
245
240
private:
246
241
struct State
247
242
{
248
- std::size_t popped_task_count{0 };
249
- std::size_t local_pop_task_count{0 };
250
243
std::size_t begin{0 };
251
244
};
252
245
253
- CacheAlignedAndPaddedObject<State> state{} ;
246
+ CacheAlignedAndPaddedObject<State> state;
254
247
255
248
public:
256
249
using ElementType = typename BaseType::ElementType;
257
250
258
251
OptionalType<ElementType> pop ()
259
252
{
260
- if (state.local_pop_task_count == state.popped_task_count )
261
- {
262
- const auto stack_local_pop_task_count = BaseType::pop_task_count.load (std::memory_order_acquire);
263
- if (state.local_pop_task_count == stack_local_pop_task_count)
264
- return OptionalType<ElementType>{};
265
- state.local_pop_task_count = stack_local_pop_task_count;
266
- }
267
-
268
- state.popped_task_count ++;
253
+ const std::size_t element_index = state.begin & BaseType::COUNT_MASK;
254
+ auto &element = BaseType::elements[element_index];
269
255
270
- while ( true )
256
+ if (element. state . load (std::memory_order_acquire) == ElementState::READY_FOR_POP )
271
257
{
272
- const std::size_t element_index = (state.begin ++) & BaseType::COUNT_MASK;
258
+ OptionalType<ElementType> result{std::move (*element.value_ptr )};
259
+ element.value_ptr ->~ElementType ();
273
260
274
- if (BaseType::elements[element_index].state .load (std::memory_order_acquire) == ElementState::READY_FOR_POP)
275
- {
276
- OptionalType<ElementType> result{std::move (*BaseType::elements[element_index].value_ptr )};
277
- BaseType::elements[element_index].value_ptr ->~ElementType ();
278
-
279
- BaseType::elements[element_index].state .store (ElementState::READY_FOR_PUSH, std::memory_order_release);
280
- BaseType::push_task_count.fetch_add (1 , std::memory_order_release);
261
+ element.state .store (ElementState::READY_FOR_PUSH, std::memory_order_release);
262
+ BaseType::push_task_count.fetch_add (1 , std::memory_order_release);
281
263
282
- return result ;
283
- }
264
+ state. begin ++ ;
265
+ return result;
284
266
}
267
+ else
268
+ return OptionalType<ElementType>{};
285
269
}
286
270
};
287
271
288
272
template <typename BaseType>
289
273
struct SharedState : BaseType
290
274
{
291
275
protected:
292
- CacheAlignedAndPaddedObject<std:: atomic_size_t > pop_task_count{std:: size_t { 0 } };
276
+ Private::DummyAtomicSizeT pop_task_count{};
293
277
};
294
278
};
295
279
0 commit comments