Skip to content

Commit 89dd68b

Browse files
marko-bekhtamarko-bekhta
committed
HSEARCH-5309 Limit the time that the outbox polling processor spends trying to lock/load event entities after processing
1 parent 29a03c0 commit 89dd68b

File tree

9 files changed

+180
-9
lines changed

9 files changed

+180
-9
lines changed

‎integrationtest/mapper/orm-outbox-polling/src/test/java/org/hibernate/search/integrationtest/mapper/orm/outboxpolling/automaticindexing/OutboxPollingAutomaticIndexingEdgeCasesIT.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,33 +6,42 @@
66

77
import static org.assertj.core.api.Assertions.assertThat;
88
import static org.hibernate.search.util.impl.integrationtest.mapper.orm.OrmUtils.with;
9+
import static org.junit.jupiter.api.Assumptions.assumeTrue;
910

1011
import java.util.Collections;
1112
import java.util.List;
1213
import java.util.UUID;
14+
import java.util.concurrent.TimeUnit;
1315

1416
import jakarta.persistence.Entity;
1517
import jakarta.persistence.Id;
1618
import jakarta.persistence.OneToMany;
1719
import jakarta.persistence.OneToOne;
1820

21+
import org.hibernate.LockMode;
1922
import org.hibernate.SessionFactory;
23+
import org.hibernate.engine.spi.SessionFactoryImplementor;
2024
import org.hibernate.search.integrationtest.mapper.orm.outboxpolling.testsupport.util.OutboxEventFilter;
2125
import org.hibernate.search.integrationtest.mapper.orm.outboxpolling.testsupport.util.TestingOutboxPollingInternalConfigurer;
26+
import org.hibernate.search.mapper.orm.outboxpolling.cfg.HibernateOrmMapperOutboxPollingSettings;
2227
import org.hibernate.search.mapper.orm.outboxpolling.cfg.impl.HibernateOrmMapperOutboxPollingImplSettings;
28+
import org.hibernate.search.mapper.orm.outboxpolling.event.impl.OutboxEvent;
2329
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
2430
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.IndexedEmbedded;
2531
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.KeywordField;
2632
import org.hibernate.search.util.impl.integrationtest.common.extension.BackendMock;
2733
import org.hibernate.search.util.impl.integrationtest.mapper.orm.CoordinationStrategyExpectations;
2834
import org.hibernate.search.util.impl.integrationtest.mapper.orm.OrmSetupHelper;
35+
import org.hibernate.search.util.impl.test.extension.ExpectedLog4jLog;
2936

3037
import org.junit.jupiter.api.BeforeAll;
3138
import org.junit.jupiter.api.BeforeEach;
3239
import org.junit.jupiter.api.Test;
3340
import org.junit.jupiter.api.TestInstance;
3441
import org.junit.jupiter.api.extension.RegisterExtension;
3542

43+
import org.awaitility.Awaitility;
44+
3645
/**
3746
* Extensive tests with edge cases for automatic indexing with the outbox-polling strategy.
3847
*/
@@ -44,6 +53,9 @@ class OutboxPollingAutomaticIndexingEdgeCasesIT {
4453
@RegisterExtension
4554
public static BackendMock backendMock = BackendMock.create();
4655

56+
@RegisterExtension
57+
public ExpectedLog4jLog logged = ExpectedLog4jLog.create();
58+
4759
@RegisterExtension
4860
public static OrmSetupHelper ormSetupHelper =
4961
OrmSetupHelper.withCoordinationStrategy( CoordinationStrategyExpectations.outboxPolling() )
@@ -65,6 +77,8 @@ void setup() {
6577
HibernateOrmMapperOutboxPollingImplSettings.COORDINATION_INTERNAL_CONFIGURER,
6678
new TestingOutboxPollingInternalConfigurer().outboxEventFilter( eventFilter )
6779
)
80+
.withProperty( HibernateOrmMapperOutboxPollingSettings.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY, 0 )
81+
.withProperty( HibernateOrmMapperOutboxPollingSettings.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX, 1 )
6882
.withAnnotatedTypes( IndexedEntity.class, IndexedAndContainedEntity.class )
6983
.setup();
7084
}
@@ -184,6 +198,41 @@ void addIndexedAndContained_addAndUpdateEventsProcessedInDifferentBatches() {
184198
backendMock.verifyExpectationsMet();
185199
}
186200

201+
@Test
202+
void lockedEventRowRetries() {
203+
assumeTrue(
204+
sessionFactory.unwrap( SessionFactoryImplementor.class ).getJdbcServices().getDialect().supportsSkipLocked(),
205+
"This test only make sense if skip locked rows is supported by the underlying DB. " +
206+
"Otherwise the locking will trow an exception and the batch will be just reprocessed without a retry of locking events." );
207+
208+
eventFilter.hideAllEvents();
209+
with( sessionFactory ).runInTransaction( session -> {
210+
session.persist( new IndexedEntity( 1, "initialValue" ) );
211+
} );
212+
213+
backendMock.expectWorks( IndexedEntity.NAME )
214+
.addOrUpdate( "1", b -> b
215+
.field( "text", "initialValue" ) );
216+
217+
with( sessionFactory ).runInTransaction( session -> {
218+
List<OutboxEvent> events = eventFilter.findOutboxEventsNoFilter( session );
219+
assertThat( events ).hasSize( 1 );
220+
session.lock( events.get( 0 ), LockMode.PESSIMISTIC_WRITE );
221+
222+
// let processor see the events and as that single event is locked it should skip, and reach the max retries:
223+
eventFilter.showAllEvents();
224+
logged.expectMessage( "after 1 retries, failed to acquire a lock on the following outbox events" )
225+
.atLeast( 5 );
226+
227+
Awaitility.await().timeout( 5, TimeUnit.SECONDS )
228+
.untilAsserted( () -> logged.expectationsMet() );
229+
} );
230+
231+
with( sessionFactory ).runInTransaction( session -> {
232+
eventFilter.awaitUntilNoMoreVisibleEvents( sessionFactory );
233+
} );
234+
235+
}
187236

188237
@Entity(name = IndexedEntity.NAME)
189238
@Indexed

‎integrationtest/mapper/orm-outbox-polling/src/test/java/org/hibernate/search/integrationtest/mapper/orm/outboxpolling/testsupport/util/OutboxEventFilter.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,13 @@ public void awaitUntilNoMoreVisibleEvents(SessionFactory sessionFactory) {
125125
} ) );
126126
}
127127

128+
public void awaitUntilNumberOfVisibleEvents(SessionFactory sessionFactory, int count) {
129+
await().untilAsserted( () -> with( sessionFactory ).runInTransaction( session -> {
130+
List<OutboxEvent> outboxEntries = visibleEventsAllShardsFinder.findOutboxEvents( session, count + 1 );
131+
assertThat( outboxEntries ).hasSize( count );
132+
} ) );
133+
}
134+
128135
private class FilterById implements OutboxEventPredicate {
129136
@Override
130137
public String queryPart(String eventAlias) {

‎mapper/orm-outbox-polling/src/main/java/org/hibernate/search/mapper/orm/outboxpolling/cfg/HibernateOrmMapperOutboxPollingSettings.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,41 @@ private HibernateOrmMapperOutboxPollingSettings() {
269269
public static final String COORDINATION_EVENT_PROCESSOR_RETRY_DELAY =
270270
PREFIX + Radicals.COORDINATION_EVENT_PROCESSOR_RETRY_DELAY;
271271

272+
/**
273+
* How many times the event processor must try locking the outbox event database records for processing
274+
* before leaving them to be processed in another batch.
275+
* <p>
276+
* Only available when {@value HibernateOrmMapperSettings#COORDINATION_STRATEGY} is
277+
* {@value #COORDINATION_STRATEGY_NAME}.
278+
* <p>
279+
* Only applicable when the database supports skipping locked rows when locking.
280+
* <p>
281+
* Expects a positive integer value, such as {@code 10},
282+
* or a String that can be parsed into such Integer value.
283+
* <p>
284+
* Defaults to {@link Defaults#COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX}.
285+
*/
286+
public static final String COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX =
287+
PREFIX + Radicals.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX;
288+
289+
/**
290+
* How long the event processor must wait before retrying to lock the outbox event database records.
291+
* <p>
292+
* Only available when {@value HibernateOrmMapperSettings#COORDINATION_STRATEGY} is
293+
* {@value #COORDINATION_STRATEGY_NAME}.
294+
* <p>
295+
* Only applicable when the database supports skipping locked rows when locking.
296+
* <p>
297+
* Expects a positive integer value in seconds, such as {@code 5},
298+
* or a String that can be parsed into such Integer value.
299+
* <p>
300+
* Use the value {@code 0} to retry locking events as soon as possible, with no delay.
301+
* <p>
302+
* Defaults to {@link Defaults#COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY}.
303+
*/
304+
public static final String COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY =
305+
PREFIX + Radicals.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY;
306+
272307
/**
273308
* In the mass indexer, how long to wait for another query to the agent table
274309
* when actively waiting for event processors to suspend themselves, in milliseconds.
@@ -516,6 +551,10 @@ private Radicals() {
516551
COORDINATION_PREFIX + CoordinationRadicals.EVENT_PROCESSOR_TRANSACTION_TIMEOUT;
517552
public static final String COORDINATION_EVENT_PROCESSOR_RETRY_DELAY =
518553
COORDINATION_PREFIX + CoordinationRadicals.EVENT_PROCESSOR_RETRY_DELAY;
554+
public static final String COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX =
555+
COORDINATION_PREFIX + CoordinationRadicals.EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX;
556+
public static final String COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY =
557+
COORDINATION_PREFIX + CoordinationRadicals.EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY;
519558
public static final String COORDINATION_MASS_INDEXER_POLLING_INTERVAL =
520559
COORDINATION_PREFIX + CoordinationRadicals.MASS_INDEXER_POLLING_INTERVAL;
521560
public static final String COORDINATION_MASS_INDEXER_PULSE_INTERVAL =
@@ -564,6 +603,8 @@ private CoordinationRadicals() {
564603
public static final String EVENT_PROCESSOR_BATCH_SIZE = EVENT_PROCESSOR_PREFIX + "batch_size";
565604
public static final String EVENT_PROCESSOR_TRANSACTION_TIMEOUT = EVENT_PROCESSOR_PREFIX + "transaction_timeout";
566605
public static final String EVENT_PROCESSOR_RETRY_DELAY = EVENT_PROCESSOR_PREFIX + "retry_delay";
606+
public static final String EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX = EVENT_PROCESSOR_PREFIX + "event_lock_retry_max";
607+
public static final String EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY = EVENT_PROCESSOR_PREFIX + "event_lock_retry_delay";
567608
public static final String MASS_INDEXER_PREFIX = "mass_indexer.";
568609
public static final String MASS_INDEXER_POLLING_INTERVAL = MASS_INDEXER_PREFIX + "polling_interval";
569610
public static final String MASS_INDEXER_PULSE_INTERVAL = MASS_INDEXER_PREFIX + "pulse_interval";
@@ -600,6 +641,8 @@ private Defaults() {
600641
public static final OutboxEventProcessingOrder COORDINATION_EVENT_PROCESSOR_ORDER = OutboxEventProcessingOrder.AUTO;
601642
public static final int COORDINATION_EVENT_PROCESSOR_BATCH_SIZE = 50;
602643
public static final int COORDINATION_EVENT_PROCESSOR_RETRY_DELAY = 30;
644+
public static final int COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX = 20;
645+
public static final int COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY = 2;
603646
public static final int COORDINATION_MASS_INDEXER_POLLING_INTERVAL = 100;
604647
public static final int COORDINATION_MASS_INDEXER_PULSE_INTERVAL = 2000;
605648
public static final int COORDINATION_MASS_INDEXER_PULSE_EXPIRATION = 30000;

‎mapper/orm-outbox-polling/src/main/java/org/hibernate/search/mapper/orm/outboxpolling/event/impl/OutboxEventUpdater.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import java.time.Instant;
88
import java.util.ArrayList;
9+
import java.util.Collections;
910
import java.util.List;
1011
import java.util.Set;
1112
import java.util.UUID;
@@ -48,6 +49,10 @@ public boolean thereAreStillEventsToProcess() {
4849
return !eventsIds.isEmpty();
4950
}
5051

52+
public Set<UUID> eventsToProcess() {
53+
return Collections.unmodifiableSet( eventsIds );
54+
}
55+
5156
public void process() {
5257
List<OutboxEvent> lockedEvents = loader.loadLocking( session, eventsIds, processorName );
5358
List<OutboxEvent> eventToDelete = new ArrayList<>( lockedEvents );

‎mapper/orm-outbox-polling/src/main/java/org/hibernate/search/mapper/orm/outboxpolling/event/impl/OutboxPollingEventProcessor.java

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,22 @@ public static String namePrefix(String tenantId) {
8787
.withDefault( HibernateOrmMapperOutboxPollingSettings.Defaults.COORDINATION_EVENT_PROCESSOR_RETRY_DELAY )
8888
.build();
8989

90+
private static final ConfigurationProperty<Integer> EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX =
91+
ConfigurationProperty
92+
.forKey( HibernateOrmMapperOutboxPollingSettings.CoordinationRadicals.EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX )
93+
.asIntegerPositiveOrZero()
94+
.withDefault(
95+
HibernateOrmMapperOutboxPollingSettings.Defaults.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX )
96+
.build();
97+
98+
private static final ConfigurationProperty<Integer> EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY =
99+
ConfigurationProperty
100+
.forKey( HibernateOrmMapperOutboxPollingSettings.CoordinationRadicals.EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY )
101+
.asIntegerPositiveOrZero()
102+
.withDefault(
103+
HibernateOrmMapperOutboxPollingSettings.Defaults.COORDINATION_EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY )
104+
.build();
105+
90106
public static Factory factory(AutomaticIndexingMappingContext mapping, Clock clock, String tenantId,
91107
ConfigurationPropertySource configurationSource) {
92108
OutboxEventLoader loader = new OutboxEventLoader( mapping.sessionFactory().getJdbcServices().getDialect() );
@@ -102,8 +118,12 @@ public static Factory factory(AutomaticIndexingMappingContext mapping, Clock clo
102118
Integer transactionTimeout = TRANSACTION_TIMEOUT.get( configurationSource )
103119
.orElse( null );
104120

121+
int lockEventsMaxRetry = EVENT_PROCESSOR_EVENT_LOCK_RETRY_MAX.get( configurationSource );
122+
long lockEventsInterval =
123+
Duration.ofSeconds( EVENT_PROCESSOR_EVENT_LOCK_RETRY_DELAY.get( configurationSource ) ).toMillis();
124+
105125
return new Factory( mapping, clock, tenantId, loader, pollingInterval, pulseInterval, pulseExpiration,
106-
batchSize, retryDelay, transactionTimeout );
126+
batchSize, retryDelay, transactionTimeout, lockEventsMaxRetry, lockEventsInterval );
107127
}
108128

109129
public static class Factory {
@@ -117,10 +137,13 @@ public static class Factory {
117137
private final int batchSize;
118138
private final int retryDelay;
119139
private final Integer transactionTimeout;
140+
private final int lockEventsMaxRetry;
141+
private final long lockEventsInterval;
120142

121143
private Factory(AutomaticIndexingMappingContext mapping, Clock clock, String tenantId,
122144
OutboxEventLoader loader, Duration pollingInterval, Duration pulseInterval, Duration pulseExpiration,
123-
int batchSize, int retryDelay, Integer transactionTimeout) {
145+
int batchSize, int retryDelay, Integer transactionTimeout,
146+
int lockEventsMaxRetry, long lockEventsInterval) {
124147
this.mapping = mapping;
125148
this.clock = clock;
126149
this.tenantId = tenantId;
@@ -131,6 +154,8 @@ private Factory(AutomaticIndexingMappingContext mapping, Clock clock, String ten
131154
this.batchSize = batchSize;
132155
this.retryDelay = retryDelay;
133156
this.transactionTimeout = transactionTimeout;
157+
this.lockEventsMaxRetry = lockEventsMaxRetry;
158+
this.lockEventsInterval = lockEventsInterval;
134159
}
135160

136161
public OutboxPollingEventProcessor create(ScheduledExecutorService scheduledExecutor,
@@ -159,6 +184,8 @@ private enum Status {
159184
private final long pollingInterval;
160185
private final int batchSize;
161186
private final int retryDelay;
187+
private final int lockEventsMaxRetry;
188+
private final long lockEventsInterval;
162189

163190
private final AtomicReference<Status> status = new AtomicReference<>( Status.STOPPED );
164191
private final OutboxPollingEventProcessorClusterLink clusterLink;
@@ -180,6 +207,8 @@ public OutboxPollingEventProcessor(String name, Factory factory,
180207
this.pollingInterval = factory.pollingInterval.toMillis();
181208
this.batchSize = factory.batchSize;
182209
this.retryDelay = factory.retryDelay;
210+
this.lockEventsMaxRetry = factory.lockEventsMaxRetry;
211+
this.lockEventsInterval = factory.lockEventsInterval;
183212
this.clusterLink = clusterLink;
184213

185214
transactionHelper = new TransactionHelper( mapping.sessionFactory(), factory.transactionTimeout );
@@ -317,8 +346,28 @@ public CompletableFuture<?> work() {
317346
// it locked a page instead of just a row.
318347
// For more information, see
319348
// org.hibernate.search.mapper.orm.outboxpolling.impl.OutboxEventLoader.tryLoadLocking
320-
while ( eventUpdater.thereAreStillEventsToProcess() ) {
349+
int retryCount = 0;
350+
while ( true ) {
351+
if ( retryCount > lockEventsMaxRetry ) {
352+
OutboxPollingEventsLog.INSTANCE.eventLockingRetryLimitReached( clusterLink.selfReference(),
353+
lockEventsMaxRetry, eventUpdater.eventsToProcess() );
354+
// not failing to not produce an error log:
355+
return CompletableFuture.completedFuture( null );
356+
}
321357
transactionHelper.inTransaction( session, eventUpdater::process );
358+
if ( eventUpdater.thereAreStillEventsToProcess() ) {
359+
try {
360+
Thread.sleep( lockEventsInterval );
361+
}
362+
catch (InterruptedException e) {
363+
Thread.currentThread().interrupt();
364+
return CompletableFuture.failedFuture( e );
365+
}
366+
retryCount++;
367+
}
368+
else {
369+
break;
370+
}
322371
}
323372

324373
return CompletableFuture.completedFuture( null );

‎mapper/orm-outbox-polling/src/main/java/org/hibernate/search/mapper/orm/outboxpolling/logging/impl/OutboxPollingEventsLog.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,4 +266,10 @@ void clusterMembersAreInClusterReachedExpectedStates(AgentReference agentReferen
266266
@LogMessage(level = TRACE)
267267
@Message(id = ID_OFFSET + 67, value = "Persisted %d outbox events: '%s'")
268268
void eventPlanNumberOfPersistedEvents(int size, List<OutboxEvent> events);
269+
270+
@LogMessage(level = WARN)
271+
@Message(id = ID_OFFSET + 70,
272+
value = "Agent '%s': after %d retries, failed to acquire a lock on the following outbox events: %s. " +
273+
"Events will be re-processed at a later time.")
274+
void eventLockingRetryLimitReached(AgentReference agentReference, int numberOfRetries, Set<UUID> events);
269275
}

‎mapper/orm-outbox-polling/src/main/java/org/hibernate/search/mapper/orm/outboxpolling/logging/impl/OutboxPollingLog.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ public interface OutboxPollingLog extends ConfigurationLog, DeprecationLog, Outb
3030
* here to the next value.
3131
*/
3232
@LogMessage(level = TRACE)
33-
@Message(id = ID_OFFSET + 70, value = "")
33+
@Message(id = ID_OFFSET + 71, value = "")
3434
void nextLoggerIdForConvenience();
3535
}

‎util/internal/test/common/src/main/java/org/hibernate/search/util/impl/test/extension/ExpectedLog4jLog.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*/
55
package org.hibernate.search.util.impl.test.extension;
66

7+
import static org.assertj.core.api.Assertions.assertThat;
78
import static org.assertj.core.api.Assertions.fail;
89

910
import java.util.ArrayList;
@@ -234,4 +235,8 @@ private static String buildFailureMessage(Set<LogChecker> failingCheckers) {
234235
}
235236
return description.toString();
236237
}
238+
239+
public void expectationsMet() {
240+
assertThat( currentAppender.getFailingCheckers() ).isEmpty();
241+
}
237242
}

‎util/internal/test/common/src/main/java/org/hibernate/search/util/impl/test/extension/log4j/LogExpectation.java

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@
99
public class LogExpectation {
1010

1111
private final Matcher<?> matcher;
12-
private Integer expectedCount;
12+
private Integer expectedMinCount;
13+
private Integer expectedMaxCount;
1314

1415
public LogExpectation(Matcher<?> matcher) {
1516
this.matcher = matcher;
@@ -24,10 +25,16 @@ public void once() {
2425
}
2526

2627
public void times(int expectedCount) {
27-
if ( this.expectedCount != null ) {
28+
if ( this.expectedMinCount != null || this.expectedMaxCount != null ) {
2829
throw new IllegalStateException( "Can only set log expectations once" );
2930
}
30-
this.expectedCount = expectedCount;
31+
this.expectedMinCount = expectedCount;
32+
this.expectedMaxCount = expectedCount;
33+
}
34+
35+
public void atLeast(int expectedCount) {
36+
this.expectedMaxCount = Integer.MAX_VALUE;
37+
this.expectedMinCount = expectedCount;
3138
}
3239

3340
public LogChecker createChecker() {
@@ -39,10 +46,10 @@ Matcher<?> getMatcher() {
3946
}
4047

4148
int getMinExpectedCount() {
42-
return expectedCount == null ? 1 : expectedCount;
49+
return expectedMinCount == null ? 1 : expectedMinCount;
4350
}
4451

4552
Integer getMaxExpectedCount() {
46-
return expectedCount;
53+
return expectedMaxCount;
4754
}
4855
}

0 commit comments

Comments
 (0)