Skip to content

Commit 6000159

Browse files
marko-bekhtamarko-bekhta
committed
HSEARCH-2945 Update mass indexer monitor configuration
1 parent 63607f7 commit 6000159

File tree

21 files changed

+717
-130
lines changed

21 files changed

+717
-130
lines changed

‎documentation/src/main/asciidoc/migration/index.adoc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,10 @@ But there are next changes:
9595
This was done to address the scenarios where the total number of identifiers to load is not known ahead of time.
9696
- Deprecated `org.hibernate.search.mapper.orm.massindexing.MassIndexingFailureHandler`, `org.hibernate.search.mapper.orm.massindexing.MassIndexingMonitor`
9797
interfaces are removed in this version. They have their alternatives in a `org.hibernate.search.mapper.pojo.massindexing` for a while now.
98+
- `org.hibernate.search.mapper.pojo.massindexing.MassIndexingMonitor#addToTotalCount(..)` gets deprecated for removal.
99+
Instead, we are introducing the `org.hibernate.search.mapper.pojo.massindexing.MassIndexingTypeGroupMonitor`
100+
that can be obtained through `org.hibernate.search.mapper.pojo.massindexing.MassIndexingMonitor#typeGroupMonitor(..)`.
101+
This new type group monitor has more flexibility and also allows implementors to skip total count computations if needed.
98102

99103
[[spi]]
100104
== SPI

‎documentation/src/main/asciidoc/public/reference/_indexing-massindexer.adoc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,9 @@ The default, built-in monitor logs progress periodically at the `INFO` level,
318318
but a custom monitor can be set by implementing the `MassIndexingMonitor` interface
319319
and passing an instance using the `monitor` method.
320320

321+
The built-in monitor's behaviour can be customized through `DefaultMassIndexingMonitor` builder,
322+
e.g. `indexer.monitor( DefaultMassIndexingMonitor.builder().countOnStart( false ).build() ) )`
323+
321324
Implementations of `MassIndexingMonitor` must be thread-safe.
322325

323326
|`failureHandler(MassIndexingFailureHandler)`

‎integrationtest/mapper/orm/src/test/java/org/hibernate/search/integrationtest/mapper/orm/massindexing/MassIndexingMonitorIT.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ public String getAuthor() {
167167
}
168168
}
169169

170+
@SuppressWarnings("removal")
170171
public static class StaticCountersMonitor implements MassIndexingMonitor {
171172

172173
public static StaticCounters.Key ADDED = StaticCounters.createKey();

‎integrationtest/mapper/pojo-base/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/massindexing/MassIndexingDefaultMonitorIT.java

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
* Copyright Red Hat Inc. and Hibernate Authors
4+
*/
5+
package org.hibernate.search.integrationtest.mapper.pojo.massindexing;
6+
7+
import static org.assertj.core.api.Fail.fail;
8+
9+
import java.lang.invoke.MethodHandles;
10+
import java.util.concurrent.CompletableFuture;
11+
import java.util.function.Consumer;
12+
13+
import org.hibernate.search.engine.backend.work.execution.DocumentCommitStrategy;
14+
import org.hibernate.search.engine.backend.work.execution.DocumentRefreshStrategy;
15+
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.loading.PersistenceTypeKey;
16+
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.loading.StubEntityLoadingBinder;
17+
import org.hibernate.search.integrationtest.mapper.pojo.testsupport.loading.StubLoadingContext;
18+
import org.hibernate.search.mapper.pojo.loading.mapping.annotation.EntityLoadingBinderRef;
19+
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.DocumentId;
20+
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.GenericField;
21+
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
22+
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.SearchEntity;
23+
import org.hibernate.search.mapper.pojo.massindexing.DefaultMassIndexingMonitor;
24+
import org.hibernate.search.mapper.pojo.standalone.mapping.SearchMapping;
25+
import org.hibernate.search.mapper.pojo.standalone.massindexing.MassIndexer;
26+
import org.hibernate.search.mapper.pojo.standalone.session.SearchSession;
27+
import org.hibernate.search.util.impl.integrationtest.common.extension.BackendMock;
28+
import org.hibernate.search.util.impl.integrationtest.mapper.pojo.standalone.StandalonePojoMappingSetupHelper;
29+
import org.hibernate.search.util.impl.test.extension.ExpectedLog4jLog;
30+
31+
import org.junit.jupiter.api.Test;
32+
import org.junit.jupiter.api.extension.RegisterExtension;
33+
import org.junit.jupiter.params.ParameterizedTest;
34+
import org.junit.jupiter.params.provider.ValueSource;
35+
36+
import org.apache.logging.log4j.Level;
37+
38+
class MassIndexingDefaultMonitorIT {
39+
40+
private static final int COUNT = 100;
41+
@RegisterExtension
42+
public final BackendMock backendMock = BackendMock.create();
43+
44+
@RegisterExtension
45+
public final StandalonePojoMappingSetupHelper setupHelper =
46+
StandalonePojoMappingSetupHelper.withBackendMock( MethodHandles.lookup(), backendMock );
47+
48+
@RegisterExtension
49+
public ExpectedLog4jLog logged = ExpectedLog4jLog.create();
50+
51+
private final StubLoadingContext loadingContext = new StubLoadingContext();
52+
53+
@ValueSource(booleans = { true, false })
54+
@ParameterizedTest
55+
void countOnBeforeType(boolean doCounts) {
56+
actualTest( () -> {
57+
logged.expectEvent( Level.INFO, "Mass indexing complete in ", ". Indexed 100/100 entities" );
58+
if ( doCounts ) {
59+
logged.expectEvent( Level.INFO, "Mass indexing is going to index 100 entities" ).once();
60+
}
61+
else {
62+
logged.expectEvent( Level.INFO, "Mass indexing is going to index 100 entities" ).never();
63+
}
64+
}, indexer -> indexer.monitor( DefaultMassIndexingMonitor.builder().countOnBeforeType( doCounts ).build() ) );
65+
}
66+
67+
@ValueSource(booleans = { true, false })
68+
@ParameterizedTest
69+
void countOnStart(boolean doCounts) {
70+
actualTest( () -> {
71+
logged.expectEvent( Level.INFO, "Mass indexing complete in ", ". Indexed 100/100 entities" );
72+
logged.expectEvent( Level.INFO, "Mass indexing is going to index 100 entities" ).once();
73+
if ( doCounts ) {
74+
logged.expectEvent( Level.INFO,
75+
"Mass indexing is going to index approx. 100 entities ([ Book ]). Actual number may change once the indexing starts." )
76+
.once();
77+
}
78+
else {
79+
logged.expectEvent( Level.INFO,
80+
"Mass indexing is going to index approx. 100 entities ([ Book ]). Actual number may change once the indexing starts." )
81+
.never();
82+
}
83+
}, indexer -> indexer.monitor( DefaultMassIndexingMonitor.builder().countOnStart( doCounts ).build() ) );
84+
}
85+
86+
@Test
87+
void noCountsAtAll() {
88+
actualTest( () -> {
89+
logged.expectEvent( Level.INFO, "Mass indexing complete in ", ". Indexed 100/100 entities" ).once();
90+
logged.expectEvent( Level.INFO, "Mass indexing is going to index 100 entities" ).never();
91+
logged.expectEvent( Level.INFO,
92+
"Mass indexing is going to index approx. 100 entities ([ Book ]). Actual number may change once the indexing starts." )
93+
.never();
94+
}, indexer -> indexer
95+
.monitor( DefaultMassIndexingMonitor.builder().countOnStart( false ).countOnBeforeType( false ).build() ) );
96+
}
97+
98+
private void actualTest(Runnable expectedLogs, Consumer<MassIndexer> massIndexerConfiguration) {
99+
backendMock.expectAnySchema( Book.NAME );
100+
101+
SearchMapping mapping = setupHelper.start()
102+
.expectCustomBeans()
103+
.setup( Book.class );
104+
105+
backendMock.verifyExpectationsMet();
106+
107+
initData();
108+
109+
expectedLogs.run();
110+
111+
try ( SearchSession searchSession = mapping.createSession() ) {
112+
MassIndexer indexer = searchSession.massIndexer()
113+
// Simulate passing information to connect to a DB, ...
114+
.context( StubLoadingContext.class, loadingContext );
115+
116+
CompletableFuture<?> indexingFuture = new CompletableFuture<>();
117+
indexingFuture.completeExceptionally( new SimulatedFailure( "Indexing error" ) );
118+
119+
// add operations on indexes can follow any random order,
120+
// since they are executed by different threads
121+
BackendMock.DocumentWorkCallListContext expectWorks = backendMock.expectWorks(
122+
Book.NAME, DocumentCommitStrategy.NONE, DocumentRefreshStrategy.NONE
123+
);
124+
for ( int i = 0; i < COUNT; i++ ) {
125+
final String id = Integer.toString( i );
126+
expectWorks
127+
.add( id, b -> b
128+
.field( "title", "TITLE_" + id )
129+
.field( "author", "AUTHOR_" + id )
130+
);
131+
}
132+
133+
// purgeAtStart and mergeSegmentsAfterPurge are enabled by default,
134+
// so we expect 1 purge, 1 mergeSegments and 1 flush calls in this order:
135+
backendMock.expectIndexScaleWorks( Book.NAME, searchSession.tenantIdentifierValue() )
136+
.purge()
137+
.mergeSegments()
138+
.flush()
139+
.refresh();
140+
141+
try {
142+
massIndexerConfiguration.accept( indexer );
143+
indexer.startAndWait();
144+
}
145+
catch (InterruptedException e) {
146+
fail( "Unexpected InterruptedException: " + e.getMessage() );
147+
}
148+
}
149+
150+
backendMock.verifyExpectationsMet();
151+
}
152+
153+
private void initData() {
154+
for ( int i = 0; i < COUNT; i++ ) {
155+
persist( new Book( i, "TITLE_" + i, "AUTHOR_" + i ) );
156+
}
157+
}
158+
159+
private void persist(Book book) {
160+
loadingContext.persistenceMap( Book.PERSISTENCE_KEY ).put( book.id, book );
161+
}
162+
163+
@SearchEntity(name = Book.NAME,
164+
loadingBinder = @EntityLoadingBinderRef(type = StubEntityLoadingBinder.class))
165+
@Indexed(index = Book.NAME)
166+
public static class Book {
167+
168+
public static final String NAME = "Book";
169+
public static final PersistenceTypeKey<Book, Integer> PERSISTENCE_KEY =
170+
new PersistenceTypeKey<>( Book.class, Integer.class );
171+
172+
@DocumentId
173+
private Integer id;
174+
175+
@GenericField
176+
private String title;
177+
178+
@GenericField
179+
private String author;
180+
181+
public Book() {
182+
}
183+
184+
public Book(Integer id, String title, String author) {
185+
this.id = id;
186+
this.title = title;
187+
this.author = author;
188+
}
189+
190+
public Integer getId() {
191+
return id;
192+
}
193+
194+
public String getTitle() {
195+
return title;
196+
}
197+
198+
public String getAuthor() {
199+
return author;
200+
}
201+
}
202+
203+
private static class SimulatedFailure extends RuntimeException {
204+
SimulatedFailure(String message) {
205+
super( message );
206+
}
207+
}
208+
}

‎integrationtest/mapper/pojo-base/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/massindexing/MassIndexingMonitorIT.java

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.Indexed;
2323
import org.hibernate.search.mapper.pojo.mapping.definition.annotation.SearchEntity;
2424
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingMonitor;
25+
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingTypeGroupMonitor;
26+
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingTypeGroupMonitorContext;
27+
import org.hibernate.search.mapper.pojo.massindexing.MassIndexingTypeGroupMonitorCreateContext;
2528
import org.hibernate.search.mapper.pojo.standalone.mapping.SearchMapping;
2629
import org.hibernate.search.mapper.pojo.standalone.massindexing.MassIndexer;
2730
import org.hibernate.search.mapper.pojo.standalone.session.SearchSession;
@@ -128,6 +131,66 @@ void simple() {
128131
assertThat( staticCounters.get( StaticCountersMonitor.INDEXING_COMPLETED ) ).isEqualTo( 1 );
129132
}
130133

134+
@Test
135+
void skipTotalCount() {
136+
SearchMapping mapping = setup( null );
137+
138+
try ( SearchSession searchSession = mapping.createSession() ) {
139+
MassIndexer indexer = searchSession.massIndexer()
140+
// Simulate passing information to connect to a DB, ...
141+
.context( StubLoadingContext.class, loadingContext );
142+
143+
CompletableFuture<?> indexingFuture = new CompletableFuture<>();
144+
indexingFuture.completeExceptionally( new SimulatedFailure( "Indexing error" ) );
145+
146+
// add operations on indexes can follow any random order,
147+
// since they are executed by different threads
148+
backendMock.expectWorks(
149+
Book.NAME, DocumentCommitStrategy.NONE, DocumentRefreshStrategy.NONE
150+
)
151+
.add( "1", b -> b
152+
.field( "title", TITLE_1 )
153+
.field( "author", AUTHOR_1 )
154+
)
155+
.add( "3", b -> b
156+
.field( "title", TITLE_3 )
157+
.field( "author", AUTHOR_3 )
158+
)
159+
.createAndExecuteFollowingWorks( indexingFuture )
160+
.add( "2", b -> b
161+
.field( "title", TITLE_2 )
162+
.field( "author", AUTHOR_2 )
163+
);
164+
165+
// purgeAtStart and mergeSegmentsAfterPurge are enabled by default,
166+
// so we expect 1 purge, 1 mergeSegments and 1 flush calls in this order:
167+
backendMock.expectIndexScaleWorks( Book.NAME, searchSession.tenantIdentifierValue() )
168+
.purge()
169+
.mergeSegments()
170+
.flush()
171+
.refresh();
172+
173+
try {
174+
indexer.monitor( new StaticCountersMonitor( false ) )
175+
.startAndWait();
176+
}
177+
catch (SearchException ignored) {
178+
// Expected, but not relevant to this test
179+
}
180+
catch (InterruptedException e) {
181+
fail( "Unexpected InterruptedException: " + e.getMessage() );
182+
}
183+
}
184+
185+
backendMock.verifyExpectationsMet();
186+
187+
assertThat( staticCounters.get( StaticCountersMonitor.LOADED ) ).isEqualTo( 3 );
188+
assertThat( staticCounters.get( StaticCountersMonitor.BUILT ) ).isEqualTo( 3 );
189+
assertThat( staticCounters.get( StaticCountersMonitor.ADDED ) ).isEqualTo( 2 );
190+
assertThat( staticCounters.get( StaticCountersMonitor.TOTAL ) ).isEqualTo( 0 );
191+
assertThat( staticCounters.get( StaticCountersMonitor.INDEXING_COMPLETED ) ).isEqualTo( 1 );
192+
}
193+
131194
private void initData() {
132195
persist( new Book( 1, TITLE_1, AUTHOR_1 ) );
133196
persist( new Book( 2, TITLE_2, AUTHOR_2 ) );
@@ -178,6 +241,7 @@ public String getAuthor() {
178241
}
179242
}
180243

244+
@SuppressWarnings("removal")
181245
public static class StaticCountersMonitor implements MassIndexingMonitor {
182246

183247
public static StaticCounters.Key ADDED = StaticCounters.createKey();
@@ -186,6 +250,42 @@ public static class StaticCountersMonitor implements MassIndexingMonitor {
186250
public static StaticCounters.Key TOTAL = StaticCounters.createKey();
187251
public static StaticCounters.Key INDEXING_COMPLETED = StaticCounters.createKey();
188252

253+
254+
private final boolean requiresTotalCount;
255+
256+
public StaticCountersMonitor() {
257+
this( true );
258+
}
259+
260+
public StaticCountersMonitor(boolean requiresTotalCount) {
261+
this.requiresTotalCount = requiresTotalCount;
262+
}
263+
264+
@Override
265+
public MassIndexingTypeGroupMonitor typeGroupMonitor(MassIndexingTypeGroupMonitorCreateContext context) {
266+
if ( requiresTotalCount ) {
267+
return MassIndexingMonitor.super.typeGroupMonitor( context );
268+
}
269+
else {
270+
return new MassIndexingTypeGroupMonitor() {
271+
@Override
272+
public void documentsIndexed(long increment) {
273+
// do nothing
274+
}
275+
276+
@Override
277+
public void indexingStarted(MassIndexingTypeGroupMonitorContext context) {
278+
// do nothing
279+
}
280+
281+
@Override
282+
public void indexingCompleted(MassIndexingTypeGroupMonitorContext context) {
283+
// do nothing
284+
}
285+
};
286+
}
287+
}
288+
189289
@Override
190290
public void documentsAdded(long increment) {
191291
StaticCounters.get().add( ADDED, (int) increment );

‎mapper/orm/src/main/java/org/hibernate/search/mapper/orm/massindexing/MassIndexer.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,8 @@ public interface MassIndexer {
228228
* <p>
229229
* With fail-fast option enabled, the mass indexer will request cancelling all internal mass-indexing processes
230230
* right after the first error is reported to the {@link MassIndexingFailureHandler}.
231-
*
231+
* <p>
232+
* Defaults to {@code false}.
232233
* @param failFast Whether to enabled fail fast option for this mass indexer.
233234
*
234235
* @return {@code this} for method chaining

‎mapper/pojo-base/src/main/java/org/hibernate/search/mapper/pojo/logging/impl/Log.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1041,4 +1041,9 @@ void indexingProgressWithRemainingTime(float estimatePercentileComplete, long do
10411041
@LogMessage(level = INFO)
10421042
@Message(id = ID_OFFSET + 168, value = "Mass indexing complete in %3$s. Indexed %1$d/%2$d entities.")
10431043
void indexingEntitiesCompleted(long indexed, long total, Duration indexingTime);
1044+
1045+
@LogMessage(level = INFO)
1046+
@Message(id = ID_OFFSET + 169,
1047+
value = "Mass indexing is going to index approx. %1$d entities (%2$s). Actual number may change once the indexing starts.")
1048+
void indexingEntitiesApprox(long count, String types);
10441049
}

0 commit comments

Comments
 (0)