Skip to content

Commit 2f4ef92

Browse files
ja-openaija-openai
committed
Add a CLI and service to bulk review translations in a repository
- Refactor the current code that reviews a single string to read from the database if a review for a translation already exists. - Query translations that have not yet been reviewed, perform the bulk review, and save the result in AiReviewProto as a JSON blob that will be processed using MySQL JSON queries instead of separate columns. An option to remove an existing review can be added later, but for now this can be done directly in the database.
1 parent e47c7e7 commit 2f4ef92

File tree

11 files changed

+432
-426
lines changed

11 files changed

+432
-426
lines changed

‎cli/src/main/java/com/box/l10n/mojito/cli/command/RepositoryAiReviewCommand.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package com.box.l10n.mojito.cli.command;
2+
3+
import com.beust.jcommander.Parameter;
4+
import com.beust.jcommander.Parameters;
5+
import com.box.l10n.mojito.cli.command.param.Param;
6+
import com.box.l10n.mojito.cli.console.ConsoleWriter;
7+
import com.box.l10n.mojito.rest.client.RepositoryAiReviewClient;
8+
import com.box.l10n.mojito.rest.entity.PollableTask;
9+
import java.util.List;
10+
import java.util.stream.Collectors;
11+
import org.fusesource.jansi.Ansi.Color;
12+
import org.slf4j.Logger;
13+
import org.slf4j.LoggerFactory;
14+
import org.springframework.beans.factory.annotation.Autowired;
15+
import org.springframework.context.annotation.Scope;
16+
import org.springframework.stereotype.Component;
17+
18+
/**
19+
* Command to machine review strings in a repository.
20+
*
21+
* @author jaurambault
22+
*/
23+
@Component
24+
@Scope("prototype")
25+
@Parameters(
26+
commandNames = {"repository-ai-review"},
27+
commandDescription = "Ai review translated strings in a repository")
28+
public class RepositoryAiReviewCommand extends Command {
29+
30+
/** logger */
31+
static Logger logger = LoggerFactory.getLogger(RepositoryAiReviewCommand.class);
32+
33+
@Autowired ConsoleWriter consoleWriter;
34+
35+
@Parameter(
36+
names = {Param.REPOSITORY_LONG, Param.REPOSITORY_SHORT},
37+
arity = 1,
38+
required = true,
39+
description = Param.REPOSITORY_DESCRIPTION)
40+
String repositoryParam;
41+
42+
@Parameter(
43+
names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT},
44+
variableArity = true,
45+
description =
46+
"List of locales (bcp47 tags) to translate, if not provided translate all locales in the repository")
47+
List<String> locales;
48+
49+
@Parameter(
50+
names = {"--source-text-max-count"},
51+
arity = 1,
52+
description =
53+
"Source text max count per locale sent to MT (this param is used to avoid "
54+
+ "sending too many strings to MT)")
55+
int sourceTextMaxCount = 100;
56+
57+
@Parameter(
58+
names = {"--text-unit-ids"},
59+
arity = 1,
60+
description = "The list of TmTextUnitIds to translate")
61+
List<Long> textUnitIds;
62+
63+
@Parameter(
64+
names = {"--use-batch"},
65+
arity = 1,
66+
description = "To use the batch API or not")
67+
boolean useBatch = false;
68+
69+
@Autowired CommandHelper commandHelper;
70+
71+
@Autowired RepositoryAiReviewClient repositoryAiReviewClient;
72+
73+
@Override
74+
public boolean shouldShowInCommandList() {
75+
return false;
76+
}
77+
78+
@Override
79+
public void execute() throws CommandException {
80+
81+
consoleWriter
82+
.newLine()
83+
.a("Ai review repository: ")
84+
.fg(Color.CYAN)
85+
.a(repositoryParam)
86+
.reset()
87+
.a(" for locales: ")
88+
.fg(Color.CYAN)
89+
.a(locales == null ? "<all>" : locales.stream().collect(Collectors.joining(", ", "[", "]")))
90+
.println(2);
91+
92+
RepositoryAiReviewClient.ProtoAiReviewResponse protoAiTranslateResponse =
93+
repositoryAiReviewClient.reviewRepository(
94+
new RepositoryAiReviewClient.ProtoAiReviewRequest(
95+
repositoryParam, locales, sourceTextMaxCount, textUnitIds, useBatch));
96+
97+
PollableTask pollableTask = protoAiTranslateResponse.pollableTask();
98+
commandHelper.waitForPollableTask(pollableTask.getId());
99+
}
100+
}

‎restclient/src/main/java/com/box/l10n/mojito/rest/client/RepositoryAiReviewClient.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package com.box.l10n.mojito.rest.client;
2+
3+
import com.box.l10n.mojito.rest.entity.PollableTask;
4+
import java.util.List;
5+
import org.slf4j.Logger;
6+
import org.slf4j.LoggerFactory;
7+
import org.springframework.stereotype.Component;
8+
9+
/**
10+
* @author jaurambault
11+
*/
12+
@Component
13+
public class RepositoryAiReviewClient extends BaseClient {
14+
15+
/** logger */
16+
static Logger logger = LoggerFactory.getLogger(RepositoryAiReviewClient.class);
17+
18+
@Override
19+
public String getEntityName() {
20+
return "proto-ai-review";
21+
}
22+
23+
/** Ai review strings in a repository for a given list of locales */
24+
public ProtoAiReviewResponse reviewRepository(ProtoAiReviewRequest protoAiReviewRequest) {
25+
26+
return authenticatedRestTemplate.postForObject(
27+
getBasePathForEntity(), protoAiReviewRequest, ProtoAiReviewResponse.class);
28+
}
29+
30+
public record ProtoAiReviewRequest(
31+
String repositoryName,
32+
List<String> targetBcp47tags,
33+
int sourceTextMaxCountPerLocale,
34+
List<Long> tmTextUnitIds,
35+
boolean useBatch) {}
36+
37+
public record ProtoAiReviewResponse(PollableTask pollableTask) {}
38+
}

‎webapp/src/main/java/com/box/l10n/mojito/entity/AiReviewProto.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.box.l10n.mojito.entity;
2+
3+
import jakarta.persistence.Column;
4+
import jakarta.persistence.Entity;
5+
import jakarta.persistence.FetchType;
6+
import jakarta.persistence.ForeignKey;
7+
import jakarta.persistence.Index;
8+
import jakarta.persistence.JoinColumn;
9+
import jakarta.persistence.OneToOne;
10+
import jakarta.persistence.Table;
11+
12+
/**
13+
* We keep a single review per text unit variant for now. Meaning it has to be updated / cleaned up
14+
* to re-review. The review is stored a JSON blob defined in the {@link
15+
* com.box.l10n.mojito.service.oaireview.AiReviewService.AiReviewSingleTextUnitOutput} but that
16+
* format could change any time.
17+
*/
18+
@Entity
19+
@Table(
20+
name = "ai_review_proto",
21+
indexes = {
22+
@Index(
23+
name = "UK__AI_REVIEW_PROTO__TM_TEXT_UNIT_VARIANT_ID",
24+
columnList = "tm_text_unit_variant_id",
25+
unique = true)
26+
})
27+
public class AiReviewProto extends AuditableEntity {
28+
29+
@OneToOne(fetch = FetchType.LAZY)
30+
@JoinColumn(
31+
name = "tm_text_unit_variant_id",
32+
foreignKey = @ForeignKey(name = "FK__AI_REVIEW_PROTO__TM_TEXT_UNIT_VARIANT__ID"))
33+
TMTextUnitVariant tmTextUnitVariant;
34+
35+
@Column(name = "json_review", length = Integer.MAX_VALUE)
36+
String jsonReview;
37+
38+
public TMTextUnitVariant getTmTextUnitVariant() {
39+
return tmTextUnitVariant;
40+
}
41+
42+
public void setTmTextUnitVariant(TMTextUnitVariant tmTextUnitVariant) {
43+
this.tmTextUnitVariant = tmTextUnitVariant;
44+
}
45+
46+
public String getJsonReview() {
47+
return jsonReview;
48+
}
49+
50+
public void setJsonReview(String jsonReview) {
51+
this.jsonReview = jsonReview;
52+
}
53+
}

‎webapp/src/main/java/com/box/l10n/mojito/rest/textunit/AiReviewWS.java‎

Lines changed: 84 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,20 @@
11
package com.box.l10n.mojito.rest.textunit;
22

3+
import com.box.l10n.mojito.entity.AiReviewProto;
4+
import com.box.l10n.mojito.entity.PollableTask;
5+
import com.box.l10n.mojito.json.ObjectMapper;
36
import com.box.l10n.mojito.service.oaireview.AiReviewService;
7+
import com.box.l10n.mojito.service.pollableTask.PollableFuture;
8+
import com.box.l10n.mojito.service.tm.AiReviewProtoRepository;
49
import com.box.l10n.mojito.service.tm.search.TextUnitDTO;
510
import com.box.l10n.mojito.service.tm.search.TextUnitSearcher;
611
import com.box.l10n.mojito.service.tm.search.TextUnitSearcherParameters;
7-
812
import java.util.List;
913
import org.slf4j.Logger;
1014
import org.slf4j.LoggerFactory;
11-
import org.springframework.beans.factory.annotation.Autowired;
15+
import org.springframework.beans.factory.annotation.Qualifier;
1216
import org.springframework.http.HttpStatus;
17+
import org.springframework.web.bind.annotation.RequestBody;
1318
import org.springframework.web.bind.annotation.RequestMapping;
1419
import org.springframework.web.bind.annotation.RequestMethod;
1520
import org.springframework.web.bind.annotation.ResponseStatus;
@@ -21,24 +26,62 @@ public class AiReviewWS {
2126
/** logger */
2227
static Logger logger = LoggerFactory.getLogger(AiReviewWS.class);
2328

29+
private final AiReviewProtoRepository aiReviewProtoRepository;
30+
private final ObjectMapper objectMapper;
31+
2432
TextUnitSearcher textUnitSearcher;
2533

26-
@Autowired
2734
AiReviewService aiReviewService;
2835

36+
AiReviewProtoRepository AiReviewProtoRepository;
37+
2938
public AiReviewWS(
3039
TextUnitSearcher textUnitSearcher,
31-
AiReviewService aiReviewService) {
40+
AiReviewService aiReviewService,
41+
AiReviewProtoRepository AiReviewProtoRepository,
42+
AiReviewProtoRepository aiReviewProtoRepository,
43+
@Qualifier("AiTranslate") ObjectMapper objectMapper) {
3244
this.textUnitSearcher = textUnitSearcher;
3345
this.aiReviewService = aiReviewService;
46+
this.AiReviewProtoRepository = AiReviewProtoRepository;
47+
this.aiReviewProtoRepository = aiReviewProtoRepository;
48+
this.objectMapper = objectMapper;
49+
}
50+
51+
@RequestMapping(method = RequestMethod.POST, value = "/api/proto-ai-review")
52+
@ResponseStatus(HttpStatus.OK)
53+
public ProtoAiReviewResponse aiReview(@RequestBody ProtoAiReviewRequest protoAiReviewRequest) {
54+
55+
PollableFuture<Void> pollableFuture =
56+
aiReviewService.aiReviewAsync(
57+
new AiReviewService.AiReviewInput(
58+
protoAiReviewRequest.repositoryName(),
59+
protoAiReviewRequest.targetBcp47tags(),
60+
protoAiReviewRequest.sourceTextMaxCountPerLocale(),
61+
protoAiReviewRequest.tmTextUnitIds(),
62+
protoAiReviewRequest.useBatch()));
63+
64+
return new ProtoAiReviewResponse(pollableFuture.getPollableTask());
3465
}
3566

36-
@RequestMapping(method = RequestMethod.GET, value = "/api/proto-ai-review")
67+
public record ProtoAiReviewRequest(
68+
String repositoryName,
69+
List<String> targetBcp47tags,
70+
int sourceTextMaxCountPerLocale,
71+
boolean useBatch,
72+
List<Long> tmTextUnitIds,
73+
boolean allLocales) {}
74+
75+
public record ProtoAiReviewResponse(PollableTask pollableTask) {}
76+
77+
@RequestMapping(method = RequestMethod.GET, value = "/api/proto-ai-review-single-text-unit")
3778
@ResponseStatus(HttpStatus.OK)
38-
public ProtoAiReviewResponse getTextUnitsWithGet(ProtoAiReviewRequest protoAiReviewRequest) {
79+
public ProtoAiReviewSingleTextUnitResponse getTextUnitsWithGet(
80+
ProtoAiReviewSingleTextUnitRequest protoAiReviewSingleTextUnitRequest) {
3981

4082
TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters();
41-
textUnitSearcherParameters.setTmTextUnitVariantId(protoAiReviewRequest.tmTextUnitVariantId);
83+
textUnitSearcherParameters.setTmTextUnitVariantId(
84+
protoAiReviewSingleTextUnitRequest.tmTextUnitVariantId);
4285

4386
List<TextUnitDTO> search = textUnitSearcher.search(textUnitSearcherParameters);
4487
if (search.isEmpty()) {
@@ -47,20 +90,43 @@ public ProtoAiReviewResponse getTextUnitsWithGet(ProtoAiReviewRequest protoAiRev
4790

4891
TextUnitDTO textUnit = search.getFirst();
4992

50-
AiReviewService.AiReviewSingleTextUnitInput input =
51-
new AiReviewService.AiReviewSingleTextUnitInput(
52-
textUnit.getTargetLocale(),
53-
textUnit.getSource(),
54-
textUnit.getComment(),
55-
new AiReviewService.AiReviewSingleTextUnitInput.ExistingTarget(
56-
textUnit.getTarget(), !textUnit.isIncludedInLocalizedFile()));
93+
logger.info("Check for pre-computed review");
5794

58-
AiReviewService.AiReviewSingleTextUnitOutput aiReviewSingleTextUnitOutput = aiReviewService.getAiReviewSingleTextUnit(input, textUnit);
95+
AiReviewProto alreadyReviewed =
96+
aiReviewProtoRepository.findByTmTextUnitVariantId(
97+
protoAiReviewSingleTextUnitRequest.tmTextUnitVariantId());
98+
99+
AiReviewService.AiReviewSingleTextUnitOutput aiReviewSingleTextUnitOutput = null;
100+
101+
if (alreadyReviewed != null) {
102+
try {
103+
aiReviewSingleTextUnitOutput =
104+
objectMapper.readValueUnchecked(
105+
alreadyReviewed.getJsonReview(),
106+
AiReviewService.AiReviewSingleTextUnitOutput.class);
107+
} catch (RuntimeException e) {
108+
logger.warn("Can't deserialize the existing review, we will recompute");
109+
}
110+
}
111+
112+
if (aiReviewSingleTextUnitOutput == null) {
113+
114+
AiReviewService.AiReviewSingleTextUnitInput input =
115+
new AiReviewService.AiReviewSingleTextUnitInput(
116+
textUnit.getTargetLocale(),
117+
textUnit.getSource(),
118+
textUnit.getComment(),
119+
new AiReviewService.AiReviewSingleTextUnitInput.ExistingTarget(
120+
textUnit.getTarget(), !textUnit.isIncludedInLocalizedFile()));
121+
122+
aiReviewSingleTextUnitOutput = aiReviewService.getAiReviewSingleTextUnit(input);
123+
}
59124

60-
return new ProtoAiReviewResponse(textUnit, aiReviewSingleTextUnitOutput);
125+
return new ProtoAiReviewSingleTextUnitResponse(textUnit, aiReviewSingleTextUnitOutput);
61126
}
62127

63-
public record ProtoAiReviewRequest(long tmTextUnitVariantId) {}
128+
public record ProtoAiReviewSingleTextUnitRequest(long tmTextUnitVariantId) {}
64129

65-
public record ProtoAiReviewResponse(TextUnitDTO textUnitDTO, AiReviewService.AiReviewSingleTextUnitOutput aiReviewOutput) {}
130+
public record ProtoAiReviewSingleTextUnitResponse(
131+
TextUnitDTO textUnitDTO, AiReviewService.AiReviewSingleTextUnitOutput aiReviewOutput) {}
66132
}

‎webapp/src/main/java/com/box/l10n/mojito/service/oaireview/AiReviewConfig.java‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,12 @@
33
import com.box.l10n.mojito.json.ObjectMapper;
44
import com.box.l10n.mojito.openai.OpenAIClient;
55
import com.box.l10n.mojito.openai.OpenAIClientPool;
6-
import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler;
7-
import com.box.l10n.mojito.service.blobstorage.StructuredBlobStorage;
8-
import com.box.l10n.mojito.service.repository.RepositoryRepository;
9-
import com.box.l10n.mojito.service.repository.RepositoryService;
10-
import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService;
11-
import com.box.l10n.mojito.service.tm.search.TextUnitSearcher;
6+
import java.time.Duration;
127
import org.springframework.context.annotation.Bean;
138
import org.springframework.context.annotation.Configuration;
149
import reactor.util.retry.Retry;
1510
import reactor.util.retry.RetryBackoffSpec;
1611

17-
import java.time.Duration;
18-
1912
@Configuration
2013
public class AiReviewConfig {
2114

@@ -25,28 +18,6 @@ public AiReviewConfig(AiReviewConfigurationProperties aiReviewConfigurationPrope
2518
this.aiReviewConfigurationProperties = aiReviewConfigurationProperties;
2619
}
2720

28-
// TextUnitSearcher textUnitSearcher;
29-
// RepositoryRepository repositoryRepository;
30-
// RepositoryService repositoryService;
31-
// TextUnitBatchImporterService textUnitBatchImporterService;
32-
// StructuredBlobStorage structuredBlobStorage;
33-
// QuartzPollableTaskScheduler quartzPollableTaskScheduler;
34-
//
35-
// public AiReviewConfig(AiReviewConfigurationProperties aiReviewConfigurationProperties, TextUnitSearcher textUnitSearcher, RepositoryRepository repositoryRepository, RepositoryService repositoryService, TextUnitBatchImporterService textUnitBatchImporterService, StructuredBlobStorage structuredBlobStorage, QuartzPollableTaskScheduler quartzPollableTaskScheduler) {
36-
// this.aiReviewConfigurationProperties = aiReviewConfigurationProperties;
37-
// this.textUnitSearcher = textUnitSearcher;
38-
// this.repositoryRepository = repositoryRepository;
39-
// this.repositoryService = repositoryService;
40-
// this.textUnitBatchImporterService = textUnitBatchImporterService;
41-
// this.structuredBlobStorage = structuredBlobStorage;
42-
// this.quartzPollableTaskScheduler = quartzPollableTaskScheduler;
43-
// }
44-
//
45-
// @Bean
46-
// AiReviewService aiReviewService() {
47-
// return new AiReviewService(textUnitSearcher, repositoryRepository, repositoryService, textUnitBatchImporterService, structuredBlobStorage, aiReviewConfigurationProperties, openAIClient(), openAIClientPool(), objectMapper(), retryBackoffSpec(), quartzPollableTaskScheduler);
48-
// }
49-
5021
@Bean("openAIClientReview")
5122
OpenAIClient openAIClient() {
5223
String openaiClientToken = aiReviewConfigurationProperties.getOpenaiClientToken();
@@ -62,8 +33,7 @@ OpenAIClientPool openAIClientPool() {
6233
if (openaiClientToken == null) {
6334
return null;
6435
}
65-
return new OpenAIClientPool(
66-
10, 50, 5, aiReviewConfigurationProperties.getOpenaiClientToken());
36+
return new OpenAIClientPool(10, 50, 5, aiReviewConfigurationProperties.getOpenaiClientToken());
6737
}
6838

6939
@Bean("objectMapperReview")

0 commit comments

Comments
 (0)