Skip to content

Commit

Permalink
Add a CLI and service to bulk review translations in a repository
Browse files Browse the repository at this point in the history
- Refactor the current code that reviews a single string to read from the database if a review for a translation already exists.
- Query translations that have not yet been reviewed, perform the bulk review, and save the result in AiReviewProto as a JSON blob that will be processed using MySQL JSON queries instead of separate columns. An option to remove an existing review can be added later, but for now this can be done directly in the database.
  • Loading branch information
ja-openai committed Feb 18, 2025
1 parent 3121267 commit 23889ec
Show file tree
Hide file tree
Showing 13 changed files with 931 additions and 157 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package com.box.l10n.mojito.cli.command;

import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.box.l10n.mojito.cli.command.param.Param;
import com.box.l10n.mojito.cli.console.ConsoleWriter;
import com.box.l10n.mojito.rest.client.RepositoryAiReviewClient;
import com.box.l10n.mojito.rest.entity.PollableTask;
import java.util.List;
import java.util.stream.Collectors;
import org.fusesource.jansi.Ansi.Color;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

/**
* Command to machine review strings in a repository.
*
* @author jaurambault
*/
@Component
@Scope("prototype")
@Parameters(
commandNames = {"repository-ai-review"},
commandDescription = "Ai review translated strings in a repository")
public class RepositoryAiReviewCommand extends Command {

/** logger */
static Logger logger = LoggerFactory.getLogger(RepositoryAiReviewCommand.class);

@Autowired ConsoleWriter consoleWriter;

@Parameter(
names = {Param.REPOSITORY_LONG, Param.REPOSITORY_SHORT},
arity = 1,
required = true,
description = Param.REPOSITORY_DESCRIPTION)
String repositoryParam;

@Parameter(
names = {Param.REPOSITORY_LOCALES_LONG, Param.REPOSITORY_LOCALES_SHORT},
variableArity = true,
description =
"List of locales (bcp47 tags) to translate, if not provided translate all locales in the repository")
List<String> locales;

@Parameter(
names = {"--source-text-max-count"},
arity = 1,
description =
"Source text max count per locale sent to MT (this param is used to avoid "
+ "sending too many strings to MT)")
int sourceTextMaxCount = 100;

@Parameter(
names = {"--text-unit-ids"},
arity = 1,
description = "The list of TmTextUnitIds to translate")
List<Long> textUnitIds;

@Parameter(
names = {"--use-batch"},
arity = 1,
description = "To use the batch API or not")
boolean useBatch = false;

@Autowired CommandHelper commandHelper;

@Autowired RepositoryAiReviewClient repositoryAiReviewClient;

@Override
public boolean shouldShowInCommandList() {
return false;
}

@Override
public void execute() throws CommandException {

consoleWriter
.newLine()
.a("Ai review repository: ")
.fg(Color.CYAN)
.a(repositoryParam)
.reset()
.a(" for locales: ")
.fg(Color.CYAN)
.a(locales == null ? "<all>" : locales.stream().collect(Collectors.joining(", ", "[", "]")))
.println(2);

RepositoryAiReviewClient.ProtoAiReviewResponse protoAiTranslateResponse =
repositoryAiReviewClient.reviewRepository(
new RepositoryAiReviewClient.ProtoAiReviewRequest(
repositoryParam, locales, sourceTextMaxCount, textUnitIds, useBatch));

PollableTask pollableTask = protoAiTranslateResponse.pollableTask();
commandHelper.waitForPollableTask(pollableTask.getId());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package com.box.l10n.mojito.rest.client;

import com.box.l10n.mojito.rest.entity.PollableTask;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

/**
* @author jaurambault
*/
@Component
public class RepositoryAiReviewClient extends BaseClient {

/** logger */
static Logger logger = LoggerFactory.getLogger(RepositoryAiReviewClient.class);

@Override
public String getEntityName() {
return "proto-ai-review";
}

/** Ai review strings in a repository for a given list of locales */
public ProtoAiReviewResponse reviewRepository(ProtoAiReviewRequest protoAiReviewRequest) {

return authenticatedRestTemplate.postForObject(
getBasePathForEntity(), protoAiReviewRequest, ProtoAiReviewResponse.class);
}

public record ProtoAiReviewRequest(
String repositoryName,
List<String> targetBcp47tags,
int sourceTextMaxCountPerLocale,
List<Long> tmTextUnitIds,
boolean useBatch) {}

public record ProtoAiReviewResponse(PollableTask pollableTask) {}
}
53 changes: 53 additions & 0 deletions webapp/src/main/java/com/box/l10n/mojito/entity/AiReviewProto.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package com.box.l10n.mojito.entity;

import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.ForeignKey;
import jakarta.persistence.Index;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;

/**
* We keep a single review per text unit variant for now. Meaning it has to be updated / cleaned up
* to re-review. The review is stored a JSON blob defined in the {@link
* com.box.l10n.mojito.service.oaireview.AiReviewService.AiReviewSingleTextUnitOutput} but that
* format could change any time.
*/
@Entity
@Table(
name = "ai_review_proto",
indexes = {
@Index(
name = "UK__AI_REVIEW_PROTO__TM_TEXT_UNIT_VARIANT_ID",
columnList = "tm_text_unit_variant_id",
unique = true)
})
public class AiReviewProto extends AuditableEntity {

@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(
name = "tm_text_unit_variant_id",
foreignKey = @ForeignKey(name = "FK__AI_REVIEW_PROTO__TM_TEXT_UNIT_VARIANT__ID"))
TMTextUnitVariant tmTextUnitVariant;

@Column(name = "json_review", length = Integer.MAX_VALUE)
String jsonReview;

public TMTextUnitVariant getTmTextUnitVariant() {
return tmTextUnitVariant;
}

public void setTmTextUnitVariant(TMTextUnitVariant tmTextUnitVariant) {
this.tmTextUnitVariant = tmTextUnitVariant;
}

public String getJsonReview() {
return jsonReview;
}

public void setJsonReview(String jsonReview) {
this.jsonReview = jsonReview;
}
}

This file was deleted.

Loading

0 comments on commit 23889ec

Please sign in to comment.