Skip to content
This repository was archived by the owner on Jul 18, 2025. It is now read-only.

Commit fef5512

Browse files
StefanBratanovStefanBratanov
committed
Upgrade the Moderation API with the new multimodal moderation model
1 parent 5f9aa5c commit fef5512

File tree

5 files changed

+116
-17
lines changed

5 files changed

+116
-17
lines changed

‎src/main/java/io/github/stefanbratanov/jvm/openai/Moderation.java

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,19 @@
66
/** Represents if a given text input is potentially harmful. */
77
public record Moderation(String id, String model, List<Result> results) {
88

9-
public record Result(boolean flagged, Categories categories, CategoryScores categoryScores) {
9+
public record Result(
10+
boolean flagged,
11+
Categories categories,
12+
CategoryScores categoryScores,
13+
CategoryAppliedInputTypes categoryAppliedInputTypes) {
1014

1115
public record Categories(
1216
boolean hate,
1317
@JsonProperty("hate/threatening") boolean hateThreatening,
1418
boolean harassment,
1519
@JsonProperty("harassment/threatening") boolean harassmentThreatening,
20+
boolean illicit,
21+
@JsonProperty("illicit/violent") boolean illicitViolent,
1622
@JsonProperty("self-harm") boolean selfHarm,
1723
@JsonProperty("self-harm/intent") boolean selfHarmIntent,
1824
@JsonProperty("self-harm/instructions") boolean selfHarmInstructions,
@@ -26,12 +32,29 @@ public record CategoryScores(
2632
@JsonProperty("hate/threatening") Double hateThreatening,
2733
Double harassment,
2834
@JsonProperty("harassment/threatening") Double harassmentThreatening,
35+
Double illicit,
36+
@JsonProperty("illicit/violent") Double illicitViolent,
2937
@JsonProperty("self-harm") Double selfHarm,
3038
@JsonProperty("self-harm/intent") Double selfHarmIntent,
3139
@JsonProperty("self-harm/instructions") Double selfHarmInstructions,
3240
Double sexual,
3341
@JsonProperty("sexual/minors") Double sexualMinors,
3442
Double violence,
3543
@JsonProperty("violence/graphic") Double violenceGraphic) {}
44+
45+
public record CategoryAppliedInputTypes(
46+
List<String> hate,
47+
@JsonProperty("hate/threatening") List<String> hateThreatening,
48+
List<String> harassment,
49+
@JsonProperty("harassment/threatening") List<String> harassmentThreatening,
50+
List<String> illicit,
51+
@JsonProperty("illicit/violent") List<String> illicitViolent,
52+
@JsonProperty("self-harm") List<String> selfHarm,
53+
@JsonProperty("self-harm/intent") List<String> selfHarmIntent,
54+
@JsonProperty("self-harm/instructions") List<String> selfHarmInstructions,
55+
List<String> sexual,
56+
@JsonProperty("sexual/minors") List<String> sexualMinors,
57+
List<String> violence,
58+
@JsonProperty("violence/graphic") List<String> violenceGraphic) {}
3659
}
3760
}

‎src/main/java/io/github/stefanbratanov/jvm/openai/ModerationRequest.java

Lines changed: 59 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,106 @@
11
package io.github.stefanbratanov.jvm.openai;
22

3+
import com.fasterxml.jackson.annotation.JsonProperty;
4+
import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput;
5+
import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput.ImageUrl;
6+
import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.TextInput;
37
import java.util.LinkedList;
48
import java.util.List;
59
import java.util.Optional;
610

7-
public record ModerationRequest(List<String> input, Optional<String> model) {
11+
public record ModerationRequest(List<Object> input, Optional<String> model) {
812

913
public static Builder newBuilder() {
1014
return new Builder();
1115
}
1216

1317
public static class Builder {
1418

15-
private final List<String> input = new LinkedList<>();
19+
private final List<Object> input = new LinkedList<>();
1620

1721
private Optional<String> model = Optional.empty();
1822

1923
/**
20-
* @param input input to append to the list of input texts to classify
24+
* @param input a string of text to append to the list of inputs to classify for moderation
2125
*/
2226
public Builder input(String input) {
2327
this.input.add(input);
2428
return this;
2529
}
2630

2731
/**
28-
* @param inputs inputs to append to the list of input texts to classify
32+
* @param inputs an array of strings to append to the list of inputs to classify for moderation
2933
*/
3034
public Builder inputs(List<String> inputs) {
3135
input.addAll(inputs);
3236
return this;
3337
}
3438

3539
/**
36-
* @param model Two content moderations models are available: text-moderation-stable and
37-
* text-moderation-latest.
38-
* <p>The default is text-moderation-latest which will be automatically upgraded over time.
39-
* This ensures you are always using our most accurate model. If you use
40-
* text-moderation-stable, we will provide advanced notice before updating the model.
41-
* Accuracy of text-moderation-stable may be slightly lower than for text-moderation-latest.
40+
* @param input multi-modal input to append to the list of inputs to classify for moderation
41+
*/
42+
public Builder multiModalInput(MultiModalInput input) {
43+
this.input.add(input);
44+
return this;
45+
}
46+
47+
/**
48+
* @param inputs an array of multi-modal inputs to append to the list of inputs to classify for
49+
* moderation
50+
*/
51+
public Builder multiModalInputs(List<MultiModalInput> inputs) {
52+
this.input.addAll(inputs);
53+
return this;
54+
}
55+
56+
/**
57+
* @param model The content moderation model you would like to use.
4258
*/
4359
public Builder model(String model) {
4460
this.model = Optional.of(model);
4561
return this;
4662
}
4763

4864
/**
49-
* @param model Two content moderations models are available: {@link
50-
* OpenAIModel#TEXT_MODERATION_LATEST} and {@link OpenAIModel#TEXT_MODERATION_STABLE}.
65+
* @param model The content moderation {@link OpenAIModel} you would like to use.
5166
*/
5267
public Builder model(OpenAIModel model) {
5368
this.model = Optional.of(model.getId());
5469
return this;
5570
}
5671

72+
public sealed interface MultiModalInput permits ImageUrlInput, TextInput {
73+
74+
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
75+
String type();
76+
77+
record ImageUrlInput(ImageUrl imageUrl) implements MultiModalInput {
78+
79+
@Override
80+
public String type() {
81+
return "image_url";
82+
}
83+
84+
public record ImageUrl(String url) {}
85+
}
86+
87+
record TextInput(String text) implements MultiModalInput {
88+
89+
@Override
90+
public String type() {
91+
return "text";
92+
}
93+
}
94+
95+
static ImageUrlInput imageUrl(ImageUrl imageUrl) {
96+
return new ImageUrlInput(imageUrl);
97+
}
98+
99+
static TextInput text(String text) {
100+
return new TextInput(text);
101+
}
102+
}
103+
57104
public ModerationRequest build() {
58105
return new ModerationRequest(List.copyOf(input), model);
59106
}

‎src/main/java/io/github/stefanbratanov/jvm/openai/ModerationsClient.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import java.util.Optional;
99

1010
/**
11-
* Given some input text, outputs if the model classifies it as potentially harmful across several
11+
* Given text and/or image inputs, classifies if those inputs are potentially harmful across several
1212
* categories.
1313
*
1414
* <p>Based on <a href="https://platform.openai.com/docs/api-reference/moderations">Moderations</a>
@@ -27,7 +27,7 @@ public final class ModerationsClient extends OpenAIClient {
2727
}
2828

2929
/**
30-
* Classifies if text is potentially harmful.
30+
* Classifies if text and/or image inputs are potentially harmful.
3131
*
3232
* @throws OpenAIException in case of API errors
3333
*/

‎src/main/java/io/github/stefanbratanov/jvm/openai/OpenAIModel.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ public enum OpenAIModel {
5353
TEXT_EMBEDDING_ADA_002("text-embedding-ada-002"),
5454

5555
// Moderation (https://platform.openai.com/docs/models/moderation)
56+
OMNI_MODERATION_LATEST("omni-moderation-latest"),
5657
TEXT_MODERATION_LATEST("text-moderation-latest"),
5758
TEXT_MODERATION_STABLE("text-moderation-stable");
5859

‎src/test/java/io/github/stefanbratanov/jvm/openai/TestDataUtil.java

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import io.github.stefanbratanov.jvm.openai.CompletionUsage.CompletionTokensDetails;
2626
import io.github.stefanbratanov.jvm.openai.CreateChatCompletionRequest.StreamOptions;
2727
import io.github.stefanbratanov.jvm.openai.FineTuningJobIntegration.Wandb;
28+
import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput;
29+
import io.github.stefanbratanov.jvm.openai.ModerationRequest.Builder.MultiModalInput.ImageUrlInput.ImageUrl;
2830
import io.github.stefanbratanov.jvm.openai.ProjectApiKey.Owner;
2931
import io.github.stefanbratanov.jvm.openai.ProjectApiKeysClient.PaginatedProjectApiKeys;
3032
import io.github.stefanbratanov.jvm.openai.ProjectServiceAccountsClient.ApiKey;
@@ -343,12 +345,20 @@ public ModerationRequest randomModerationRequest() {
343345
ModerationRequest.Builder builder = ModerationRequest.newBuilder();
344346
runOne(
345347
() -> builder.input(randomString(10)),
346-
() -> builder.inputs(listOf(randomInt(1, 5), () -> randomString(10))));
348+
() -> builder.inputs(listOf(randomInt(1, 5), () -> randomString(10))),
349+
() -> builder.multiModalInput(randomMultiModalInput()),
350+
() -> builder.multiModalInputs(listOf(randomInt(1, 5), this::randomMultiModalInput)));
347351
return builder
348352
.model(oneOf(OpenAIModel.TEXT_MODERATION_LATEST, OpenAIModel.TEXT_MODERATION_STABLE))
349353
.build();
350354
}
351355

356+
public MultiModalInput randomMultiModalInput() {
357+
return oneOf(
358+
MultiModalInput.imageUrl(new ImageUrl("https://example.com/image.jpg")),
359+
MultiModalInput.text(randomString(10)));
360+
}
361+
352362
public Moderation randomModeration() {
353363
return new Moderation(
354364
randomString(6),
@@ -369,6 +379,8 @@ public Moderation randomModeration() {
369379
randomBoolean(),
370380
randomBoolean(),
371381
randomBoolean(),
382+
randomBoolean(),
383+
randomBoolean(),
372384
randomBoolean()),
373385
new Moderation.Result.CategoryScores(
374386
randomDouble(),
@@ -381,7 +393,23 @@ public Moderation randomModeration() {
381393
randomDouble(),
382394
randomDouble(),
383395
randomDouble(),
384-
randomDouble()))));
396+
randomDouble(),
397+
randomDouble(),
398+
randomDouble()),
399+
new Moderation.Result.CategoryAppliedInputTypes(
400+
List.of("text"),
401+
List.of("text"),
402+
List.of("text"),
403+
List.of("text"),
404+
List.of("text"),
405+
List.of("text"),
406+
List.of("text", "image"),
407+
List.of("text", "image"),
408+
List.of("text", "image"),
409+
List.of("text", "image"),
410+
List.of("text"),
411+
List.of("text", "image"),
412+
List.of("text", "image")))));
385413
}
386414

387415
public CreateAssistantRequest randomCreateAssistantRequest() {

0 commit comments

Comments
 (0)