Skip to content

Commit aa0b2d7

Browse files
dbpdbp
committed
Add configurable feedbot rate limiting
1 parent 52cb789 commit aa0b2d7

File tree

4 files changed

+133
-28
lines changed

4 files changed

+133
-28
lines changed

‎app/api/llm-hint/route.ts

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { ChatAnthropic } from "@langchain/anthropic";
66
import { ChatPromptTemplate } from "@langchain/core/prompts";
77

88
import * as Sentry from "@sentry/nextjs";
9-
import { GraderResultTestExtraData } from "@/utils/supabase/DatabaseTypes";
9+
import { GraderResultTestExtraData, LLMRateLimitConfig } from "@/utils/supabase/DatabaseTypes";
1010

1111
/**
1212
* Custom error class for errors that should be displayed to users
@@ -99,6 +99,81 @@ async function getPrompt(input: GraderResultTestExtraData["llm"]) {
9999
return ChatPromptTemplate.fromMessages([["human", input.prompt]]);
100100
}
101101

102+
async function checkRateLimits(
103+
testResult: any,
104+
rateLimit: LLMRateLimitConfig,
105+
serviceSupabase: any
106+
): Promise<string | null> {
107+
const submissionId = testResult.grader_results.submissions.id;
108+
const classId = testResult.class_id;
109+
const assignmentId = testResult.grader_results.submissions.assignment_id;
110+
111+
// Check cooldown (minutes since last inference on this assignment, excluding current submission)
112+
if (rateLimit.cooldown) {
113+
const { data: lastUsage } = await serviceSupabase
114+
.from("llm_inference_usage")
115+
.select(`
116+
created_at,
117+
submissions!inner (
118+
assignment_id
119+
)
120+
`)
121+
.eq("submissions.assignment_id", assignmentId)
122+
.neq("submission_id", submissionId)
123+
.order("created_at", { ascending: false })
124+
.limit(1)
125+
.single();
126+
127+
if (lastUsage) {
128+
const minutesSinceLastUsage = Math.floor(
129+
(Date.now() - new Date(lastUsage.created_at).getTime()) / (1000 * 60)
130+
);
131+
132+
if (minutesSinceLastUsage < rateLimit.cooldown) {
133+
const remainingMinutes = rateLimit.cooldown - minutesSinceLastUsage;
134+
return `Rate limit: Please wait ${remainingMinutes} more minute(s) before requesting Feedbot feedback for this assignment.`;
135+
}
136+
}
137+
}
138+
139+
// Check assignment total limit
140+
if (rateLimit.assignment_total) {
141+
// First get all submissions for this assignment
142+
const { data: assignmentSubmissions } = await serviceSupabase
143+
.from("submissions")
144+
.select("id")
145+
.eq("assignment_id", assignmentId);
146+
147+
if (assignmentSubmissions && assignmentSubmissions.length > 0) {
148+
const submissionIds = assignmentSubmissions.map((s: any) => s.id);
149+
150+
// Count usage across all submissions for this assignment
151+
const { count: assignmentUsageCount } = await serviceSupabase
152+
.from("llm_inference_usage")
153+
.select("*", { count: "exact", head: true })
154+
.in("submission_id", submissionIds);
155+
156+
if (assignmentUsageCount && assignmentUsageCount >= rateLimit.assignment_total) {
157+
return `Rate limit: Maximum number of Feedbot responses (${rateLimit.assignment_total}) for this assignment has been reached.`;
158+
}
159+
}
160+
}
161+
162+
// Check class total limit
163+
if (rateLimit.class_total) {
164+
const { count: classUsageCount } = await serviceSupabase
165+
.from("llm_inference_usage")
166+
.select("*", { count: "exact", head: true })
167+
.eq("class_id", classId);
168+
169+
if (classUsageCount && classUsageCount >= rateLimit.class_total) {
170+
return `Rate limit: Maximum number of Feedbot responses (${rateLimit.class_total}) for this class has been reached.`;
171+
}
172+
}
173+
174+
return null; // No rate limiting issues
175+
}
176+
102177
export async function POST(request: NextRequest) {
103178
try {
104179
const { testId } = await request.json();
@@ -134,7 +209,8 @@ export async function POST(request: NextRequest) {
134209
grader_results!inner (
135210
submissions!inner (
136211
id,
137-
class_id
212+
class_id,
213+
assignment_id
138214
)
139215
)
140216
`
@@ -166,8 +242,24 @@ export async function POST(request: NextRequest) {
166242
});
167243
}
168244

245+
// Use service role client for the update since users might not have update permissions
246+
const serviceSupabase = createServiceClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!);
247+
248+
// Check rate limiting if configured
249+
if (extraData.llm.rate_limit) {
250+
251+
const rateLimitError = await checkRateLimits(
252+
testResult,
253+
extraData.llm.rate_limit,
254+
serviceSupabase
255+
);
256+
if (rateLimitError) {
257+
throw new UserVisibleError(rateLimitError, 429);
258+
}
259+
}
260+
169261
const modelName = extraData.llm.model || process.env.OPENAI_MODEL || "gpt-4o-mini";
170-
const providerName = extraData.llm.provider || "anthropic";
262+
const providerName = extraData.llm.provider || "openai";
171263
const accountName = extraData.llm.account;
172264

173265
const chatModel = await getChatModel({
@@ -200,9 +292,7 @@ export async function POST(request: NextRequest) {
200292
}
201293
};
202294

203-
// Use service role client for the update since users might not have update permissions
204-
const serviceSupabase = createServiceClient(process.env.SUPABASE_URL!, process.env.SUPABASE_SERVICE_ROLE_KEY!);
205-
295+
206296
const { error: updateError } = await serviceSupabase
207297
.from("grader_result_tests")
208298
.update({ extra_data: updatedExtraData })

‎app/course/[course_id]/assignments/[assignment_id]/submissions/[submissions_id]/results/page.tsx

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ function LLMHintButton({ testId, onHintGenerated }: { testId: number; onHintGene
7676
case 404:
7777
errorMessage = "Test result not found or access denied";
7878
break;
79+
case 429:
80+
errorMessage = "Rate limit exceeded.";
81+
break;
7982
case 500:
8083
errorMessage = "Server error - please try again later";
8184
break;
@@ -1037,11 +1040,11 @@ export default function GraderResults() {
10371040
})}
10381041
</CardBody>
10391042
{hiddenExtraData?.pyret_repl && (
1040-
<Box mt={3}>
1041-
<PyretRepl testId={result.id} config={hiddenExtraData.pyret_repl} hidden />
1042-
</Box>
1043-
)}
1044-
</CardRoot>
1043+
<Box mt={3}>
1044+
<PyretRepl testId={result.id} config={hiddenExtraData.pyret_repl} hidden />
1045+
</Box>
1046+
)}
1047+
</CardRoot>;
10451048
})}
10461049
</CardRoot>
10471050
);

‎scripts/DatabaseSeedingUtils.ts

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1803,25 +1803,31 @@ export class DatabaseSeeder {
18031803
const isRecentlyDue = new Date(assignment.due_date) < now;
18041804

18051805
if (assignment.groups && assignment.groups.length > 0) {
1806-
// Group assignment - 75% chance to create a group submission
1806+
// Group assignment - 75% chance to create 1-3 submissions per assignment
18071807
assignment.groups.forEach((group) => {
18081808
if (Math.random() < 0.75) {
1809-
submissionsToCreate.push({
1810-
assignment: { ...assignment },
1811-
group,
1812-
isRecentlyDue
1813-
});
1809+
const numSubmissions = Math.floor(Math.random() * 3) + 1; // 1-3 submissions
1810+
for (let i = 0; i < numSubmissions; i++) {
1811+
submissionsToCreate.push({
1812+
assignment: { ...assignment },
1813+
group,
1814+
isRecentlyDue
1815+
});
1816+
}
18141817
}
18151818
});
18161819
} else {
1817-
// Individual assignment - 95% chance student submitted
1820+
// Individual assignment - 95% chance each student submits, with 1-4 submissions per student
18181821
students.forEach((student) => {
18191822
if (Math.random() < 0.95) {
1820-
submissionsToCreate.push({
1821-
assignment: { ...assignment },
1822-
student,
1823-
isRecentlyDue
1824-
});
1823+
const numSubmissions = Math.floor(Math.random() * 4) + 1; // 1-4 submissions per student
1824+
for (let i = 0; i < numSubmissions; i++) {
1825+
submissionsToCreate.push({
1826+
assignment: { ...assignment },
1827+
student,
1828+
isRecentlyDue
1829+
});
1830+
}
18251831
}
18261832
});
18271833
}
@@ -3832,7 +3838,7 @@ public class Entrypoint {
38323838
}
38333839
}`;
38343840

3835-
const submissionFileInserts = submissionData.map((submission, index) => ({
3841+
const submissionFileInserts = submissionData.map((submission: any, index) => ({
38363842
name: "sample.java",
38373843
contents: sampleJavaCode,
38383844
class_id: class_id,
@@ -3855,7 +3861,7 @@ public class Entrypoint {
38553861
}
38563862

38573863
// Prepare grader results for this chunk
3858-
const graderResultInserts = submissionData.map((submission, index) => ({
3864+
const graderResultInserts = submissionData.map((submission: any, index) => ({
38593865
submission_id: submission.id,
38603866
score: 5,
38613867
class_id: class_id,
@@ -3881,7 +3887,7 @@ public class Entrypoint {
38813887
}
38823888

38833889
// Prepare grader result tests (5 per submission: 2 regular + 3 with LLM hints) for this chunk
3884-
const graderResultTestInserts = graderResultData.flatMap((graderResult, index) => [
3890+
const graderResultTestInserts = graderResultData.flatMap((graderResult: any, index) => [
38853891
{
38863892
score: 5,
38873893
max_score: 5,
@@ -3997,7 +4003,7 @@ public class Entrypoint {
39974003

39984004
// Return the results for this chunk
39994005
return chunk.map((item, index) => ({
4000-
submission_id: submissionData[index].id,
4006+
submission_id: (submissionData[index] as any).id,
40014007
assignment: { id: item.assignment.id, due_date: item.assignment.due_date },
40024008
student: item.student,
40034009
group: item.group,

‎utils/supabase/DatabaseTypes.d.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,18 @@ export type GradebookColumnExternalData = {
99
creator: string;
1010
};
1111

12-
1312
export type PyretReplConfig = {
1413
initial_code?: string;
1514
initial_interactions?: string[];
1615
repl_contents?: string;
1716
};
1817

18+
export type LLMRateLimitConfig = {
19+
cooldown?: number;
20+
assignment_total?: number;
21+
class_total?: number;
22+
};
23+
1924
export type GraderResultTestExtraData = {
2025
llm?: {
2126
prompt: string;
@@ -25,6 +30,7 @@ export type GraderResultTestExtraData = {
2530
provider?: "openai" | "azure" | "anthropic";
2631
temperature?: number;
2732
max_tokens?: number;
33+
rate_limit?: LLMRateLimitConfig;
2834
type: "v1";
2935
};
3036
hide_score?: string;

0 commit comments

Comments
 (0)