Skip to content

Commit 0a9b892

Browse files
manivoxel51manivoxel51
authored
Merge pull request #5600 from voxel51/feat/model-eval-subsets-app-2
Model Evaluation Scenarios
2 parents 9dc9ef5 + cdc6b84 commit 0a9b892

File tree

5 files changed

+365
-22
lines changed

5 files changed

+365
-22
lines changed

‎app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import Error from "./Error";
4949
import EvaluationIcon from "./EvaluationIcon";
5050
import EvaluationNotes from "./EvaluationNotes";
5151
import EvaluationPlot from "./EvaluationPlot";
52+
import EvaluationScenarioAnalysis from "./EvaluationScenarioAnalysis";
5253
import Status from "./Status";
5354
import { ConcreteEvaluationType } from "./Types";
5455
import {
@@ -79,6 +80,7 @@ export default function Evaluation(props: EvaluationProps) {
7980
notes = {},
8081
loadView,
8182
onRename,
83+
loadScenario,
8284
} = props;
8385
const theme = useTheme();
8486
const [expanded, setExpanded] = React.useState("summary");
@@ -537,6 +539,8 @@ export default function Evaluation(props: EvaluationProps) {
537539
? [classPerformance.findIndex((c) => c.id === activeFilter.value)]
538540
: undefined;
539541

542+
const labels = ["Accuracy", "F-score", "Precision", "Recall", "Support"];
543+
540544
return (
541545
<Stack spacing={2} sx={{ p: 2 }}>
542546
<Stack direction="row" sx={{ justifyContent: "space-between" }}>
@@ -1393,6 +1397,21 @@ export default function Evaluation(props: EvaluationProps) {
13931397
</Stack>
13941398
</AccordionDetails>
13951399
</Accordion>
1400+
<Accordion
1401+
disableGutters
1402+
sx={{ borderRadius: 1, "&::before": { display: "none" } }}
1403+
>
1404+
<AccordionSummary expandIcon={<ExpandMore />}>
1405+
Scenario Analysis
1406+
</AccordionSummary>
1407+
<AccordionDetails>
1408+
<EvaluationScenarioAnalysis
1409+
evaluation={evaluation}
1410+
data={data}
1411+
loadScenario={loadScenario}
1412+
/>
1413+
</AccordionDetails>
1414+
</Accordion>
13961415
</Stack>
13971416
)}
13981417
{mode === "info" && (
@@ -1675,6 +1694,7 @@ type EvaluationProps = {
16751694
id: string;
16761695
navigateBack: () => void;
16771696
loadEvaluation: (key?: string) => void;
1697+
loadScenario: (id?: string, subset?: string) => void;
16781698
onChangeCompareKey: (compareKey: string) => void;
16791699
compareKey?: string;
16801700
data: any;

‎app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationScenarioAnalysis.tsx

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
import {
2+
Button,
3+
CircularProgress,
4+
MenuItem,
5+
Select,
6+
Stack,
7+
Typography,
8+
} from "@mui/material";
9+
import React, { useEffect, useState } from "react";
10+
import EvaluationPlot from "./EvaluationPlot";
11+
import { isNullish } from "@fiftyone/utilities";
12+
import { usePanelEvent } from "@fiftyone/operators";
13+
import { usePanelId } from "@fiftyone/spaces";
14+
15+
const CONFIGURE_SCENARIO_ACTION = "@voxel51/scenario/configure_scenario";
16+
17+
export default function EvaluationScenarioAnalysis(props) {
18+
const { evaluation, data, loadScenario } = props;
19+
const { scenarios } = evaluation;
20+
const [selectedScenario, setSelectedScenario] = useState(null);
21+
const panelId = usePanelId();
22+
const promptOperator = usePanelEvent();
23+
const evaluationInfo = evaluation.info;
24+
const evaluationConfig = evaluationInfo.config;
25+
26+
const scenariosArray = scenarios ? Object.values(scenarios) : [];
27+
const isEmpty = scenariosArray.length === 0;
28+
29+
return (
30+
<Stack spacing={2}>
31+
<Stack direction="row" sx={{ justifyContent: "flex-end" }}>
32+
<Button
33+
variant="contained"
34+
onClick={() => {
35+
promptOperator(panelId, {
36+
params: {
37+
gt_field: evaluationConfig.gt_field,
38+
scenario_type: "custom_code",
39+
scenario_name: "test", // # TODO: Edit will pass current name
40+
},
41+
operator: CONFIGURE_SCENARIO_ACTION,
42+
prompt: true,
43+
// callback: (result, opts) => {
44+
// console.log("params", opts.ctx.params);
45+
// onSaveScenario({ subset: opts.ctx.params });
46+
// // TODO: save the subset
47+
48+
// // TODO: error handling
49+
// },
50+
});
51+
}}
52+
>
53+
Create New Scenario
54+
</Button>
55+
</Stack>
56+
{isEmpty ? (
57+
<Typography>No scenarios found!</Typography>
58+
) : (
59+
<Stack>
60+
<Typography>Select a scenario:</Typography>
61+
<Select
62+
size="small"
63+
onChange={(e) => {
64+
setSelectedScenario(e.target.value);
65+
}}
66+
>
67+
{scenariosArray.map((scenario) => {
68+
const { id, name } = scenario;
69+
return (
70+
<MenuItem value={id} key={id}>
71+
<Typography>{name}</Typography>
72+
</MenuItem>
73+
);
74+
})}
75+
</Select>
76+
</Stack>
77+
)}
78+
{selectedScenario && (
79+
<Scenario
80+
key={selectedScenario}
81+
id={selectedScenario}
82+
data={data}
83+
loadScenario={loadScenario}
84+
/>
85+
)}
86+
</Stack>
87+
);
88+
}
89+
90+
function Scenario(props) {
91+
const { id, data, loadScenario } = props;
92+
const [loading, setLoading] = useState(true);
93+
const [subset, setSubset] = useState("");
94+
const scenario = data?.[`scenario_${id}`];
95+
const subsets = scenario?.subsets || [];
96+
const subsetsData = scenario?.subsets_data || {};
97+
98+
useEffect(() => {
99+
if (!scenario) {
100+
loadScenario(id);
101+
}
102+
}, [scenario]);
103+
104+
if (!scenario) {
105+
return <CircularProgress />;
106+
}
107+
108+
return (
109+
<Stack>
110+
<Typography>Select a subset:</Typography>
111+
<Select
112+
size="small"
113+
onChange={(e) => {
114+
setSubset(e.target.value);
115+
}}
116+
>
117+
{subsets.map((subset) => {
118+
return (
119+
<MenuItem value={subset} key={subset}>
120+
<Typography>{subset}</Typography>
121+
</MenuItem>
122+
);
123+
})}
124+
</Select>
125+
{subset && <ScenarioModelPerformance data={subsetsData[subset]} />}
126+
</Stack>
127+
);
128+
}
129+
130+
const MODEL_PERFORMANCE_METRICS = [
131+
{ label: "F1 Score", key: "fscore" },
132+
{ label: "Precision", key: "precision" },
133+
{ label: "Recall", key: "recall" },
134+
{ label: "IoU", key: "iou" },
135+
{ label: "mAP", key: "mAP" },
136+
{ label: "Average Confidence", key: "average_confidence" },
137+
];
138+
139+
function ScenarioModelPerformance(props) {
140+
const { data } = props;
141+
142+
const theta = [];
143+
const r = [];
144+
for (const metric of MODEL_PERFORMANCE_METRICS) {
145+
const { label, key } = metric;
146+
const value = data[key];
147+
if (isNullish(value)) continue;
148+
theta.push(label);
149+
r.push(value);
150+
}
151+
152+
return (
153+
<EvaluationPlot
154+
data={[
155+
{
156+
type: "scatterpolar",
157+
r,
158+
theta,
159+
fill: "toself",
160+
},
161+
]}
162+
layout={{
163+
polar: {
164+
bgcolor: "#272727",
165+
radialaxis: {
166+
visible: true,
167+
},
168+
},
169+
}}
170+
/>
171+
);
172+
}

‎app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/index.tsx

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export default function NativeModelEvaluationView(props) {
3232
load_view,
3333
rename_evaluation,
3434
delete_evaluation,
35+
load_scenario,
3536
} = view;
3637
const {
3738
evaluations = [],
@@ -148,6 +149,9 @@ export default function NativeModelEvaluationView(props) {
148149
loadEvaluation={(key?: string) => {
149150
triggerEvent(load_evaluation, { key, id: keyToId[key as string] });
150151
}}
152+
loadScenario={(id?: string, subset?: string) => {
153+
triggerEvent(load_scenario, { id, subset });
154+
}}
151155
onChangeCompareKey={(compareKey) => {
152156
onChange("view.compareKey", compareKey);
153157
}}

‎app/packages/core/src/plugins/SchemaIO/components/RadioView.tsx

Lines changed: 77 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,41 @@ import {
55
RadioGroup as MUIRadioGroup,
66
Radio,
77
} from "@mui/material";
8-
import React from "react";
9-
import { HeaderView } from ".";
8+
import React, { useMemo } from "react";
9+
import { ButtonView, HeaderView } from ".";
1010
import { autoFocus, getComponentProps } from "../utils";
1111
import { useKey } from "../hooks";
1212

1313
export default function RadioView(props: RadioGroupProps) {
1414
const { schema, onChange, path, data } = props;
1515
const { view = {} } = schema;
16-
const { choices, label, description, orientation, readOnly } = view;
16+
const {
17+
choices,
18+
label,
19+
description,
20+
orientation,
21+
readOnly,
22+
variant = "default",
23+
} = view;
1724

25+
const useButtons = variant === "button";
1826
const [key, setUserChanged] = useKey(path, schema, data, true);
1927

28+
const radioGroupSx = useMemo(() => {
29+
const choicesLen = choices.length;
30+
if (useButtons && choicesLen > 0) {
31+
return {
32+
"> label": {
33+
width: choicesLen === 1 ? "100%" : "50%",
34+
margin: "0",
35+
padding: ".25rem",
36+
boxSizing: "border-box",
37+
},
38+
};
39+
}
40+
return { alignItems: "flex-start" };
41+
}, [useButtons, choices]);
42+
2043
return (
2144
<FormControl {...getComponentProps(props, "container")}>
2245
{(label || description) && (
@@ -34,35 +57,67 @@ export default function RadioView(props: RadioGroupProps) {
3457
onChange(path, value);
3558
setUserChanged();
3659
}}
37-
sx={{ alignItems: "flex-start" }}
60+
sx={radioGroupSx}
3861
row={orientation !== "vertical"}
3962
{...getComponentProps(props, "radioGroup")}
4063
>
41-
{choices.map(({ value, label, description, caption }, i) => (
64+
{choices.map(({ value, label, description, caption, icon }, i) => (
4265
<FormControlLabel
4366
key={value}
4467
value={value}
4568
control={
46-
<Radio
47-
disabled={readOnly}
48-
autoFocus={autoFocus(props)}
49-
{...getComponentProps(props, "radio")}
50-
/>
69+
useButtons ? (
70+
<ButtonView
71+
schema={{
72+
view: {
73+
label,
74+
icon,
75+
componentsProps: {
76+
container: { width: "100%" },
77+
button: {
78+
sx: {
79+
width: "100%",
80+
justifyContent: "flex-start",
81+
color: data === value ? "#FF6D04" : "auto",
82+
border:
83+
data === value ? "1px solid #FF6D04" : "none",
84+
},
85+
},
86+
icon: {
87+
sx: { color: data === value ? "#FF6D04" : "" },
88+
},
89+
},
90+
},
91+
}}
92+
onClick={() => {
93+
onChange(path, value);
94+
setUserChanged();
95+
}}
96+
/>
97+
) : (
98+
<Radio
99+
disabled={readOnly}
100+
autoFocus={autoFocus(props)}
101+
{...getComponentProps(props, "radio")}
102+
/>
103+
)
51104
}
52105
label={
53-
<HeaderView
54-
schema={{
55-
view: {
56-
label,
57-
description,
58-
caption,
59-
componentsProps: {
60-
header: getComponentProps(props, "radioHeader"),
106+
useButtons ? null : (
107+
<HeaderView
108+
schema={{
109+
view: {
110+
label,
111+
description,
112+
caption,
113+
componentsProps: {
114+
header: getComponentProps(props, "radioHeader"),
115+
},
61116
},
62-
},
63-
}}
64-
nested
65-
/>
117+
}}
118+
nested
119+
/>
120+
)
66121
}
67122
sx={{
68123
alignItems: "center",

0 commit comments

Comments
 (0)