/
challenge_config.yaml
executable file
·145 lines (141 loc) · 4.56 KB
/
challenge_config.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
# If you are not sure what all these fields mean, please refer our documentation here:
# https://evalai.readthedocs.io/en/latest/configuration.html
title: Random Number Generator Challenge
short_description: Random number generation challenge for each submission
description: templates/description.html
evaluation_details: templates/evaluation_details.html
terms_and_conditions: templates/terms_and_conditions.html
image: logo.jpg
submission_guidelines: templates/submission_guidelines.html
leaderboard_description: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras egestas a libero nec sagittis.
evaluation_script: evaluation_script.zip
remote_evaluation: False
is_docker_based: False
start_date: 2019-01-01 00:00:00
end_date: 2099-05-31 23:59:59
published: True
leaderboard:
- id: 1
schema:
{
"labels": ["Metric1", "Metric2", "Metric3", "Total"],
"default_order_by": "Total",
"metadata": {
"Metric1": {
"sort_ascending": True,
"description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
},
"Metric2": {
"sort_ascending": True,
"description": "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
}
}
}
challenge_phases:
- id: 1
name: Dev Phase
description: templates/challenge_phase_1_description.html
leaderboard_public: False
is_public: True
is_submission_public: True
start_date: 2019-01-19 00:00:00
end_date: 2099-04-25 23:59:59
test_annotation_file: annotations/test_annotations_devsplit.json
codename: dev
max_submissions_per_day: 5
max_submissions_per_month: 50
max_submissions: 50
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
submission_meta_attributes:
- name: TextAttribute
description: Sample
type: text
required: False
- name: SingleOptionAttribute
description: Sample
type: radio
options: ["A", "B", "C"]
- name: MultipleChoiceAttribute
description: Sample
type: checkbox
options: ["alpha", "beta", "gamma"]
- name: TrueFalseField
description: Sample
type: boolean
required: True
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
allowed_submission_file_types: ".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy, .npz"
- id: 2
name: Test Phase
description: templates/challenge_phase_2_description.html
leaderboard_public: True
is_public: True
is_submission_public: True
start_date: 2019-01-01 00:00:00
end_date: 2099-05-24 23:59:59
test_annotation_file: annotations/test_annotations_testsplit.json
codename: test
max_submissions_per_day: 5
max_submissions_per_month: 50
max_submissions: 50
default_submission_meta_attributes:
- name: method_name
is_visible: True
- name: method_description
is_visible: True
- name: project_url
is_visible: True
- name: publication_url
is_visible: True
submission_meta_attributes:
- name: TextAttribute
description: Sample
type: text
- name: SingleOptionAttribute
description: Sample
type: radio
options: ["A", "B", "C"]
- name: MultipleChoiceAttribute
description: Sample
type: checkbox
options: ["alpha", "beta", "gamma"]
- name: TrueFalseField
description: Sample
type: boolean
is_restricted_to_select_one_submission: False
is_partial_submission_evaluation_enabled: False
dataset_splits:
- id: 1
name: Train Split
codename: train_split
- id: 2
name: Test Split
codename: test_split
challenge_phase_splits:
- challenge_phase_id: 1
leaderboard_id: 1
dataset_split_id: 1
visibility: 1
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 1
visibility: 3
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True
- challenge_phase_id: 2
leaderboard_id: 1
dataset_split_id: 2
visibility: 1
leaderboard_decimal_precision: 2
is_leaderboard_order_descending: True