Skip to content

Commit 468f6ac

Browse files
myungjinmyungjin
authored
feature: task rest api revision (#444)
REST API for task returns limited information, which makes it hard to visualize a job in GUI. The revised task rest api returns job's name and task's group association to improve GUI.
1 parent 85b5535 commit 468f6ac

File tree

23 files changed

+149
-155
lines changed

23 files changed

+149
-155
lines changed

‎api/job_components.partials.yml

+15-17
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ BasicJobInfo:
3434
properties:
3535
id:
3636
type: string
37+
name:
38+
type: string
3739
userId:
3840
type: string
3941
designId:
@@ -51,11 +53,13 @@ BasicJobInfo:
5153
format: int32
5254
default: 600 # 600s = 10min
5355
required:
56+
- name
5457
- designId
5558
- schemaVersion
5659
- codeVersion
5760
- backend
5861
example:
62+
name: mnist
5963
userId: "john"
6064
designId: "60d0d66716af12b787d9ef0a"
6165
schemaVersion: "1"
@@ -190,6 +194,8 @@ JobStatus:
190194
properties:
191195
id:
192196
type: string
197+
name:
198+
type: string
193199
state:
194200
$ref: '#/components/schemas/JobState'
195201
createdAt:
@@ -253,8 +259,8 @@ TaskInfo:
253259
type: string
254260
role:
255261
type: string
256-
type:
257-
$ref: '#/components/schemas/TaskType'
262+
groupAssociation:
263+
$ref: '#/components/schemas/GroupAssociation'
258264
key:
259265
type: string
260266
state:
@@ -266,20 +272,12 @@ TaskInfo:
266272
timestamp:
267273
type: string
268274
format: date-time
269-
270-
########################################
271-
# Task Type
272-
#
273-
# user: users bring compute resource and
274-
# dataset
275-
# system: system allocates compute
276-
# resource and configures
277-
# dataset access
278-
########################################
279-
TaskType:
280-
enum:
281-
- user
282-
- system
275+
required:
276+
- jobId
277+
- taskId
278+
- role
279+
- groupAssociation
280+
- state
283281

284282
########################################
285283
# Task Status
@@ -294,4 +292,4 @@ TaskStatus:
294292
type: string
295293
timestamp:
296294
type: string
297-
format: date-time
295+
format: date-time

‎cmd/controller/app/database/mongodb/task.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,15 @@ func (db *MongoService) CreateTasks(tasks []objects.Task, dirty bool) error {
5858
filter := bson.M{util.DBFieldJobId: task.JobId, util.DBFieldTaskId: task.TaskId}
5959
update := bson.M{
6060
"$set": bson.M{
61-
util.DBFieldRole: task.Role,
62-
util.DBFieldTaskType: task.Type,
63-
"config": cfgData,
64-
"code": task.ZippedCode,
65-
util.DBFieldComputeId: task.ComputeId,
66-
util.DBFieldTaskDirty: dirty,
67-
util.DBFieldTaskKey: task.Key,
68-
util.DBFieldState: openapi.READY,
69-
util.DBFieldTimestamp: time.Now(),
61+
util.DBFieldRole: task.Role,
62+
util.DBFieldGroupAssociation: task.JobConfig.GroupAssociation,
63+
"config": cfgData,
64+
"code": task.ZippedCode,
65+
util.DBFieldComputeId: task.ComputeId,
66+
util.DBFieldTaskDirty: dirty,
67+
util.DBFieldTaskKey: task.Key,
68+
util.DBFieldState: openapi.READY,
69+
util.DBFieldTimestamp: time.Now(),
7070
},
7171
}
7272
zap.S().Debugf("taskID %s is assigned compute %s", task.TaskId, task.ComputeId)

‎cmd/controller/app/job/builder.go

-2
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@ func (b *JobBuilder) buildTasks(templates map[string]*taskTemplate) []objects.Ta
261261
task := tmpl.Task
262262

263263
task.ComputeId = util.DefaultRealm
264-
task.Type = openapi.SYSTEM
265264
task.Key = util.RandString(taskKeyLen)
266265
task.JobConfig.GroupAssociation = associations
267266

@@ -286,7 +285,6 @@ func (b *JobBuilder) buildTasks(templates map[string]*taskTemplate) []objects.Ta
286285
task := tmpl.Task
287286

288287
task.ComputeId = dataset.ComputeId
289-
task.Type = openapi.SYSTEM
290288
task.Key = util.RandString(taskKeyLen)
291289
task.JobConfig.DatasetUrl = dataset.Url
292290
task.JobConfig.GroupAssociation = b.getGroupAssociationByGroup(roleName, groupName)

‎cmd/controller/app/job/job_state.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -386,11 +386,10 @@ func setTerminated(h *DefaultHandler) {
386386
}
387387

388388
func setTaskStateToTerminated(h *DefaultHandler) {
389-
// For any tasks in running state and created by system,
389+
// For any tasks in running state
390390
// set them terminated as they will be deleted by the system
391391
filter := map[string]interface{}{
392-
util.DBFieldState: openapi.RUNNING,
393-
util.DBFieldTaskType: openapi.SYSTEM,
392+
util.DBFieldState: openapi.RUNNING,
394393
}
395394
_ = h.dbService.UpdateTaskStateByFilter(h.jobId, openapi.TERMINATED, filter)
396395
}

‎cmd/controller/app/objects/task.go

+5-6
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,11 @@ import (
2727
)
2828

2929
type Task struct {
30-
JobId string `json:"jobid"`
31-
TaskId string `json:"taskid"`
32-
Role string `json:"role"`
33-
Type openapi.TaskType `json:"type"`
34-
Key string `json:"key"`
35-
ComputeId string `json:"computeid"`
30+
JobId string `json:"jobid"`
31+
TaskId string `json:"taskid"`
32+
Role string `json:"role"`
33+
Key string `json:"key"`
34+
ComputeId string `json:"computeid"`
3635

3736
// the following are config and code
3837
JobConfig JobConfig

‎cmd/flamectl/resources/job/job.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ type Params struct {
4141

4242
type CreateJobRequest struct {
4343
Id string `json:"id,omitempty"`
44+
Name string `json:"name"`
4445
UserId string `json:"userId,omitempty"`
4546
DesignId string `json:"designId"`
4647
SchemaVersion string `json:"schemaVersion"`
@@ -84,6 +85,7 @@ func createJobSpec(data []byte, jobFile string) (bool, openapi.JobSpec) {
8485

8586
jobSpec := openapi.JobSpec{
8687
Id: createJobRequest.Id,
88+
Name: createJobRequest.Name,
8789
UserId: createJobRequest.UserId,
8890
DesignId: createJobRequest.DesignId,
8991
SchemaVersion: createJobRequest.SchemaVersion,
@@ -209,10 +211,10 @@ func GetMany(params Params) error {
209211
}
210212

211213
table := tablewriter.NewWriter(os.Stdout)
212-
table.SetHeader([]string{"Job ID", "State", "created At", "started At", "ended At"})
214+
table.SetHeader([]string{"Job ID", "Name", "State", "created At", "started At", "ended At"})
213215

214216
for _, v := range infoList {
215-
table.Append([]string{v.Id, string(v.State), v.CreatedAt.String(), v.StartedAt.String(), v.EndedAt.String()})
217+
table.Append([]string{v.Id, v.Name, string(v.State), v.CreatedAt.String(), v.StartedAt.String(), v.EndedAt.String()})
216218
}
217219

218220
table.Render() // Send output
@@ -300,7 +302,8 @@ func Remove(params Params) error {
300302
url := restapi.CreateURL(params.Endpoint, restapi.DeleteJobEndPoint, uriMap)
301303

302304
jobStatus := openapi.JobStatus{
303-
Id: params.JobId,
305+
Id: params.JobId,
306+
State: openapi.READY, // state can't be empty; any state can be specified for remove
304307
}
305308

306309
code, body, err := restapi.HTTPDelete(url, jobStatus, "application/json")

‎cmd/flamectl/resources/task/task.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,10 @@ func GetMany(params Params) error {
9898
}
9999

100100
table := tablewriter.NewWriter(os.Stdout)
101-
table.SetHeader([]string{"Job ID", "Task ID", "Type", "State", "Timestamp"})
101+
table.SetHeader([]string{"Job ID", "Task ID", "State", "Timestamp"})
102102

103103
for _, v := range infoList {
104-
table.Append([]string{v.JobId, v.TaskId, string(v.Type), string(v.State), v.Timestamp.String()})
104+
table.Append([]string{v.JobId, v.TaskId, string(v.State), v.Timestamp.String()})
105105
}
106106

107107
table.Render() // Send output

‎docs/index.html

+13-13
Large diffs are not rendered by default.

‎examples/asyncfl_hier_mnist/job.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "asynchronous hierachical fl example",
23
"designId": "asyncfl_hier_mnist",
34
"schemaVersion": "1",
45
"codeVersion": "1",

‎examples/asyncfl_hier_mnist_lb/job.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "asynchronous hierarchical fl with coordinator",
23
"designId": "asyncfl_hier_mnist_lb",
34
"schemaVersion": "1",
45
"codeVersion": "1",

‎examples/distributed_training/job.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "distributed learning example",
23
"designId": "distributed_training",
34
"schemaVersion": "1",
45
"codeVersion": "1",
@@ -8,4 +9,4 @@
89

910
"dataSpecPath": "dataSpec.json",
1011
"modelSpecPath": "modelSpec.json"
11-
}
12+
}

‎examples/hier_mnist/job.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "hierarchical mnist example",
23
"designId": "hier_mnist",
34
"schemaVersion": "1",
45
"codeVersion": "1",
@@ -8,4 +9,4 @@
89

910
"dataSpecPath": "dataSpec.json",
1011
"modelSpecPath": "modelSpec.json"
11-
}
12+
}

‎examples/hybrid/job.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "hybrid fl example",
23
"designId": "hybrid",
34
"schemaVersion": "1",
45
"codeVersion": "1",

‎examples/medmnist/job.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "medical mnist example",
23
"designId": "medmnist",
34
"schemaVersion": "1",
45
"codeVersion": "1",
@@ -8,4 +9,4 @@
89

910
"dataSpecPath": "dataSpec.json",
1011
"modelSpecPath": "modelSpec.json"
11-
}
12+
}

‎examples/mnist/job.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "mnist example",
23
"designId": "mnist",
34
"schemaVersion": "1",
45
"codeVersion": "1",
@@ -8,4 +9,4 @@
89

910
"dataSpecPath": "dataSpec.json",
1011
"modelSpecPath": "modelSpec.json"
11-
}
12+
}

‎examples/parallel_experiment/job.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "parallel experiment example",
23
"designId": "parallel_experiment",
34
"schemaVersion": "1",
45
"codeVersion": "1",
@@ -10,4 +11,4 @@
1011

1112
"dataSpecPath": "dataSpec.json",
1213
"modelSpecPath": "modelSpec.json"
13-
}
14+
}

‎examples/syncfl_hier_coord_mnist/job.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "synchronous hierarchical fl with coordinator",
23
"designId": "syncfl_hier_coord_mnist",
34
"schemaVersion": "1",
45
"codeVersion": "1",

‎lib/python/flame/mode/enums.py

+16
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
# Copyright 2023 Cisco Systems, Inc. and its affiliates
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# SPDX-License-Identifier: Apache-2.0
16+
117
from enum import Flag, auto
218

319

‎pkg/openapi/model_job_spec.go

+24-14
Original file line numberDiff line numberDiff line change
@@ -25,30 +25,40 @@
2525

2626
package openapi
2727

28-
import "github.com/cisco-open/flame/pkg/openapi/constants"
29-
3028
// JobSpec - Job specification
3129
type JobSpec struct {
32-
Id string `json:"id,omitempty"`
33-
UserId string `json:"userId,omitempty"`
34-
DesignId string `json:"designId"`
35-
SchemaVersion string `json:"schemaVersion"`
36-
CodeVersion string `json:"codeVersion"`
37-
Priority JobPriority `json:"priority,omitempty"`
38-
Backend CommBackend `json:"backend"`
39-
MaxRunTime int32 `json:"maxRunTime,omitempty"`
40-
ModelSpec ModelSpec `json:"modelSpec,omitempty"`
30+
Id string `json:"id,omitempty"`
31+
32+
Name string `json:"name"`
33+
34+
UserId string `json:"userId,omitempty"`
35+
36+
DesignId string `json:"designId"`
37+
38+
SchemaVersion string `json:"schemaVersion"`
39+
40+
CodeVersion string `json:"codeVersion"`
41+
42+
Priority JobPriority `json:"priority,omitempty"`
43+
44+
Backend CommBackend `json:"backend"`
45+
46+
MaxRunTime int32 `json:"maxRunTime,omitempty"`
4147

4248
// Dataset specification
4349
DataSpec []RoleDatasetGroups `json:"dataSpec,omitempty"`
50+
51+
ModelSpec ModelSpec `json:"modelSpec,omitempty"`
4452
}
4553

4654
// AssertJobSpecRequired checks if the required fields are not zero-ed
4755
func AssertJobSpecRequired(obj JobSpec) error {
4856
elements := map[string]interface{}{
49-
constants.ParamDesignID: obj.DesignId,
50-
"schemaVersion": obj.SchemaVersion,
51-
"codeVersion": obj.CodeVersion,
57+
"name": obj.Name,
58+
"designId": obj.DesignId,
59+
"schemaVersion": obj.SchemaVersion,
60+
"codeVersion": obj.CodeVersion,
61+
"backend": obj.Backend,
5262
}
5363
for name, el := range elements {
5464
if isZero := IsZeroValue(el); isZero {

‎pkg/openapi/model_job_status.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@ package openapi
2727

2828
import (
2929
"time"
30-
31-
"github.com/cisco-open/flame/pkg/openapi/constants"
3230
)
3331

3432
// JobStatus - Job status
3533
type JobStatus struct {
3634
Id string `json:"id"`
3735

36+
Name string `json:"name,omitempty"`
37+
3838
State JobState `json:"state"`
3939

4040
CreatedAt time.Time `json:"createdAt,omitempty"`
@@ -49,8 +49,8 @@ type JobStatus struct {
4949
// AssertJobStatusRequired checks if the required fields are not zero-ed
5050
func AssertJobStatusRequired(obj JobStatus) error {
5151
elements := map[string]interface{}{
52-
"id": obj.Id,
53-
constants.ParamState: obj.State,
52+
"id": obj.Id,
53+
"state": obj.State,
5454
}
5555
for name, el := range elements {
5656
if isZero := IsZeroValue(el); isZero {

0 commit comments

Comments
 (0)