From 954a6e6214dc9525971e37b0af9b5f0333d87303 Mon Sep 17 00:00:00 2001 From: Yash Bhardwaj Date: Mon, 5 Dec 2022 14:30:26 +0530 Subject: [PATCH] fix(optimus):remove old unsupported APIs (#217) * fix; remove old un supported APIs * fix: remove old un supported APIs * fix: add uploadToSCheduler api for optimus * fix: remove redundant event names from jobEvent proto enum * fix: import cycles * fix: optimus reserve old used enums * fix: optimus: event type name fix * fix: reserved an old used feild --- odpf/optimus/core/v1beta1/job_run.proto | 135 ++++------------------- odpf/optimus/core/v1beta1/job_spec.proto | 9 +- odpf/optimus/core/v1beta1/project.proto | 3 +- odpf/optimus/core/v1beta1/runtime.proto | 22 +--- 4 files changed, 24 insertions(+), 145 deletions(-) diff --git a/odpf/optimus/core/v1beta1/job_run.proto b/odpf/optimus/core/v1beta1/job_run.proto index 10aa6da90..9ba291f7f 100644 --- a/odpf/optimus/core/v1beta1/job_run.proto +++ b/odpf/optimus/core/v1beta1/job_run.proto @@ -6,8 +6,6 @@ import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "odpf/optimus/core/v1beta1/job_spec.proto"; -import "odpf/optimus/core/v1beta1/namespace.proto"; -import "odpf/optimus/core/v1beta1/project.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; option go_package = "github.com/odpf/proton/optimus"; @@ -27,22 +25,8 @@ option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = { base_path: "/api"; }; +// todo: should this not be called, scheduler service, as this will also, manage the runtime functions , like airflow events service JobRunService { - // GetJobTask provides task details specific to plugin used in a job - rpc GetJobTask(GetJobTaskRequest) returns (GetJobTaskResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/task" - }; - } - // RegisterInstance is an internal admin command used during task/hook execution - // to pull task/hook compiled configuration and assets. - rpc RegisterInstance(RegisterInstanceRequest) returns (RegisterInstanceResponse) { - option deprecated = true; - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/job/{job_name}/instance" - body: "*" - }; - } // JobRunInput is used to fetch task/hook compiled configuration and assets. rpc JobRunInput(JobRunInputRequest) returns (JobRunInputResponse) { option (google.api.http) = { @@ -50,60 +34,49 @@ service JobRunService { body: "*" }; } - // JobStatus returns the current and past run status of jobs - rpc JobStatus(JobStatusRequest) returns (JobStatusResponse) { - option (google.api.http) = { - get: "/v1beta1/project/{project_name}/job/{job_name}/status" - }; - } // JobRun returns the current and past run status of jobs on a given range rpc JobRun(JobRunRequest) returns (JobRunResponse) { option (google.api.http) = { get: "/v1beta1/project/{project_name}/job/{job_name}/run" }; } - // GetWindow provides the start and end dates provided a scheduled date - // of the execution window - rpc GetWindow(GetWindowRequest) returns (GetWindowResponse) { + // RegisterJobEvent notifies optimus service about an event related to job + rpc RegisterJobEvent(RegisterJobEventRequest) returns (RegisterJobEventResponse) { option (google.api.http) = { - get: "/v1beta1/window" + post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/event" + body: "*" }; } - // RunJob creates a job run and executes all included tasks/hooks instantly - // this doesn't necessarily deploy the job in db first - rpc RunJob(RunJobRequest) returns (RunJobResponse) { + // UploadToScheduler comiles jobSpec from database into DAGs and uploads the generated DAGs to scheduler + rpc UploadToScheduler(UploadToSchedulerRequest) returns (UploadToSchedulerResponse) { option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/run" + put: "/v1beta1/project/{project_name}/upload" body: "*" }; } } -message GetJobTaskRequest { +message UploadToSchedulerRequest { string project_name = 1; - string namespace_name = 2; - string job_name = 3; + optional string namespace_name = 2; } -message GetJobTaskResponse { - JobTask task = 1; +message UploadToSchedulerResponse { + bool status = 1; + string error_message = 2; } -message RegisterInstanceRequest { - reserved 3; + +message RegisterJobEventRequest { string project_name = 1; string job_name = 2; - google.protobuf.Timestamp scheduled_at = 4; + string namespace_name = 3; - string instance_name = 5; - InstanceSpec.Type instance_type = 6; - - // either set job_name if this is a scheduled execution - // or set jobrun_id if this is a manual triggered execution - // and not really registered as a valid job - string jobrun_id = 7; + JobEvent event = 4; } +message RegisterJobEventResponse {} + message JobRunInputRequest { reserved 3; @@ -120,27 +93,6 @@ message JobRunInputRequest { string jobrun_id = 7; } -message RegisterInstanceResponse { - ProjectSpecification project = 1; - NamespaceSpecification namespace = 4; - JobSpecification job = 2; - - InstanceSpec instance = 3; - - JobRunInputResponse context = 5; -} - -message JobStatusRequest { - reserved 3; - - string project_name = 1; - string job_name = 2; -} - -message JobStatusResponse { - repeated JobStatus statuses = 1; -} - message JobRunRequest { string project_name = 1; string job_name = 2; @@ -153,50 +105,6 @@ message JobRunResponse { repeated JobRun job_runs = 1; } -message GetWindowRequest { - google.protobuf.Timestamp scheduled_at = 1; - string size = 2; - string offset = 3; - string truncate_to = 4; - int32 version = 5; -} - -message GetWindowResponse { - google.protobuf.Timestamp start = 1; - google.protobuf.Timestamp end = 2; -} - -message RunJobRequest { - string project_name = 1; - string namespace_name = 2; - - // job specification order of execution is undefined - // attributes realted to schedule behaviour are ignored like interval, - // start_date, end_date, catchup, etc - repeated JobSpecification specifications = 3; -} - -message RunJobResponse {} - -// JobTask is part of a job that dictates main transformation -// each job has exactly one task -message JobTask { - string name = 1; - string description = 2; - string image = 3; - - message Destination { - string destination = 1; - string type = 2; - } - Destination destination = 4; - - message Dependency { - string dependency = 1; - } - repeated Dependency dependencies = 5; -} - message InstanceSpec { reserved 2, 4; @@ -240,11 +148,6 @@ message JobRunInputResponse { map secrets = 3; } -message JobStatus { - string state = 1; - google.protobuf.Timestamp scheduled_at = 2; -} - message TaskWindow { google.protobuf.Duration size = 1; google.protobuf.Duration offset = 2; diff --git a/odpf/optimus/core/v1beta1/job_spec.proto b/odpf/optimus/core/v1beta1/job_spec.proto index 5392a0066..f08e5031a 100644 --- a/odpf/optimus/core/v1beta1/job_spec.proto +++ b/odpf/optimus/core/v1beta1/job_spec.proto @@ -317,14 +317,11 @@ message JobConfigItem { message JobEvent { enum Type { TYPE_UNSPECIFIED = 0; + TYPE_SLA_MISS = 1; - TYPE_FAILURE = 2; - TYPE_SUCCESS = 3; - TYPE_RETRY = 4; - - TYPE_JOB_START = 5; + reserved 2 to 5; TYPE_JOB_SUCCESS = 6; - TYPE_JOB_FAIL = 7; + TYPE_FAILURE = 7; TYPE_TASK_RETRY = 8; TYPE_TASK_SUCCESS = 9; diff --git a/odpf/optimus/core/v1beta1/project.proto b/odpf/optimus/core/v1beta1/project.proto index df46536f6..80c5cd533 100644 --- a/odpf/optimus/core/v1beta1/project.proto +++ b/odpf/optimus/core/v1beta1/project.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package odpf.optimus.core.v1beta1; import "google/api/annotations.proto"; -import "odpf/optimus/core/v1beta1/namespace.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; option go_package = "github.com/odpf/proton/optimus"; @@ -46,8 +45,8 @@ service ProjectService { } message RegisterProjectRequest { + reserved 2; ProjectSpecification project = 1; - NamespaceSpecification namespace = 2 [deprecated = true]; } message RegisterProjectResponse { diff --git a/odpf/optimus/core/v1beta1/runtime.proto b/odpf/optimus/core/v1beta1/runtime.proto index acb667fd0..1a3bac5ec 100644 --- a/odpf/optimus/core/v1beta1/runtime.proto +++ b/odpf/optimus/core/v1beta1/runtime.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package odpf.optimus.core.v1beta1; import "google/api/annotations.proto"; -import "odpf/optimus/core/v1beta1/job_spec.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; option go_package = "github.com/odpf/proton/optimus"; @@ -32,16 +31,7 @@ service RuntimeService { body: "*" }; } - // RegisterJobEvent notifies optimus service about an event related to job - rpc RegisterJobEvent(RegisterJobEventRequest) returns (RegisterJobEventResponse) { - option (google.api.http) = { - post: "/v1beta1/project/{project_name}/namespace/{namespace_name}/job/{job_name}/event" - body: "*" - }; - } } -// rpc req/res - message VersionRequest { string client = 1; @@ -49,14 +39,4 @@ message VersionRequest { message VersionResponse { string server = 1; -} - -message RegisterJobEventRequest { - string project_name = 1; - string job_name = 2; - string namespace_name = 3; - - JobEvent event = 4; -} - -message RegisterJobEventResponse {} +} \ No newline at end of file