Merge pull request #193 from fastlabel/add-dataset-training

add dataset training evaluation job

Author: takahiro-tamenishi

README.md

@@ -3011,6 +3011,14 @@ def get_training_jobs() -> list[dict]:
 
 ```
 
+#### Find Training job
+
+Find a single training job.
+
+```python
+task = client.find_training_job(id="YOUR_TRAINING_ID")
+```
+
 #### Response
 
 Example of two training jobs.
@@ -3070,6 +3078,110 @@ Example of two training jobs.
 ]
 ```
 
+### Execute training job
+
+Get training jobs.
+
+```python
+training_job = client.execute_training_job(
+    dataset_name="dataset_name",
+    base_model_name="fastlabel_object_detection_light",  // "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy"
+    epoch=300,
+    use_dataset_train_val=True
+)
+
+```
+
+### Get evaluation jobs
+
+Get evaluation jobs.
+
+```python
+def get_evaluation_jobs() -> list[dict]:
+    all_evaluation_jobs = []
+    offset = None
+    while True:
+        time.sleep(1)
+
+        evaluation_jobs = client.get_evaluation_jobs(offset=offset)
+        all_evaluation_jobs.extend(evaluation_jobs)
+
+        if len(evaluation_jobs) > 0:
+            offset = len(all_evaluation_jobs)
+        else:
+            break
+    return all_evaluation_jobs
+
+```
+
+#### Find Evaluation job
+
+Find a single evaluation job.
+
+```python
+evaluation_job = client.find_evaluation_job(id="YOUR_EVALUATION_ID")
+```
+
+#### Response
+
+Example of two evaluation jobs.
+
+```python
+
+{
+  id: "50873ea1-e008-48db-a368-241ca88d6f67",
+  version: 59,
+  status: "in_progress",
+  modelType: "builtin",
+  modelName: "FastLabel Object Detection Light - 汎用",
+  customModelId: None,
+  iouThreshold: 0.8,
+  confidenceThreshold: 0.4,
+  contentCount: 0,
+  gtCount: 0,
+  predCount: 0,
+  mAP: 0,
+  recall: 0,
+  precision: 0,
+  f1: 0,
+  confusionMatrix: None,
+  duration: 0,
+  evaluationSource: "dataset",
+  projects: [],
+  statuses: [],
+  tags: [],
+  datasetId: "deacbe6d-406f-4086-bd87-80ffb1c1a393",
+  dataset: {
+    id: "deacbe6d-406f-4086-bd87-80ffb1c1a393",
+    workspaceId: "df201d3c-af00-423a-aa7f-827376fd96de",
+    name: "sample-dataset",
+    createdAt: "2023-12-20T10:44:12.198Z",
+    updatedAt: "2023-12-20T10:44:12.198Z",
+  },
+  datasetRevisionId: "2d26ab64-dfc0-482d-9211-ce8feb3d480b",
+  useDatasetTest: True,
+  userName: "",
+  completedAt: None,
+  createdAt: "2023-12-21T09:08:16.111Z",
+  updatedAt: "2023-12-21T09:08:18.414Z",
+};
+
+```
+
+### Execute evaluation job
+
+Execute evaluation jobs.
+
+```python
+training_job = client.execute_evaluation_job(
+    dataset_name="DATASET_NAME",
+    model_name="fastlabel_object_detection_light",
+    // If you want to use the built-in model, select the following. "fastlabel_object_detection_light" or "fastlabel_object_detection_high_accuracy"
+    // If you want to use the custom model, please fill　out model name.
+    use_dataset_test=True,
+)
+
+```
 
 ### Execute endpoint
 

fastlabel/__init__.py

@@ -4231,6 +4231,124 @@ def get_training_jobs(
 
         return self.api.get_request(endpoint, params=params)
 
+    def execute_training_job(
+        self,
+        dataset_name: str,
+        base_model_name: str,
+        epoch: int,
+        dataset_revision_id: str = None,
+        use_dataset_train_val: bool = False,
+        instance_type: str = "ml.p3.2xlarge",
+        batch_size: int = None,
+        learning_rate: float = None,
+    ) -> list:
+        """
+        Returns a list of training jobs.
+        Returns up to 1000 at a time, to get more, set offset as the starting position
+        to fetch.
+        offset is the starting position number to fetch (Optional).
+        limit is the max number to fetch (Optional).
+        """
+        endpoint = "trainings"
+        payload = {
+            "datasetName": dataset_name,
+            "baseModelName": base_model_name,
+            "epoch": epoch,
+            "useDatasetTrainVal": use_dataset_train_val,
+            "datasetRevisionId": dataset_revision_id,
+            "instanceType": instance_type,
+            "batchSize": batch_size,
+            "learningRate": learning_rate,
+        }
+
+        return self.api.post_request(
+            endpoint,
+            payload={key: value for key, value in payload.items() if value is not None},
+        )
+
+    def find_training_job(self, id: str) -> list:
+        """
+        Returns training job.
+        id is id of training job (Required).
+        """
+
+        endpoint = f"trainings/{id}"
+
+        return self.api.get_request(
+            endpoint,
+        )
+
+    def get_evaluation_jobs(
+        self,
+        offset: int = None,
+        limit: int = 1000,
+    ) -> list:
+        """
+        Returns a list of training jobs.
+        Returns up to 1000 at a time, to get more, set offset as the starting position
+        to fetch.
+        offset is the starting position number to fetch (Optional).
+        limit is the max number to fetch (Optional).
+        """
+        if limit > 1000:
+            raise FastLabelInvalidException(
+                "Limit must be less than or equal to 1000.", 422
+            )
+        endpoint = "evaluations"
+        params = {}
+        if offset:
+            params["offset"] = offset
+        if limit:
+            params["limit"] = limit
+
+        return self.api.get_request(endpoint, params=params)
+
+    def find_evaluation_job(self, id: str) -> list:
+        """
+        Returns evaluation job.
+        id is id of evaluation job (Required).
+        """
+
+        endpoint = f"evaluations/{id}"
+
+        return self.api.get_request(
+            endpoint,
+        )
+
+    def execute_evaluation_job(
+        self,
+        dataset_name: str,
+        model_name: str,
+        iou_threshold: float = 0.5,
+        confidence_threshold: float = 0.4,
+        dataset_revision_id: str = None,
+        use_dataset_test: bool = False,
+        instance_type: str = "ml.p3.2xlarge",
+        batch_size: int = None,
+        learning_rate: float = None,
+    ) -> list:
+        """
+        Returns a list of training jobs.
+        Returns up to 1000 at a time, to get more, set offset as the starting position
+        to fetch.
+        offset is the starting position number to fetch (Optional).
+        limit is the max number to fetch (Optional).
+        """
+        endpoint = "evaluations"
+        payload = {
+            "modelName": model_name,
+            "datasetName": dataset_name,
+            "iouThreshold": iou_threshold,
+            "confidenceThreshold": confidence_threshold,
+            "datasetRevisionId": dataset_revision_id,
+            "useDatasetTest": use_dataset_test,
+        }
+
+        return self.api.post_request(
+            endpoint,
+            payload={key: value for key, value in payload.items() if value is not None},
+        )
+
     def execute_endpoint(
         self,
         endpoint_name: str,