fix

Author: edublancas

.github/workflows/ci-mac.yml

@@ -23,8 +23,10 @@ jobs:
       run: |
         pip install invoke
         pip install -e tests/assets/my_project
-        pip install https://github.com/ploomber/ploomber@cloud
+        pip install git+https://github.com/ploomber/ploomber@cloud
         pip install ".[dev]"
+        # one of airflow's dependencies is downgrading clic, which breaks ploomber
+        pip install click --upgrade
     - name: Test with pytest
       env:
         PLOOMBER_STATS_ENABLED: false

.github/workflows/ci-windows.yml

@@ -23,8 +23,10 @@ jobs:
       run: |
         pip install invoke
         pip install -e tests/assets/my_project
-        pip install https://github.com/ploomber/ploomber@cloud
+        pip install git+https://github.com/ploomber/ploomber@cloud
         pip install ".[dev]"
+        # one of airflow's dependencies is downgrading clic, which breaks ploomber
+        pip install click --upgrade
     - name: Test with pytest
       env:
         PLOOMBER_STATS_ENABLED: false

.github/workflows/ci.yml

@@ -28,8 +28,10 @@ jobs:
       run: |
         pip install invoke
         pip install -e tests/assets/my_project
-        pip install https://github.com/ploomber/ploomber@cloud
+        pip install git+https://github.com/ploomber/ploomber@cloud
         pip install ".[dev]"
+        # one of airflow's dependencies is downgrading clic, which breaks ploomber
+        pip install click --upgrade
     - name: Test with pytest
       env:
         CLIENT_ID: ${{ secrets.CLIENT_ID }}

environment.yml

@@ -0,0 +1,4 @@
+dependencies:
+- a
+- b
+- pip: 1

soopervisor.yaml

@@ -0,0 +1,3 @@
+
+env:
+  backend: backend-value

src/soopervisor/aws/batch.py

@@ -3,7 +3,6 @@
 """
 import json
 from pathlib import Path
-from functools import partial
 
 from ploomber.io._commander import Commander, CommanderStop
 from ploomber.util.util import requires
@@ -50,7 +49,7 @@ def _submit_dag(
     container_properties,
     region_name,
     cmdr,
-    call_api,
+    is_cloud,
 ):
     client = boto3.client('batch', region_name=region_name)
     container_properties['image'] = remote_name
@@ -65,14 +64,16 @@ def _submit_dag(
 
     cmdr.info('Submitting jobs...')
 
-    # docker.build moves to the env folder
-    params = json.loads(Path('../.ploomber-cloud').read_text())
-
-    out = api.runs_update(params['runid'], tasks)
+    if is_cloud:
+        # docker.build moves to the env folder
+        params = json.loads(Path('../.ploomber-cloud').read_text())
+        out = api.runs_update(params['runid'], tasks)
+    else:
+        out, params = None, None
 
     for name, upstream in tasks.items():
 
-        if out:
+        if is_cloud:
             ploomber_task = [
                 'python',
                 '-m',
@@ -98,16 +99,16 @@ def _submit_dag(
 
         cmdr.print(f'Submitted task {name!r}...')
 
-    api.runs_register_ids(params['runid'], job_ids)
-
-
-submit_dag_no_api = partial(_submit_dag, call_api=False)
-submit_dag_api = partial(_submit_dag, call_api=True)
+    if is_cloud:
+        api.runs_register_ids(params['runid'], job_ids)
 
 
 class AWSBatchExporter(abc.AbstractExporter):
     CONFIG_CLASS = AWSBatchConfig
-    SUBMIT_DAG = submit_dag_no_api
+
+    @classmethod
+    def _submit_dag(cls, *args, **kwargs):
+        return _submit_dag(*args, **kwargs, is_cloud=False)
 
     @staticmethod
     def _validate(cfg, dag, env_name):
@@ -153,18 +154,22 @@ def _export(cls, cfg, env_name, mode, until, skip_tests, ignore_git):
 
             cmdr.info('Submitting jobs to AWS Batch')
 
-            cls.SUBMIT_DAG(tasks=tasks,
-                           args=cli_args,
-                           job_def=pkg_name,
-                           remote_name=remote_name,
-                           job_queue=cfg.job_queue,
-                           container_properties=cfg.container_properties,
-                           region_name=cfg.region_name,
-                           cmdr=cmdr)
+            cls._submit_dag(tasks=tasks,
+                            args=cli_args,
+                            job_def=pkg_name,
+                            remote_name=remote_name,
+                            job_queue=cfg.job_queue,
+                            container_properties=cfg.container_properties,
+                            region_name=cfg.region_name,
+                            cmdr=cmdr)
 
             cmdr.success('Done. Submitted to AWS Batch')
 
 
+# TODO: add tests
 class CloudExporter(AWSBatchExporter):
     CONFIG_CLASS = CloudConfig
-    SUBMIT_DAG = submit_dag_api
+
+    @classmethod
+    def _submit_dag(cls, *args, **kwargs):
+        return _submit_dag(*args, **kwargs, is_cloud=True)

tests/test_cli.py

@@ -22,7 +22,7 @@ def monkeypatch_external(monkeypatch):
     monkeypatch.setattr(argo_export, 'Commander', CustomCommander)
     monkeypatch.setattr(airflow_export, 'Commander', CustomCommander)
     monkeypatch.setattr(batch, 'Commander', CustomCommander)
-    monkeypatch.setattr(batch, 'submit_dag', Mock())
+    monkeypatch.setattr(batch, '_submit_dag', Mock())
     mock_copy = Mock(wraps=docker.source.copy)
     monkeypatch.setattr(docker.source, 'copy', mock_copy)
     return mock_copy

tests/test_config.py

@@ -51,7 +51,7 @@ def test_get_backend_invalid_backend_value(tmp_empty):
     expected = ("Misconfigured environment: 'invalid' is not a "
                 "valid backend. backend must be one of: "
                 "'aws_batch', 'aws_lambda', 'argo_workflows', "
-                "'airflow', 'kubeflow', 'slurm'")
+                "'airflow', 'kubeflow', 'slurm', 'cloud'")
     assert str(excinfo.value) == expected
 
 

tests/test_slurm_export.py

@@ -171,7 +171,10 @@ def factory(value):
         mock.stdout = value
         return mock
 
-    run_mock = Mock(side_effect=[factory(b'0'), factory(b'1'), factory(b'2')])
+    run_mock = Mock()
+    run_mock.run = Mock(
+        side_effect=[factory(
+            b'0'), factory(b'1'), factory(b'2')])
     monkeypatch.setattr(commons, 'load_tasks', load_tasks_mock)
     monkeypatch.setattr(export, 'run', run_mock)